diff --git a/.gavel.yaml b/.gavel.yaml new file mode 100644 index 0000000..8e621c1 --- /dev/null +++ b/.gavel.yaml @@ -0,0 +1,5 @@ +lint: + ignore: + - file: internal/cli/** + rule: cobra-outside-cmd + source: uir diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f9bec0..08ca384 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: with: node-version-file: .node-version cache: pnpm - - uses: flanksource/deps@v1.0.49 + - uses: flanksource/deps@v1.0.50 with: tools: | task @@ -53,6 +53,7 @@ jobs: .bin/telltale webapp/dist pkg/agentinstall/java/assets + sdk/java/build/libs # test runs the suites in parallel shards. --ignore pkg keeps the root # module's own packages; the SDK modules are their own roots, so that shard @@ -79,6 +80,7 @@ jobs: java: true - shard: pkg args: test --framework go,ginkgo --ignore pkg/tracing ./pkg + java: true name: test (${{ matrix.shard }}) steps: - uses: actions/checkout@v4 @@ -96,7 +98,7 @@ jobs: with: name: build - run: chmod +x .bin/telltale - - uses: flanksource/gavel@78e109b65a8889fafd5fdfc74a14995b1330054f # v0.0.68 + - uses: flanksource/gavel@a3eede860078cf601387143eac421888611144f9 # v0.0.71 with: version: v0.0.68 args: ${{ matrix.args }} --timeout 15m --test-timeout 10m @@ -153,7 +155,7 @@ jobs: with: node-version-file: .node-version cache: pnpm - - uses: flanksource/deps@v1.0.49 + - uses: flanksource/deps@v1.0.50 with: tools: | task @@ -164,7 +166,7 @@ jobs: name: build # make lint runs gavel lint; the action leaves gavel on the PATH. - name: Install gavel - uses: flanksource/gavel@78e109b65a8889fafd5fdfc74a14995b1330054f # v0.0.68 + uses: flanksource/gavel@a3eede860078cf601387143eac421888611144f9 # v0.0.71 with: version: v0.0.68 args: version @@ -175,15 +177,44 @@ jobs: java-compatibility: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - java: ['8', '11', '17', '21'] + java: ['8', '11', '17', '21', '25'] steps: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 + id: fixture-java with: distribution: temurin java-version: ${{ matrix.java }} - - run: ./sdk/java/gradlew -p sdk/java test shadowJar + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '17' + - name: Test Java integrations + if: matrix.java == '17' + run: ./sdk/java/gradlew -p sdk/java test shadowJar war extensionApiJar exampleExtensionJar + - uses: actions/upload-artifact@v4 + if: always() && matrix.java == '17' + with: + name: java-integrations-17 + if-no-files-found: error + path: sdk/java/build/test-results/test + - name: Verify captures in the selected JVM without restart + env: + FIXTURE_JAVA: ${{ steps.fixture-java.outputs.path }}/bin/java + FIXTURE_JAVA_VERSION: ${{ matrix.java == '8' && '1.8' || matrix.java }} + run: | + ./sdk/java/gradlew -p sdk/java test \ + --tests com.flanksource.telltale.observer.NoRestartOtelTest \ + "-Dtelltale.gate.fixtureJava=$FIXTURE_JAVA" \ + "-Dtelltale.gate.expectedJava=$FIXTURE_JAVA_VERSION" + - uses: actions/upload-artifact@v4 + if: always() + with: + name: no-restart-java-${{ matrix.java }} + if-no-files-found: error + path: sdk/java/build/test-results/test release: needs: [build, test, test-web, lint, java-compatibility] diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a2d3cdb..be3f5e9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,7 +39,7 @@ jobs: with: node-version-file: .node-version cache: pnpm - - uses: flanksource/deps@v1.0.49 + - uses: flanksource/deps@v1.0.50 with: tools: | task @@ -56,6 +56,20 @@ jobs: env: RELEASE_TAG: ${{ steps.version.outputs.tag }} run: bash hack/release.sh + - name: Retain Java diagnostics artifacts + uses: actions/upload-artifact@v4 + with: + name: jvm-diagnostics-${{ steps.version.outputs.tag }} + path: sdk/java/build/libs/*.jar + if-no-files-found: error + retention-days: 90 + - name: Retain standalone diagnostics WAR + uses: actions/upload-artifact@v4 + with: + name: telltale-diag-${{ steps.version.outputs.tag }} + path: sdk/java/build/libs/telltale-diag.war + if-no-files-found: error + retention-days: 90 - name: Push SDK tags env: RELEASE_TAG: ${{ steps.version.outputs.tag }} diff --git a/.gitignore b/.gitignore index d423a37..d47db67 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,7 @@ sdk/java/.gradle/ sdk/java/build/ pkg/agentinstall/java/assets/* !pkg/agentinstall/java/assets/.gitkeep +/docs/plans/ +/docs/doc_build/ +/docs/.rspress/ +/victoria-metrics-data/ diff --git a/README.md b/README.md index 1dce349..30c2a0e 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,25 @@ telltale serve `telltale trace list` filters runs by `--query` (command, run id or `pid N`), `--status` (`running`, `succeeded`, `failed`, `unfinished`), `--kind`, `--since` and `--limit`. The 20 newest runs are kept. Set `-Phttp.har=` on any command to capture telltale's own outbound HTTP as a HAR archive. `telltale trace run --help` lists every trace kind with its params. +### Following armed captures + +Every run form, including definitions and presets, accepts `--follow` to print records from every capture it arms. Use `--format ndjson` for one complete JSON record per line or `--format ansi` (`pretty` also works) for readable terminal output. NDJSON stdout contains records only; command output and the final summary go to stderr. Each record identifies its run, capture kind, stream, local sequence and timestamp, and keeps the native fields under `record` so payload fields cannot collide with that identity. + +```sh +telltale trace run --follow --format ndjson -- curl -sS https://example.com +telltale run --definition checkout --follow --format ansi -- ./service +telltale run --definition checkout --tui -- ./service +telltale run --definition checkout --ui --sort timestamp -- ./service +``` + +`--tui` opens a full-screen monitor with a tab per enabled kind and separate **Stdout** and **Stderr** tabs. Switch tabs with Tab/Shift+Tab; use arrows or j/k to scroll, `s` to change record order, `n`/`p` to page, Enter for details, space to pause, `r` to refresh, and `q` to quit. Each kind shows its capture summary above schema-based record columns. Command output is stored separately in `stdout.log` and `stderr.log`; Home/End and `n`/`p` browse large output in bounded blocks. The viewer owns terminal input and remains open after capture completion. `--ui` opens a loopback web server showing that run's armed captures and remains available until Ctrl+C. + +The normal server uses the same paged viewer and `GET /api/traces/{run}/records` API. `kind` and `stream` narrow the armed captures, `sort=seq|timestamp` selects ascending order, `limit` defaults to 100 (maximum 500), `from`/`until` bound record time, and `cursor` continues a frozen sequence window. `follow=1` streams SSE; add `format=ndjson` for NDJSON. SSE reconnects carry independent stream positions through `Last-Event-ID`. Sequences are local to each stream: live arrivals interleave while preserving each stream's order. Timestamp ordering applies to pages and reorderable views; late live terminal records print immediately. + +### Replaying recorded traces + +`telltale trace replay [run-id...]` reads saved captures without executing their commands. Add `--ui` for a local viewer with a run selector, `--tui` for an interactive terminal, or `--format ndjson --sort timestamp` for merged records. New IDs use up to nine dash-free hostname characters followed by a compact UTC timestamp, such as `buildhost20261107T123000Z`; same-second collisions receive `_2`, `_3`, and subsequent suffixes. See [replay commands and ID format](docs/replay.md). + ### One kind at a time, or a running process `telltale run ` arms a single trace kind, its params as flags. There is one subcommand for every kind in the registry, which `pkg/traces` fills in `init()` before the CLI is built, so a kind registered by another package's `init()` gets its own command with no CLI change. `telltale run --help` shows what the kind records, its flags with their defaults and choices, the variables it exports and examples. @@ -37,27 +56,30 @@ Flag types, descriptions, defaults, choices, and required inputs come from the k telltale run http --seam proxy --mode mitm --hosts '*.example.com' -- curl -sS https://api.example.com telltale run sql --seam proxy --env ORDERS_DB_URL --params -- go test ./... telltale run stacks --every 10s -- ./server # dump the server's stacks every 10s -telltale run process_samples --pid 1234 --facets ports --duration 1m +telltale run process --pid 1234 --facets sockets --duration 1m ``` -Kinds that observe a process tree (`process_tree`, `process_samples`, `stacks`, `goprof`) can attach to a running process with `--pid`, recording until Ctrl+C, `--duration`, or the process exits; the process is left running. The `http` and `sql` proxy seams work on a command they are pointed at; their agent seams attach to instrumented processes. `stacks` and `goprof` read at checkpoints: every `--every`, and once as an attached run stops. A checkpoint that fails fails the run, listed under `checkpointErrors`. +Kinds that observe a process tree (`process`, `stacks`, `goprof`) can attach to a running process with `--pid`, recording until Ctrl+C, `--duration`, or the process exits; the process is left running. The `http` and `sql` proxy seams work on a command they are pointed at; their agent seams attach to instrumented processes. `stacks` and `goprof` read at checkpoints: every `--every`, and once as an attached run stops. A checkpoint that fails fails the run, listed under `checkpointErrors`. The built-in [trace definitions](#trace-definitions) are named runs of several kinds, each a top-level command: | Command | Records | |---|---| | `telltale ps ` | the process tree with each process's CPU, memory, disk IO, open files, sockets, ports, connections and environment | +| `telltale clicky ` | everything a clicky process's [clicky/telltale](https://github.com/flanksource/clicky/tree/main/telltale) agent observes: its executions, entity operations, tasks, commons HTTP and commons logs; a process without the agent is refused | | `telltale gops ` | the goroutines and heap profile of a Go process running a [gops agent](https://github.com/google/gops), every 30s; the agent is found from its port file (`$GOPS_CONFIG_DIR/`), and a process without one is refused before the run starts | Both also take `-- command [args...]`. They are `pkg/definition/builtin/ps.yaml` and `gops.yaml`. `telltale serve` mounts the runs of the data dir at `/api/traces`, and the web UI starts, stops, filters and shows them, live while they run: +The overview at `/` combines OpenTelemetry trace summaries and recorded trace runs in one list, newest first. Use **All activity**, **OpenTelemetry traces**, or **Recorded runs** and filter by service, command, ID, status, or capture kind. The trace source and time window apply to OpenTelemetry results; recorded runs show the retained run history. Filters stay in the URL. Select a trace to open its span view, or a run to open its captures and process header; existing `/?run=` links select that run. **Record a run** opens the existing recording form. A source failure is shown explicitly while the other results remain available. + | Route | Does | |---|---| | `GET /api/traces?q=&status=&kind=&since=&limit=` | Lists runs, newest first (`since` is RFC 3339) | | `GET /api/traces/start` | The start form: a JSON schema and its defaults | -| `POST /api/traces` | Starts `{"command": "...", "tree": {"facets": false}, "traces": {"http": {"enabled": true, "target": {"seam": "proxy"}, "mode": "mitm"}}}`, or a stored definition: `{"definition": "checkout", "command": "...", "duration": "20s"}` (`command`, `pid` or neither for a host run; `traces` and `definition` exclude each other). The command runs with `sh -c` in the work dir and its output goes to `output.log` in the run's store. A refused request is a 400 `{"errors": [{"instancePath", "message"}]}` | +| `POST /api/traces` | Starts `{"command": "...", "tree": {"facets": false}, "traces": {"http": {"enabled": true, "target": {"seam": "proxy"}, "mode": "mitm"}}}`, or a stored definition: `{"definition": "checkout", "command": "...", "duration": "20s"}` (`command`, `pid` or neither for a host run; `traces` and `definition` exclude each other). The command runs with `sh -c` in the work dir and its output goes to `stdout.log` and `stderr.log` in the run's store, named by the optional `output` metadata. A refused request is a 400 `{"errors": [{"instancePath", "message"}]}` | | `/api/traces/definitions/...` | The trace definitions: see [Trace definitions](#trace-definitions) | | `DELETE /api/traces/` | Stops a run the server is running | | `/api/traces//api/v1/...` | The run's records, exports and checkpoints, live from its running captures | @@ -66,6 +88,8 @@ Both also take `-- command [args...]`. They are `pkg/definition/builtin/ps.yaml` See [SQL Server and JVM captures](docs/sqlserver-jvm.md) for XEvents, historical and live deadlocks, local JFR, and Java agent installation over SSH or Kubernetes. +See [JVM diagnostics](docs/jvm-diagnostics-port.md) for JMX, OGNL/Groovy, watch/trace, class redefinition, hot extensions and the standalone diagnostic WAR. + ### Trace definitions A `TraceDefinition` says what telltale traces, apart from any one run of it: a run adds the command or pid, its duration and, optionally, how often its checkpointed captures are read. It is a CRD-compatible resource (`crds/telltale.flanksource.com_tracedefinitions.yaml`, generated by `make generate`), so it `kubectl apply`s cleanly, but telltale reads it from files only: `.telltale/traces/.yaml` in the working directory, then `~/.config/telltale/traces/.yaml`, then the read-only built-ins. The file name is its `metadata.name`. @@ -129,22 +153,24 @@ telltale run --definition ./logs-and-queries.yaml --duration 5m # no command o ### Prometheus metrics -`telltale run prometheus` scrapes Prometheus metrics endpoints into a Prometheus that telltale manages, one per data dir. Run alone it needs no command and scrapes until Ctrl+C or `--duration`: +`telltale run prometheus` configures VictoriaMetrics to scrape Prometheus exporters. Each data directory has its own instance. Run alone it needs no command and scrapes until Ctrl+C or `--duration`. ```sh telltale run prometheus --endpoints http://localhost:8080/metrics --interval 5s --duration 1m telltale run prometheus --endpoints http://localhost:9100/metrics,http://localhost:9090/metrics --labels env=dev +telltale run prometheus --endpoints port-forward://metrics-exporter.monitoring:9100/metrics --duration 1m +telltale run prometheus --endpoints proxy://metrics-exporter.monitoring:9100/metrics --duration 1m ``` -The first capture downloads Prometheus 3.5.0 through deps and starts it on a free loopback port, with its state and TSDB under `/services/prometheus`. Each capture writes its endpoints as a file_sd file (`/prometheus/targets/.json`) that the `file_sd` job reads. Every endpoint is one target with its own `--interval` (at least 1s, default 15s) and `--timeout` (at most the interval, default the shorter of 10s and the interval). Its query params go along with each scrape. An endpoint must be an absolute http(s) URL without credentials or a fragment. The capture records one row per completed scrape: the endpoint, its health, duration, error and the number of samples scraped. A failed scrape is a row with its error. A Prometheus that stops answering fails the capture. +The first capture installs VictoriaMetrics and starts it on a free loopback port. Every endpoint has its own `--interval` (at least 1s, default 15s) and `--timeout` (at most the interval, default the shorter of 10s and the interval). Endpoints can be direct HTTP(S) URLs, Kubernetes port-forwards (`portforward://` or `port-forward://`), or Kubernetes API proxies (`proxy://`). Use URLs without embedded credentials or fragments. Telltale uses its Kubernetes identity for the inline endpoints and passes exporter query parameters along with each scrape. The capture reports target health, and VictoriaMetrics stores the samples directly for 15 days. An unreachable VictoriaMetrics instance fails the capture. -The samples stay in the TSDB after the capture ends, labelled `telltale_run`, `telltale_stream` and `telltale_endpoint` plus `--labels`. The run shows the Prometheus URL and the PromQL selector of its series. `telltale serve` lists that Prometheus as the read-only `telltale-prometheus` connection, so the Query page can run PromQL against it without a connection file: +Samples carry `telltale_run`, `telltale_stream`, and `telltale_endpoint` plus `--labels`. The run shows the VictoriaMetrics URL and the PromQL selector for its series. The Query page exposes the read-only `telltale-prometheus` connection. You can also query the printed VictoriaMetrics URL directly. ```sh curl -sG http://localhost:/api/v1/query --data-urlencode 'query={telltale_run="run-2026-10-31T10-00-00Z"}' ``` -The Prometheus is telltale's sub-process. The run's captures share it, and it stops when the run ends. The next run starts it again on the same TSDB and port, so earlier runs' samples are queryable again while a capture runs. A Prometheus that another telltale process already supervises in that data dir is reused and left to that process. +To keep samples queryable between captures, start `telltale serve --prometheus` first and use the same `--data-dir` for both commands. Each run removes its own targets when it ends. Stopping `serve` shuts down the instance it started. A standalone run starts and stops its own instance when needed. Both modes reopen the same VictoriaMetrics database on restart. On upgrade, VictoriaMetrics starts with new samples and the previous Prometheus database stays under `services/prometheus`. ### Process groups: `telltale top` @@ -157,17 +183,23 @@ telltale top --all # also the processes no tool hold telltale top --net # with network bytes (nettop, darwin) telltale top --once --format json # one frame, once CPU can be rated telltale top --record --duration 10m # record into the data dir -telltale trace list --kind process_groups +telltale trace list --kind process ``` -Use `--group-by repo`, `--group-by worktree`, or `--group-by claude-session` to collect tool groups into expandable buckets; `--group-by tree` is the default process tree. Repository grouping combines a main checkout and its linked worktrees using their shared Git directory, while worktree grouping keeps each checkout separate and shows its branch. Claude session grouping uses the full session ID and displays its title when available; tools inherit their ancestor's session until another agent session begins. Missing metadata stays visible under `Unattributed`. Bucket totals count each tool group's own usage once and preserve unknown measurements. +Use `--group-by repo`, `--group-by worktree`, `--group-by claude-session`, or `--group-by slots` to collect tool groups into expandable buckets; `--group-by tree` is the default process tree. Slot grouping puts each gavel process in the gavel slot pool it holds (its repository's project pool before the global one), with the tests it runs under it. A gavel holding no slot goes under ` (waiting)` for the pool it is queued on. Repository grouping combines a main checkout and its linked worktrees using their shared Git directory, while worktree grouping keeps each checkout separate and shows its branch. Claude session grouping uses the full session ID and displays its title when available; tools inherit their ancestor's session until another agent session begins. Missing metadata stays visible under `Unattributed`. Bucket totals count each tool group's own usage once and preserve unknown measurements. Press `b` in the terminal to cycle grouping without restarting sampling. The web `/top` page offers the same modes through **Group by**, persists the choice in `topGroupBy`, and applies it to tables and trends. Recordings retain raw tool groups, so grouping can change during playback; older recordings without shared Git directory metadata remain unattributed in repository mode, while their recorded checkout paths still support worktree grouping. For example, `telltale top --group-by repo --once --format json` prints one grouped frame, and `telltale top --group-by worktree --record --duration 1m` records raw groups while showing worktree buckets in an interactive terminal. -In a terminal, `telltale top` takes the whole screen. The header shows the host's CPU split between processes (user), the kernel (system) and idle time and load, bars for memory and swap, and the disks' read and write IOPS and throughput. The title ends with telltale's own overhead: the CPU it and the commands it runs each tick (`ps`, `lsof`, `launchctl`) use, and how long a tick takes; JSON frames carry it as `overhead`. The groups fill the rest of the window as a tree, and the origin and context columns appear only when the window is wide enough. Keys: `↑↓` (or `j`/`k`, `g`/`G`, PgUp/PgDn) move; `←→` and space fold; `enter` lists the selected group's processes below the tree; `s` cycles the sort column and `S` reverses it; `/` filters by name, command, repo, owner or launchd label; `o` shows the processes no tool holds; `n` reads network bytes; `q` quits. When stdout is not a terminal, or with `--once`, it prints one frame instead. +In a terminal, `telltale top` takes the whole screen. The header shows the host's CPU split between processes (user), the kernel (system) and idle time and load, bars for memory and swap, and the disks' read and write IOPS and throughput. The title ends with telltale's own overhead: the CPU it and the commands it runs each tick (`ps`, `lsof`, `launchctl`) use, and how long a tick takes; JSON frames carry it as `overhead`. The groups fill the rest of the window as a tree, and the origin and context columns appear only when the window is wide enough. Keys: `↑↓` (or `j`/`k`, `g`/`G`, PgUp/PgDn) move; `←→` and space fold; `enter` lists the selected group's own processes in the tree, nested by parent process, with a process whose parent is outside the group marked `↰` and a process a gavel task runs under that task (a group's processes are otherwise only counted: a process no rule matches joins its nearest grouped ancestor); `f` opens the pane below the tree that lists the selected group's processes and those of every group under it; `s` cycles the sort column and `S` reverses it; `/` filters by name, command, repo, owner or launchd label; `o` shows the processes no tool holds; `n` reads network bytes; `q` quits. When stdout is not a terminal, or with `--once`, it prints one frame instead. + +Gavel groups also show what their gavel process is doing, read from the status each running `gavel test`, `gavel lint` and `gavel fixtures` publishes (via `github.com/flanksource/gavel/sdk/status`). The gavel row shows the machine-wide slots it holds (`●global`) or waits for (`⧗acme`). Under it, each task group with work gets its own row with a progress bar, finished of total and failures (for example `packages [██▓▒░░░░] 12/40 ✗2`; █ finished, ▓ failed, ▒ running, ░ pending), and each task it is running gets a row under that (`▶ go test ./pkg/a 42s`). The gavel rule is `skipAncestry`: the group of the process a task runs moves under that task's row, found by pid, even when the process is detached and not under gavel in the process tree. Groups no task claims stay under the gavel. The header's `SLT` line adds up every pool those processes use: held of limit and how many wait. The file is polled each second until the process opens its status socket, after its first 30 seconds, and is streamed from then on. JSON frames carry each group's `gavel` status and the frame's `slots`. Run `gavel slots` for the same pools from gavel itself. CPU hogs are highlighted when their CPU use exceeds 25% of all logical cores: on an 8-core host, the threshold is strictly greater than 200% in the per-core CPU column. In the terminal, open the process list with `enter`, choose a process with `[` / `]`, then press `x` to send SIGTERM or `r` to set its nice value to 10; `y` confirms and `n` / Escape cancels. The web process table offers **Terminate** and **Renice**, with a confirmation showing the PID and command and a nice value from -20 to 19 (higher means lower priority). These controls act on one live process, require OS permissions, and reject a process whose start-qualified identity changed. Recordings remain read-only. +Memory is split into **RSS** (resident physical memory), **Compressed** (`CMPRS` in the terminal), and **Swap**, with separate sortable columns in both interfaces (`--sort compressed` or `--sort swap`). Linux reports each process's disk swap. macOS reports its compressed and swapped-out pages together in Compressed, in bytes before compression, and leaves per-process Swap unavailable (`–`); the host's swap bar still reports actual disk swap. Unsupported or unreadable measurements stay unavailable rather than becoming zero; read failures are exposed as `memoryError` in JSON and the memory cells' tooltips. macOS compressed memory is read using the system `top` command each tick. + +A process is a memory hog when its RSS or compressed memory exceeds 25% of physical RAM, or its disk swap exceeds 25% of swap capacity. CPU and memory hogs stay pinned ahead of the selected sort for 120 seconds after their last excessive sample. Their groups and ancestor branches rise together, preserving the tree; otherwise hidden other-process hogs become visible automatically. Each resource expires independently, ongoing hogging renews its pin, and pins follow start-qualified process identities so a reused PID cannot inherit one. Recordings retain the expiry timestamps in each usage's `hogs` map, evaluated against the recorded sample time during playback. + `POST /api/top/action` accepts an `application/json` body such as `{"key":"123@1793000000","action":"terminate"}` or `{"key":"123@1793000000","action":"renice","nice":10}`. Use the key returned by the live processes endpoint. Success is 204; invalid input is 400, stale identity is 409, and OS permission failures are 403. Cross-origin browser requests are refused. PID 1 and the Telltale server itself cannot be targeted. Each group also says where its process came from (`origin`): @@ -177,9 +209,9 @@ Each group also says where its process came from (`origin`): A brew service no rule matches becomes a `service` group of its own, so `postgresql@17` holds its backends. A program inside an app bundle in an `Applications` folder (`/Applications`, `/System/Applications`, `~/Applications` or a folder in one) that no rule matches joins its app's `app` group, under `applications` unless a tool started it, so every helper of an Electron app adds up in one place. The telltale reading the host is a `telltale` group of its own, with what it keeps running such as `nettop`. The processes no tool holds sit under `other` (the user's) and `system`, grouped by program. The kernel itself (`kernel_task`) is not a process `ps` lists, so its time shows as the host's kernel CPU share. -Nothing is kept unless you pass `--record`, which starts a host run: a run with neither a command nor a process, arming the `process_groups` trace kind (also `telltale run process_groups`). The screen then shows the frames as the run's store commits them. On darwin, network bytes come from `nettop`, which uses more than a core however long its interval, so it runs only with `--net`; other platforms refuse `--net`. Socket and connection counts come from `lsof` on every platform. +Nothing is kept unless you pass `--record`, which starts a host run with `process` on the host seam (also `telltale run process --seam host`). The screen derives complete group frames from startup grouping tags and numeric samples. On darwin, network bytes come from `nettop`, which uses more than a core however long its interval, so it runs only with `--net`; other platforms refuse `--net`. Files and receiving sockets have independent delta events. Established TCP connections are counted as unique remote endpoints per process, with inferred inbound, inferred outbound, or unknown direction. -The built-in rules (`pkg/tracing/groups/catalog.yaml`) cover claude, codex, cmux, captain, gavel, go test, vitest, playwright, agent-browser, lightpanda, chrome, vite, make, pnpm/npm and gopls. Rules in `./.telltale/groups.yaml`, else the global config dir's `groups.yaml`, or in the file `--catalog` names, go ahead of the built-in ones. A rule with a built-in rule's name replaces it, and `disabled: true` drops it: +The built-in rules (`pkg/tracing/groups/catalog.yaml`) cover claude, codex, cmux, captain, gavel, go test, vitest, playwright, agent-browser, lightpanda, chrome, vite, make, pnpm/npm and gopls. Rules in `./.telltale/groups.yaml`, else the global config dir's `groups.yaml`, or in the file `--catalog` names, go ahead of the built-in ones. A rule with a built-in rule's name replaces it, and `disabled: true` drops it. `skipAncestry: true` marks a runner whose published tasks claim their processes' groups by pid before ancestry does; the built-in gavel rule has it: ```yaml groups: @@ -194,7 +226,7 @@ groups: disabled: true ``` -`telltale serve` serves the latest live frame, with the host's use, at `GET /api/top?all=true&net=true`, and keeps reading it every 2s while it is asked for. `GET /api/top/processes?name=` returns the processes running a program, `?pid=` a process and every process under it, and `?q=` the processes whose name, command or pid holds the text. The web UI's Top page (`/top`) shows the groups live with gauges for the host, and recorded `process_groups` runs over time. ⌘K (Ctrl+K) anywhere in the web UI searches processes by name, command or pid, and trace runs by id, command, pid, status or capture kind. `/top/` and `/top/` show one program's or one process tree's processes, with gauges and their history. `GET /api/top/facets?pid=` reads a process the latest frame listed, and answers 404 for one it did not. It returns the process's context with those names, its environment with its secrets masked as stored traces mask them, and its open files, sockets, listening ports and connections: the facets `process_samples` and `process_tree` record. `/top/` shows these facets for its process every 5s, and clicking a process in the table shows that process's. +`telltale serve` serves the latest live frame, with the host's use, at `GET /api/top?all=true&net=true`, and keeps reading it every 2s while it is asked for. `GET /api/top/processes?name=` returns the processes running a program, `?pid=` a process and every process under it, and `?q=` the processes whose name, command or pid holds the text. The web UI's Top page (`/top`) shows the groups live with gauges for the host, and recorded `process` runs over time. ⌘K (Ctrl+K) anywhere in the web UI searches processes by name, command or pid, and trace runs by id, command, pid, status or capture kind. `/top/` and `/top/` show one program's or one process tree's processes, with gauges and their history. `GET /api/top/facets?pid=` reads a process the latest frame listed, and answers 404 for one it did not. It returns the process's context, masked environment and current descriptors. `/top/` refreshes them every 5s; recorded process views reconstruct resources from their added/removed events. ### Resource rules @@ -237,8 +269,8 @@ Telltale never stores anything relative to the working directory. Trace data liv | `otel/spans/records.sqlite` | the OTLP receiver's spans, read as the run `receiver` | | `otel/receiver.lock` | the lock the writing receiver holds | | `monitor/events.sqlite` | the rule event journal of `serve --rules` | -| `services/prometheus` | the managed Prometheus of the `prometheus` kind: its deps state, config, log and TSDB | -| `prometheus/targets` | the file_sd target files of the running `prometheus` captures | +| `services/victoriametrics` | the managed VictoriaMetrics installation, state, configuration, and TSDB | +| `victoriametrics/targets` | the file discovery targets of running `prometheus` captures | Config is read from the working directory's `.telltale`, then the global config dir, `$XDG_CONFIG_HOME/telltale` or `~/.config/telltale`; the first that has a file wins. That covers `groups.yaml` and the hub tokens in `agent-tokens/`. `telltale agents token create` writes to the global config dir. `telltale serve` prints every path it uses when it starts. Data under an older `./.telltale/traces`, `./.telltale/otel` or `./.telltale/monitor` is not read or migrated; delete it once it is no longer needed. @@ -320,7 +352,14 @@ To install a binary built against the same local Clicky UI source (honouring `TE task install:symlinked ``` -Production builds and normal frontend tests use published Clicky UI packages from the lockfile. A plain source checkout compiles with an embed sentinel but cannot serve the production UI until `make build` has generated it. +This development workspace links `@flanksource/clicky-ui` to `../clicky-ui/packages/ui` through the pnpm workspace override. Build the sibling package after changing its source, then build Telltale: + +```sh +pnpm --dir ../clicky-ui --filter @flanksource/clicky-ui build +make build +``` + +Frontend tests share the app's React runtime with the linked package. Remove the local pnpm override and regenerate the lockfile before a published-dependency release. A plain source checkout compiles with an embed sentinel but cannot serve the production UI until `make build` has generated it. ## Checks and build output diff --git a/Taskfile.yaml b/Taskfile.yaml index c945b67..39828ab 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -22,9 +22,10 @@ tasks: - pnpm --filter @flanksource/telltale-webapp build{{if .LOCAL_UI}} --mode local-ui{{end}} generate: - desc: Write the TraceDefinition CRD reflected from pkg/definition to crds/ + desc: Write the TraceDefinition CRD reflected from pkg/definition to crds/ and the JVM operation catalog to internal/jvm cmds: - go run ./hack/crd + - ./sdk/java/gradlew -p sdk/java jvmCatalog binary: desc: Compile the binary with existing embedded assets @@ -118,15 +119,29 @@ tasks: - JFR_FILE={{.JFR_FILE}} go test ./pkg/tracing/jfr -run '^$' -bench BenchmarkCompression -benchtime {{.JFR_BENCHTIME}} java:build: - desc: Build the shaded Java 8-compatible agent + desc: Build the Java 8-compatible agent, WAR (also embedded for the tomcat deployer), extension artifacts and the Tomcat test harness cmds: - - ./sdk/java/gradlew -p sdk/java shadowJar -PbuildVersion={{.VERSION}} + - ./sdk/java/gradlew -p sdk/java shadowJar war extensionApiJar exampleExtensionJar tomcatHarnessJar -PbuildVersion={{.VERSION}} java:test: desc: Test Java agent recording lifecycle cmds: - ./sdk/java/gradlew -p sdk/java test + jvm:verify: + desc: Build and verify JVM diagnostics sequentially against disposable runtimes + cmds: + - task: build + - task: java:test + - task: lint + - go test ./internal/jvm ./pkg/tracing/jvmtrace ./pkg/tracing/seam + - go test ./internal/cli -run TestCLI -ginkgo.focus='a kind.s params as flags|kind argument OpenAPI' + - go test ./internal/server -run TestServer -ginkgo.focus='HTTP routing' + - go test ./pkg/definition -run TestDefinition -ginkgo.focus='typed lists|compiled collect entry|TraceDefinition CRD' + - go test ./pkg/traces -run TestTraces -ginkgo.focus='trace host' + - go test ./pkg/agentinstall -run TestInstallation + - pnpm exec vitest run webapp/src/jvm webapp/src/routes.test.tsx packages/api/test/agents.spec.ts + clean: desc: Remove generated assets and binaries while preserving the embed sentinel cmds: diff --git a/capture_capacity_test.go b/capture_capacity_test.go new file mode 100644 index 0000000..812f06c --- /dev/null +++ b/capture_capacity_test.go @@ -0,0 +1,30 @@ +package telltale_test + +import ( + "context" + "time" + + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/pkg/tracing/processcapture" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("run capture capacity", func() { + It("arms every requested capture alongside the automatic process captures", func() { + const requested = 6 + specs := make([]telltale.TraceSpec, requested) + for i := range specs { + specs[i].Kind = processcapture.Kind + } + run, err := telltale.Start(context.Background(), telltale.StartOptions{ + DataDir: GinkgoT().TempDir(), PID: sleeper("30").Process.Pid, + Duration: 100 * time.Millisecond, Traces: specs, + }) + Expect(err).NotTo(HaveOccurred()) + result, err := run.Wait() + Expect(err).NotTo(HaveOccurred()) + Expect(result.Captures).To(HaveLen(requested)) + Expect(result.Process).NotTo(BeNil()) + }) +}) diff --git a/cmd/telltale/main.go b/cmd/telltale/main.go index f6f61a7..c10366b 100644 --- a/cmd/telltale/main.go +++ b/cmd/telltale/main.go @@ -2,21 +2,42 @@ package main import ( "context" + "fmt" "os" "os/signal" "syscall" "github.com/flanksource/telltale/internal/cli" + "github.com/flanksource/telltale/internal/shutdown" ) var version = "dev" func main() { - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer stop() - err := cli.NewRoot(version).ExecuteContext(ctx) + os.Exit(execute()) +} + +func execute() int { + restore, err := terminalRestore() + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + return 1 + } + signals := make(chan os.Signal, 2) + signal.Notify(signals, os.Interrupt, syscall.SIGTERM) + defer signal.Stop(signals) + lifecycle, err := shutdown.New(context.Background(), shutdown.Options{ + GracePeriod: shutdown.GracePeriod, Signals: signals, Output: os.Stderr, Restore: restore, ForceExit: os.Exit, + }) + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + return 1 + } + defer lifecycle.Close() + err = cli.NewRoot(version).ExecuteContext(lifecycle.Context()) cli.FlushHAR() if err != nil { - os.Exit(1) + return 1 } + return 0 } diff --git a/cmd/telltale/terminal.go b/cmd/telltale/terminal.go new file mode 100644 index 0000000..f775ced --- /dev/null +++ b/cmd/telltale/terminal.go @@ -0,0 +1,33 @@ +package main + +import ( + "fmt" + "os" + + "golang.org/x/term" +) + +func terminalRestore() (func(bool), error) { + fd := int(os.Stdin.Fd()) + var original *term.State + if term.IsTerminal(fd) { + var err error + original, err = term.GetState(fd) + if err != nil { + return nil, fmt.Errorf("save terminal state: %w", err) + } + } + return func(alternate bool) { + if original != nil { + if err := term.Restore(fd, original); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "restore terminal: %v\n", err) + } + } + if term.IsTerminal(int(os.Stderr.Fd())) { + if alternate { + _, _ = fmt.Fprint(os.Stderr, "\x1b[?1049l") + } + _, _ = fmt.Fprint(os.Stderr, "\x1b[?25h\x1b[0m") + } + }, nil +} diff --git a/crds/telltale.flanksource.com_deployers.yaml b/crds/telltale.flanksource.com_deployers.yaml new file mode 100644 index 0000000..a0407ee --- /dev/null +++ b/crds/telltale.flanksource.com_deployers.yaml @@ -0,0 +1,141 @@ +# Generated by `task generate` from pkg/deploy; do not edit. +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: deployers.telltale.flanksource.com +spec: + group: telltale.flanksource.com + names: + kind: Deployer + listKind: DeployerList + plural: deployers + singular: deployer + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + auth: + description: The issuer of execution tokens + properties: + issuer: + description: connection:// reference holding the signingKey secret + (RSA PEM) + title: Issuer + type: string + jwks: + description: A Secret or ConfigMap the agents' keys are read from + or published to + nullable: true + properties: + key: + description: Data key holding the JWKS (default jwks.json) + title: Key + type: string + kind: + description: ConfigMap or Secret + title: Kind + type: string + mode: + description: read or publish + title: Mode + type: string + name: + title: Name + type: string + title: JWKS + type: object + jwksUrl: + description: Where the issuer runtime serves its JWKS + title: JWKS URL + type: string + title: Auth + type: object + javaContainer: + description: How the agent gets into the JVM + properties: + kind: + description: tomcat attach or javaagent + title: Kind + type: string + required: + - kind + title: Java container + type: object + x-kubernetes-preserve-unknown-fields: true + selector: + description: Pods that receive the agent + properties: + container: + description: 'The container running the JVM (default: the pod''s + only container)' + title: Container + type: string + kind: + description: Deployment StatefulSet or Pod + title: Kind + type: string + labels: + additionalProperties: + type: string + description: Labels of the workloads (or pods) to select + title: Labels + type: object + name: + description: The one workload (or pod) to select instead of labels + title: Name + type: string + namespace: + title: Namespace + type: string + title: Selector + type: object + type: object + status: + properties: + targets: + items: + properties: + agentAPIVersion: + format: int32 + type: integer + container: + type: string + endpoint: + type: string + error: + type: string + namespace: + type: string + pod: + type: string + state: + type: string + target: + type: string + required: + - target + - namespace + - pod + - container + - state + type: object + type: array + x-kubernetes-list-type: atomic + type: object + required: + - apiVersion + - kind + - spec + type: object + served: true + storage: true diff --git a/crds/telltale.flanksource.com_tracedefinitions.yaml b/crds/telltale.flanksource.com_tracedefinitions.yaml index 3b51cfb..9dba9c2 100644 --- a/crds/telltale.flanksource.com_tracedefinitions.yaml +++ b/crds/telltale.flanksource.com_tracedefinitions.yaml @@ -27,9 +27,9 @@ spec: collect: description: Captures that only read properties: - go: + clicky: properties: - exec: + entity: items: properties: agent: @@ -52,38 +52,25 @@ spec: type: boolean title: Agent type: object - executable: - description: Executable name glob - title: Executable + entity: + description: Entity name glob type: string + errorsOnly: + description: Record failed operations only + type: boolean name: description: Names the capture's session; empty takes -
--- title: Name type: string - tailBytes: - description: Maximum bytes of each output tail - format: int32 - title: Tail bytes - type: integer - type: object - title: Executions - type: array - x-kubernetes-list-type: atomic - gops: - description: Goroutine dumps over the process's gops agent - items: - properties: - name: - description: Names the capture's session; empty takes - -
--- - title: Name + verb: + description: Verb glob type: string type: object - title: gops + title: Entity operations type: array x-kubernetes-list-type: atomic - http: + exec: items: properties: agent: @@ -106,31 +93,25 @@ spec: type: boolean title: Agent type: object - dedupWindow: - description: Store an identical exchange only once within - this duration such as 1h - title: Dedup window + executable: + description: Executable name glob + title: Executable type: string - features: - description: 'Telltale''s transports to observe by their - HAR level: http prometheus loki opensearch git ...' - items: - type: string - title: Features - type: array - x-kubernetes-list-type: atomic name: description: Names the capture's session; empty takes -
--- title: Name type: string - required: - - features + tailBytes: + description: Maximum bytes of each output tail + format: int32 + title: Tail bytes + type: integer type: object - title: HTTP + title: Executions type: array x-kubernetes-list-type: atomic - llm: + http: items: properties: agent: @@ -153,22 +134,63 @@ spec: type: boolean title: Agent type: object - includePrompts: - description: Include prompts - type: boolean - includeResponses: - description: Include responses - type: boolean - models: - description: Model glob + bodies: + description: Retained bytes of each body default 256KiB + maximum 4MiB; zero keeps none + title: Bodies + type: string + dedupWindow: + description: Store an identical exchange only once within + this duration such as 1h + title: Dedup window + type: string + direction: + default: both + description: Incoming requests outgoing clients or both + enum: + - incoming + - outgoing + - both + title: Direction type: string + features: + description: 'Telltale''s transports to observe by their + HAR level: http prometheus loki opensearch git ...' + items: + type: string + title: Features + type: array + x-kubernetes-list-type: atomic + hosts: + description: Host globs to record + items: + type: string + title: Hosts + type: array + x-kubernetes-list-type: atomic name: description: Names the capture's session; empty takes -
--- title: Name type: string + paths: + description: URL path globs to record + items: + type: string + title: Paths + type: array + x-kubernetes-list-type: atomic + redact: + description: Additional header names to redact + items: + type: string + title: Redact headers + type: array + x-kubernetes-list-type: atomic + required: + - features type: object - title: LLM calls + title: HTTP type: array x-kubernetes-list-type: atomic logs: @@ -217,7 +239,8 @@ spec: type: array x-kubernetes-list-type: atomic pprof: - description: Heap cpu allocs or goroutine profiles + description: Heap cpu allocs or goroutine profiles read through + the agent at each checkpoint items: properties: agent: @@ -271,7 +294,7 @@ spec: title: pprof type: array x-kubernetes-list-type: atomic - spans: + task: items: properties: agent: @@ -294,16 +317,34 @@ spec: type: boolean title: Agent type: object + group: + description: Parent group name glob + title: Group + type: string name: description: Names the capture's session; empty takes -
--- title: Name type: string + progressIntervalMs: + description: Minimum milliseconds between progress records + of a task + format: int32 + title: Progress interval + type: integer + taskName: + description: Task name glob + title: Task name + type: string type: object - title: Spans + title: Tasks type: array x-kubernetes-list-type: atomic - sql: + title: Clicky + type: object + go: + properties: + exec: items: properties: agent: @@ -326,174 +367,190 @@ spec: type: boolean title: Agent type: object - connections: - description: 'Connections to observe: names; * for every - one; or self for telltale''s own database' - items: - type: string - title: Connections - type: array - x-kubernetes-list-type: atomic - minDuration: - description: Record only statements that ran at least - this long such as 100ms - title: Minimum duration + executable: + description: Executable name glob + title: Executable type: string name: description: Names the capture's session; empty takes -
--- title: Name type: string - required: - - connections + tailBytes: + description: Maximum bytes of each output tail + format: int32 + title: Tail bytes + type: integer type: object - title: SQL + title: Executions type: array x-kubernetes-list-type: atomic - title: Go - type: object - host: - properties: - processGroups: + gops: + description: Goroutine dumps over the process's gops agent items: properties: - all: - description: Also record the processes no tool's group - holds by program - title: All processes - type: boolean - catalog: - description: A groups.yaml whose rules are applied over - the built-in ones - title: Catalog - type: string - groups: - description: Record only these groups and those under - them such as claude or vitest - items: - type: string - title: Groups - type: array - x-kubernetes-list-type: atomic - interval: - description: How often the groups are read such as 2s - (at least 1s) - title: Interval - type: string name: description: Names the capture's session; empty takes -
--- title: Name type: string - net: - description: Read each process's network bytes with - nettop (darwin; costs over a core) - title: Network bytes - type: boolean type: object - title: Process groups + title: gops type: array x-kubernetes-list-type: atomic - title: Host - type: object - logs: - properties: - azureLogAnalytics: + http: items: properties: - connection: - description: Empty takes the definition's default connection - of the backend's type - title: Connection - type: string - match: - title: Match - type: string - minLevel: - title: Minimum level - type: string - name: - description: Names the capture's session; empty takes - -
--- - title: Name - type: string - options: - description: The backend's own options - title: Options + agent: + description: Read through a connected agent instead + of the execution's process + nullable: true + properties: + id: + title: Agent ID + type: string + selector: + additionalProperties: + type: string + description: Agent resource attributes to match + title: Selector + type: object + wait: + description: Wait for a matching agent to connect + title: Wait + type: boolean + title: Agent type: object - x-kubernetes-preserve-unknown-fields: true - query: - title: Query - type: string - timeField: - title: Time field - type: string - type: object - title: Azure Log Analytics - type: array - x-kubernetes-list-type: atomic - cloudwatch: - items: - properties: - connection: - description: Empty takes the definition's default connection - of the backend's type - title: Connection + bodies: + description: Retained bytes of each body default 256KiB + maximum 4MiB; zero keeps none + title: Bodies type: string - match: - title: Match + dedupWindow: + description: Store an identical exchange only once within + this duration such as 1h + title: Dedup window type: string - minLevel: - title: Minimum level + direction: + default: both + description: Incoming requests outgoing clients or both + enum: + - incoming + - outgoing + - both + title: Direction type: string + features: + description: 'Telltale''s transports to observe by their + HAR level: http prometheus loki opensearch git ...' + items: + type: string + title: Features + type: array + x-kubernetes-list-type: atomic + hosts: + description: Host globs to record + items: + type: string + title: Hosts + type: array + x-kubernetes-list-type: atomic name: description: Names the capture's session; empty takes -
--- title: Name type: string - options: - description: The backend's own options - title: Options - type: object - x-kubernetes-preserve-unknown-fields: true - query: - title: Query - type: string - timeField: - title: Time field - type: string + paths: + description: URL path globs to record + items: + type: string + title: Paths + type: array + x-kubernetes-list-type: atomic + redact: + description: Additional header names to redact + items: + type: string + title: Redact headers + type: array + x-kubernetes-list-type: atomic + required: + - features type: object - title: CloudWatch + title: HTTP type: array x-kubernetes-list-type: atomic - container: + llm: items: properties: - container: - title: Container - type: string - match: - title: Match + agent: + description: Read through a connected agent instead + of the execution's process + nullable: true + properties: + id: + title: Agent ID + type: string + selector: + additionalProperties: + type: string + description: Agent resource attributes to match + title: Selector + type: object + wait: + description: Wait for a matching agent to connect + title: Wait + type: boolean + title: Agent + type: object + includePrompts: + description: Include prompts + type: boolean + includeResponses: + description: Include responses + type: boolean + models: + description: Model glob type: string name: description: Names the capture's session; empty takes -
--- title: Name type: string - required: - - container type: object - title: Container + title: LLM calls type: array x-kubernetes-list-type: atomic - gcpCloudLogging: + logs: items: properties: - connection: - description: Empty takes the definition's default connection - of the backend's type - title: Connection - type: string + agent: + description: Read through a connected agent instead + of the execution's process + nullable: true + properties: + id: + title: Agent ID + type: string + selector: + additionalProperties: + type: string + description: Agent resource attributes to match + title: Selector + type: object + wait: + description: Wait for a matching agent to connect + title: Wait + type: boolean + title: Agent + type: object + levels: + additionalProperties: + type: string + description: 'Refused: setting log levels changes the + agent' + title: Levels + type: object match: title: Match type: string @@ -505,387 +562,176 @@ spec: -
--- title: Name type: string - options: - description: The backend's own options - title: Options - type: object - x-kubernetes-preserve-unknown-fields: true - query: - title: Query - type: string - timeField: - title: Time field - type: string type: object - title: GCP Cloud Logging + title: Logs type: array x-kubernetes-list-type: atomic - kubernetes: + pprof: + description: Heap cpu allocs or goroutine profiles items: properties: - container: - title: Container - type: string - match: - title: Match - type: string + agent: + description: Read through a connected agent instead + of the execution's process + nullable: true + properties: + id: + title: Agent ID + type: string + selector: + additionalProperties: + type: string + description: Agent resource attributes to match + title: Selector + type: object + wait: + description: Wait for a matching agent to connect + title: Wait + type: boolean + title: Agent + type: object name: description: Names the capture's session; empty takes -
--- title: Name type: string - namespace: - title: Namespace + profile: + enum: + - heap + - cpu + - allocs + - goroutine + title: Profile type: string - podSelector: - description: Label selector such as app=checkout - title: Pod selector + seconds: + description: Duration of a CPU profile window in seconds + format: int32 + title: Seconds + type: integer + source: + description: gops (default) or pprof + enum: + - gops + - pprof + title: Source type: string required: - - namespace - - podSelector + - profile type: object - title: Kubernetes + title: pprof type: array x-kubernetes-list-type: atomic - loki: + spans: items: properties: - connection: - description: Empty takes the definition's default connection - of the backend's type - title: Connection - type: string - match: - title: Match - type: string - minLevel: - title: Minimum level - type: string + agent: + description: Read through a connected agent instead + of the execution's process + nullable: true + properties: + id: + title: Agent ID + type: string + selector: + additionalProperties: + type: string + description: Agent resource attributes to match + title: Selector + type: object + wait: + description: Wait for a matching agent to connect + title: Wait + type: boolean + title: Agent + type: object name: description: Names the capture's session; empty takes -
--- title: Name type: string - options: - description: The backend's own options - title: Options - type: object - x-kubernetes-preserve-unknown-fields: true - query: - title: Query - type: string - timeField: - title: Time field - type: string type: object - title: Loki + title: Spans type: array x-kubernetes-list-type: atomic - opensearch: + sql: items: properties: - connection: - description: Empty takes the definition's default connection - of the backend's type - title: Connection - type: string - match: - title: Match - type: string - minLevel: - title: Minimum level + agent: + description: Read through a connected agent instead + of the execution's process + nullable: true + properties: + id: + title: Agent ID + type: string + selector: + additionalProperties: + type: string + description: Agent resource attributes to match + title: Selector + type: object + wait: + description: Wait for a matching agent to connect + title: Wait + type: boolean + title: Agent + type: object + connections: + description: 'Connections to observe: names; * for every + one; or self for telltale''s own database' + items: + type: string + title: Connections + type: array + x-kubernetes-list-type: atomic + minDuration: + description: Record only statements that ran at least + this long such as 100ms + title: Minimum duration type: string name: description: Names the capture's session; empty takes -
--- title: Name type: string - options: - description: The backend's own options - title: Options - type: object - x-kubernetes-preserve-unknown-fields: true - query: - title: Query - type: string - timeField: - title: Time field - type: string + required: + - connections type: object - title: OpenSearch + title: SQL type: array x-kubernetes-list-type: atomic - title: Logs + title: Go type: object - opensearch: + host: properties: - spans: + processes: items: properties: - connection: - title: Connection + catalog: type: string - dateField: - title: Date field - type: string - followIndex: - title: Follow index - type: boolean - format: - title: Format - type: string - from: - title: From - type: string - index: - title: Index - type: string - lag: - title: Lag - type: string - name: - description: Names the capture's session; empty takes - -
--- - title: Name - type: string - poll: - title: Poll - type: string - search: - format: json + environment: nullable: true properties: - aggregations: - additionalProperties: - format: byte - type: string - type: object - fields: + keys: + description: Environment variables to read onto + each process items: type: string + title: Keys type: array x-kubernetes-list-type: atomic - from: - format: int32 - nullable: true - type: integer - query: - nullable: true - properties: - analyzer: - type: string - boost: - format: double - nullable: true - type: number - caseInsensitive: - nullable: true - type: boolean - conditions: - items: - x-kubernetes-preserve-unknown-fields: true - type: array - x-kubernetes-list-type: atomic - escape: - nullable: true - type: boolean - field: - type: string - fields: - items: - type: string - type: array - x-kubernetes-list-type: atomic - format: - type: string - fuzziness: - type: string - gt: - nullable: true - properties: - Literal: - x-kubernetes-preserve-unknown-fields: true - Param: - type: string - required: - - Param - type: object - gte: - nullable: true - properties: - Literal: - x-kubernetes-preserve-unknown-fields: true - Param: - type: string - required: - - Param - type: object - lt: - nullable: true - properties: - Literal: - x-kubernetes-preserve-unknown-fields: true - Param: - type: string - required: - - Param - type: object - lte: - nullable: true - properties: - Literal: - x-kubernetes-preserve-unknown-fields: true - Param: - type: string - required: - - Param - type: object - matchOperator: - type: string - minimumShouldMatch: - type: string - multiMatchType: - type: string - occur: - type: string - op: - type: string - optional: - type: boolean - path: - type: string - scoreMode: - type: string - slop: - format: int32 - nullable: true - type: integer - timeZone: - type: string - value: - nullable: true - properties: - Literal: - x-kubernetes-preserve-unknown-fields: true - Param: - type: string - required: - - Param - type: object - values: - items: - properties: - Literal: - x-kubernetes-preserve-unknown-fields: true - Param: - type: string - required: - - Param - type: object - type: array - x-kubernetes-list-type: atomic - when: - type: string - required: - - op - type: object - size: - format: int32 - nullable: true - type: integer - sort: - items: - properties: - field: - type: string - missing: - type: string - mode: - type: string - order: - type: string - unmappedType: - type: string - required: - - field - type: object - type: array - x-kubernetes-list-type: atomic - source: - nullable: true - properties: - enabled: - nullable: true - type: boolean - excludes: - items: - type: string - type: array - x-kubernetes-list-type: atomic - includes: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - storedFields: + prefixes: + description: Prefixes of the environment variables + to read onto each process items: type: string + title: Prefixes type: array x-kubernetes-list-type: atomic - timeField: - type: string - timeFieldFormat: - type: string - trackTotalHits: - nullable: true - properties: - enabled: - nullable: true - type: boolean - threshold: - format: int32 - nullable: true - type: integer - type: object - title: Structured search type: object - to: - title: To - type: string - type: object - title: Spans - type: array - x-kubernetes-list-type: atomic - title: OpenSearch - type: object - process: - properties: - nativeStacks: - description: 'Thread stacks of any process: sample (darwin) - eu-stack or gdb (linux)' - items: - properties: - name: - description: Names the capture's session; empty takes - -
--- - title: Name - type: string - type: object - title: Native stacks - type: array - x-kubernetes-list-type: atomic - resources: - description: CPU memory disk IO and descriptors of every process - items: - properties: facets: - description: 'Descriptors read of every process each - tick with lsof: openFiles sockets ports or connections' items: type: string - title: Facets type: array x-kubernetes-list-type: atomic interval: @@ -898,463 +744,277 @@ spec: -
--- title: Name type: string + net: + type: boolean type: object - title: Resources + title: Processes type: array x-kubernetes-list-type: atomic - tree: - description: What the run-wide process tree reads of each - process - nullable: true - properties: - environment: - description: Environment variables read onto each process - nullable: true - properties: - keys: - description: Environment variables to read onto each - process - items: - type: string - title: Keys - type: array - x-kubernetes-list-type: atomic - prefixes: - description: Prefixes of the environment variables - to read onto each process - items: - type: string - title: Prefixes - type: array - x-kubernetes-list-type: atomic - title: Environment - type: object - facets: - description: Read every process's open files sockets and - environment into the process tree - title: Process facets - type: boolean - title: Process tree - type: object - title: Process + title: Host type: object - prometheus: + logs: properties: - scrape: + azureLogAnalytics: items: properties: - endpoints: - description: Prometheus metrics URLs to scrape such - as http://localhost:8080/metrics - items: - type: string - title: Endpoints - type: array - x-kubernetes-list-type: atomic - interval: - description: How often each endpoint is scraped such - as 15s (at least 1s) - title: Interval + connection: + description: Empty takes the definition's default connection + of the backend's type + title: Connection + type: string + match: + title: Match + type: string + minLevel: + title: Minimum level type: string - labels: - additionalProperties: - type: string - description: Extra labels every scraped sample is stored - with such as env=dev - title: Labels - type: object name: description: Names the capture's session; empty takes -
--- title: Name type: string - timeout: - description: How long one scrape may take such as 5s - (at most the interval; default the shorter of 10s - and the interval) - title: Timeout + options: + description: The backend's own options + title: Options + type: object + x-kubernetes-preserve-unknown-fields: true + query: + title: Query + type: string + timeField: + title: Time field type: string type: object - title: Scrape + title: Azure Log Analytics type: array x-kubernetes-list-type: atomic - title: Prometheus - type: object - query: - properties: - bigquery: + cloudwatch: items: properties: connection: description: Empty takes the definition's default connection - of the provider's type + of the backend's type title: Connection type: string - every: - description: Re-read the rows written since the last - read at this interval such as 30s - title: Every + match: + title: Match type: string - from: - description: Start of the window; a time or date math - such as now-1h - title: From + minLevel: + title: Minimum level type: string - limit: - description: Most rows one read keeps; 0 keeps all - format: int32 - title: Limit - type: integer - mapping: - nullable: true - properties: - host: - items: - type: string - type: array - x-kubernetes-list-type: atomic - id: - items: - type: string - type: array - x-kubernetes-list-type: atomic - ignore: - items: - type: string - type: array - x-kubernetes-list-type: atomic - message: - items: - type: string - type: array - x-kubernetes-list-type: atomic - severity: - items: - type: string - type: array - x-kubernetes-list-type: atomic - source: - items: - type: string - type: array - x-kubernetes-list-type: atomic - timestamp: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object name: description: Names the capture's session; empty takes -
--- title: Name type: string - params: - description: The query's params by name - title: Params + options: + description: The backend's own options + title: Options type: object x-kubernetes-preserve-unknown-fields: true - project: - type: string query: title: Query type: string - to: - description: End of the window; empty is now - title: To + timeField: + title: Time field type: string - untilStopped: - description: Read once when the run stops over the whole - run - title: Until stopped - type: boolean type: object - title: BigQuery + title: CloudWatch type: array x-kubernetes-list-type: atomic - cloudwatchMetrics: + container: + items: + properties: + container: + title: Container + type: string + match: + title: Match + type: string + name: + description: Names the capture's session; empty takes + -
--- + title: Name + type: string + required: + - container + type: object + title: Container + type: array + x-kubernetes-list-type: atomic + gcpCloudLogging: items: properties: - concurrency: - description: Parallel GetMetricData batches 1..32; 4 - when empty - format: int32 - title: Concurrency - type: integer connection: description: Empty takes the definition's default connection - of the provider's type + of the backend's type title: Connection type: string - dimensions: - additionalProperties: - type: string - title: Dimensions - type: object - endpoint: - description: A CloudWatch endpoint in place of the region's - title: Endpoint - type: string - every: - description: Re-read the rows written since the last - read at this interval such as 30s - title: Every - type: string - from: - description: Start of the window; a time or date math - such as now-1h - title: From + match: + title: Match type: string - language: - description: metrics (the listed metrics) metrics-insights - (the query) or sql (metric SQL) - title: Language + minLevel: + title: Minimum level type: string - limit: - description: Most rows one read keeps; 0 keeps all - format: int32 - title: Limit - type: integer - metrics: - items: - properties: - dimensions: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - statistic: - type: string - unit: - type: string - required: - - name - - namespace - type: object - title: Metrics - type: array - x-kubernetes-list-type: atomic name: description: Names the capture's session; empty takes -
--- title: Name type: string - namespace: - description: Such as AWS/EC2 - title: Namespace - type: string - params: - description: The query's params by name - title: Params + options: + description: The backend's own options + title: Options type: object x-kubernetes-preserve-unknown-fields: true query: title: Query type: string - region: - title: Region - type: string - statistic: - description: Each metric's default; Average when empty - title: Statistic + timeField: + title: Time field type: string - step: - description: The CloudWatch period such as 60s - title: Step + type: object + title: GCP Cloud Logging + type: array + x-kubernetes-list-type: atomic + kubernetes: + items: + properties: + container: + title: Container type: string - to: - description: End of the window; empty is now - title: To + match: + title: Match type: string - unit: - title: Unit + name: + description: Names the capture's session; empty takes + -
--- + title: Name + type: string + namespace: + title: Namespace + type: string + podSelector: + description: Label selector such as app=checkout + title: Pod selector type: string - untilStopped: - description: Read once when the run stops over the whole - run - title: Until stopped - type: boolean required: - - step + - namespace + - podSelector type: object - title: CloudWatch metrics + title: Kubernetes type: array x-kubernetes-list-type: atomic - http: + loki: items: properties: - body: - type: string connection: description: Empty takes the definition's default connection - of the provider's type + of the backend's type title: Connection type: string - every: - description: Re-read the rows written since the last - read at this interval such as 30s - title: Every - type: string - from: - description: Start of the window; a time or date math - such as now-1h - title: From - type: string - jsonpath: + match: + title: Match type: string - limit: - description: Most rows one read keeps; 0 keeps all - format: int32 - title: Limit - type: integer - method: + minLevel: + title: Minimum level type: string name: description: Names the capture's session; empty takes -
--- title: Name type: string - params: - description: The query's params by name - title: Params + options: + description: The backend's own options + title: Options type: object x-kubernetes-preserve-unknown-fields: true query: title: Query type: string - to: - description: End of the window; empty is now - title: To - type: string - untilStopped: - description: Read once when the run stops over the whole - run - title: Until stopped - type: boolean - url: + timeField: + title: Time field type: string type: object - title: HTTP + title: Loki type: array x-kubernetes-list-type: atomic - jaeger: + opensearch: items: properties: connection: description: Empty takes the definition's default connection - of the provider's type + of the backend's type title: Connection type: string - end: - type: string - every: - description: Re-read the rows written since the last - read at this interval such as 30s - title: Every - type: string - from: - description: Start of the window; a time or date math - such as now-1h - title: From - type: string - limit: - description: Most rows one read keeps; 0 keeps all - format: int32 - title: Limit - type: integer - lookback: - type: string - maxDuration: + match: + title: Match type: string - minDuration: + minLevel: + title: Minimum level type: string name: description: Names the capture's session; empty takes -
--- title: Name type: string - operation: - type: string - params: - description: The query's params by name - title: Params + options: + description: The backend's own options + title: Options type: object x-kubernetes-preserve-unknown-fields: true query: title: Query type: string - service: - type: string - start: - type: string - tags: - type: string - to: - description: End of the window; empty is now - title: To - type: string - untilStopped: - description: Read once when the run stops over the whole - run - title: Until stopped - type: boolean - url: + timeField: + title: Time field type: string type: object - title: Jaeger + title: OpenSearch type: array x-kubernetes-list-type: atomic - opensearch: + title: Logs + type: object + opensearch: + properties: + spans: items: properties: - address: - type: string connection: - description: Empty takes the definition's default connection - of the provider's type title: Connection type: string - every: - description: Re-read the rows written since the last - read at this interval such as 30s - title: Every + dateField: + title: Date field + type: string + followIndex: + title: Follow index + type: boolean + format: + title: Format type: string from: - description: Start of the window; a time or date math - such as now-1h title: From type: string index: + title: Index + type: string + lag: + title: Lag type: string - limit: - description: Most rows one read keeps; 0 keeps all - format: int32 - title: Limit - type: integer name: description: Names the capture's session; empty takes -
--- title: Name type: string - params: - description: The query's params by name - title: Params - type: object - x-kubernetes-preserve-unknown-fields: true - query: - title: Query + poll: + title: Poll type: string search: + format: json nullable: true properties: aggregations: - additionalProperties: - format: byte - type: string type: object + x-kubernetes-preserve-unknown-fields: true fields: items: type: string @@ -1543,698 +1203,2149 @@ spec: nullable: true type: integer type: object + title: Structured search type: object - tailLag: - type: string - tailPoll: - type: string to: - description: End of the window; empty is now title: To type: string - untilStopped: - description: Read once when the run stops over the whole - run - title: Until stopped - type: boolean type: object - title: OpenSearch + title: Spans type: array x-kubernetes-list-type: atomic - opentelemetry: + title: OpenSearch + type: object + postgres: + properties: + top: + description: Samples pg_stat_activity top style items: properties: connection: - description: Empty takes the definition's default connection - of the provider's type + description: Postgres or SQL Server connection reference + for the database seam title: Connection type: string - dateField: - type: string - every: - description: Re-read the rows written since the last - read at this interval such as 30s - title: Every - type: string - format: - type: string - from: - description: Start of the window; a time or date math - such as now-1h - title: From - type: string - index: + databases: + description: Database patterns with * wildcards and + ! exclusions; empty keeps every database + items: + type: string + title: Databases + type: array + x-kubernetes-list-type: atomic + idle: + description: Also record idle sessions; sessions holding + an open transaction are always recorded + title: Idle sessions + type: boolean + minElapsed: + description: Only record requests running at least this + long such as 500ms + title: Minimum elapsed type: string - limit: - description: Most rows one read keeps; 0 keeps all - format: int32 - title: Limit - type: integer name: description: Names the capture's session; empty takes -
--- title: Name type: string - operationField: - type: string - params: - description: The query's params by name - title: Params - type: object - x-kubernetes-preserve-unknown-fields: true - parentIdField: - type: string - parentRefType: - type: string - query: - title: Query + poll: + description: Poll interval such as 5s + title: Poll type: string - search: - nullable: true - properties: - aggregations: - additionalProperties: - format: byte - type: string - type: object - fields: - items: - type: string - type: array - x-kubernetes-list-type: atomic - from: - format: int32 - nullable: true - type: integer - query: - nullable: true - properties: - analyzer: - type: string - boost: - format: double - nullable: true - type: number - caseInsensitive: - nullable: true - type: boolean - conditions: - items: - x-kubernetes-preserve-unknown-fields: true - type: array - x-kubernetes-list-type: atomic - escape: - nullable: true - type: boolean - field: - type: string - fields: - items: - type: string - type: array - x-kubernetes-list-type: atomic - format: - type: string - fuzziness: - type: string - gt: - nullable: true - properties: - Literal: - x-kubernetes-preserve-unknown-fields: true - Param: - type: string - required: - - Param - type: object - gte: - nullable: true - properties: - Literal: - x-kubernetes-preserve-unknown-fields: true - Param: - type: string - required: - - Param - type: object - lt: - nullable: true - properties: - Literal: - x-kubernetes-preserve-unknown-fields: true - Param: - type: string - required: - - Param - type: object - lte: - nullable: true - properties: - Literal: - x-kubernetes-preserve-unknown-fields: true - Param: - type: string - required: - - Param - type: object - matchOperator: - type: string - minimumShouldMatch: - type: string - multiMatchType: - type: string - occur: - type: string - op: - type: string - optional: - type: boolean - path: - type: string - scoreMode: - type: string - slop: - format: int32 - nullable: true - type: integer - timeZone: - type: string - value: - nullable: true - properties: - Literal: - x-kubernetes-preserve-unknown-fields: true - Param: - type: string - required: - - Param - type: object - values: - items: - properties: - Literal: - x-kubernetes-preserve-unknown-fields: true - Param: - type: string - required: - - Param - type: object - type: array - x-kubernetes-list-type: atomic - when: - type: string - required: - - op - type: object - size: - format: int32 - nullable: true - type: integer - sort: + text: + description: SQL text patterns with * wildcards and + ! exclusions; bare values match substrings + items: + type: string + title: Statement text + type: array + x-kubernetes-list-type: atomic + top: + description: Maximum requests per poll ranked by elapsed + time; defaults to 50 + format: int32 + title: Top + type: integer + type: object + title: Running queries + type: array + x-kubernetes-list-type: atomic + title: Postgres + type: object + process: + properties: + nativeStacks: + description: 'Thread stacks of any process: sample (darwin) + eu-stack or gdb (linux)' + items: + properties: + name: + description: Names the capture's session; empty takes + -
--- + title: Name + type: string + type: object + title: Native stacks + type: array + x-kubernetes-list-type: atomic + resources: + description: Startup metadata resource changes and metrics + items: + properties: + catalog: + type: string + environment: + nullable: true + properties: + keys: + description: Environment variables to read onto + each process items: - properties: - field: - type: string - missing: - type: string - mode: - type: string - order: - type: string - unmappedType: - type: string - required: - - field - type: object + type: string + title: Keys type: array x-kubernetes-list-type: atomic - source: - nullable: true - properties: - enabled: - nullable: true - type: boolean - excludes: - items: - type: string - type: array - x-kubernetes-list-type: atomic - includes: - items: - type: string - type: array - x-kubernetes-list-type: atomic - type: object - storedFields: + prefixes: + description: Prefixes of the environment variables + to read onto each process items: type: string + title: Prefixes type: array x-kubernetes-list-type: atomic - timeField: + type: object + facets: + items: + type: string + type: array + x-kubernetes-list-type: atomic + interval: + description: How often the tree is sampled such as 1s + (at least 250ms) + title: Interval + type: string + name: + description: Names the capture's session; empty takes + -
--- + title: Name + type: string + net: + type: boolean + type: object + title: Processes + type: array + x-kubernetes-list-type: atomic + tree: + description: What the run-wide process tree reads of each + process + nullable: true + properties: + environment: + description: Environment variables read onto each process + nullable: true + properties: + keys: + description: Environment variables to read onto each + process + items: type: string - timeFieldFormat: + title: Keys + type: array + x-kubernetes-list-type: atomic + prefixes: + description: Prefixes of the environment variables + to read onto each process + items: + type: string + title: Prefixes + type: array + x-kubernetes-list-type: atomic + title: Environment + type: object + facets: + description: Read every process's open files sockets and + environment into the process tree + title: Process facets + type: boolean + title: Process tree + type: object + title: Process + type: object + prometheus: + properties: + scrape: + items: + properties: + endpoints: + description: Metrics URLs using http:// or https:// + or Kubernetes portforward:// (also port-forward://) + or proxy:// + items: + type: string + title: Endpoints + type: array + x-kubernetes-list-type: atomic + interval: + description: How often each endpoint is scraped such + as 15s (at least 1s) + title: Interval + type: string + labels: + additionalProperties: + type: string + description: Extra labels every scraped sample is stored + with such as env=dev + title: Labels + type: object + name: + description: Names the capture's session; empty takes + -
--- + title: Name + type: string + timeout: + description: How long one scrape may take such as 5s + (at most the interval; default the shorter of 10s + and the interval) + title: Timeout + type: string + type: object + title: Scrape + type: array + x-kubernetes-list-type: atomic + title: Prometheus + type: object + query: + properties: + bigquery: + items: + properties: + connection: + description: Empty takes the definition's default connection + of the provider's type + title: Connection + type: string + every: + description: Re-read the rows written since the last + read at this interval such as 30s + title: Every + type: string + from: + description: Start of the window; a time or date math + such as now-1h + title: From + type: string + limit: + description: Most rows one read keeps; 0 keeps all + format: int32 + title: Limit + type: integer + mapping: + nullable: true + properties: + host: + items: + type: string + type: array + x-kubernetes-list-type: atomic + id: + items: + type: string + type: array + x-kubernetes-list-type: atomic + ignore: + items: + type: string + type: array + x-kubernetes-list-type: atomic + message: + items: + type: string + type: array + x-kubernetes-list-type: atomic + severity: + items: + type: string + type: array + x-kubernetes-list-type: atomic + source: + items: + type: string + type: array + x-kubernetes-list-type: atomic + timestamp: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + name: + description: Names the capture's session; empty takes + -
--- + title: Name + type: string + params: + description: The query's params by name + title: Params + type: object + x-kubernetes-preserve-unknown-fields: true + project: + type: string + query: + title: Query + type: string + to: + description: End of the window; empty is now + title: To + type: string + untilStopped: + description: Read once when the run stops over the whole + run + title: Until stopped + type: boolean + type: object + title: BigQuery + type: array + x-kubernetes-list-type: atomic + cloudwatchMetrics: + items: + properties: + concurrency: + description: Parallel GetMetricData batches 1..32; 4 + when empty + format: int32 + title: Concurrency + type: integer + connection: + description: Empty takes the definition's default connection + of the provider's type + title: Connection + type: string + dimensions: + additionalProperties: + type: string + title: Dimensions + type: object + endpoint: + description: A CloudWatch endpoint in place of the region's + title: Endpoint + type: string + every: + description: Re-read the rows written since the last + read at this interval such as 30s + title: Every + type: string + from: + description: Start of the window; a time or date math + such as now-1h + title: From + type: string + language: + description: metrics (the listed metrics) metrics-insights + (the query) or sql (metric SQL) + title: Language + type: string + limit: + description: Most rows one read keeps; 0 keeps all + format: int32 + title: Limit + type: integer + metrics: + items: + properties: + dimensions: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + statistic: + type: string + unit: + type: string + required: + - name + - namespace + type: object + title: Metrics + type: array + x-kubernetes-list-type: atomic + name: + description: Names the capture's session; empty takes + -
--- + title: Name + type: string + namespace: + description: Such as AWS/EC2 + title: Namespace + type: string + params: + description: The query's params by name + title: Params + type: object + x-kubernetes-preserve-unknown-fields: true + query: + title: Query + type: string + region: + title: Region + type: string + statistic: + description: Each metric's default; Average when empty + title: Statistic + type: string + step: + description: The CloudWatch period such as 60s + title: Step + type: string + to: + description: End of the window; empty is now + title: To + type: string + unit: + title: Unit + type: string + untilStopped: + description: Read once when the run stops over the whole + run + title: Until stopped + type: boolean + required: + - step + type: object + title: CloudWatch metrics + type: array + x-kubernetes-list-type: atomic + http: + items: + properties: + body: + type: string + connection: + description: Empty takes the definition's default connection + of the provider's type + title: Connection + type: string + every: + description: Re-read the rows written since the last + read at this interval such as 30s + title: Every + type: string + from: + description: Start of the window; a time or date math + such as now-1h + title: From + type: string + jsonpath: + type: string + limit: + description: Most rows one read keeps; 0 keeps all + format: int32 + title: Limit + type: integer + method: + type: string + name: + description: Names the capture's session; empty takes + -
--- + title: Name + type: string + params: + description: The query's params by name + title: Params + type: object + x-kubernetes-preserve-unknown-fields: true + query: + title: Query + type: string + to: + description: End of the window; empty is now + title: To + type: string + untilStopped: + description: Read once when the run stops over the whole + run + title: Until stopped + type: boolean + url: + type: string + type: object + title: HTTP + type: array + x-kubernetes-list-type: atomic + jaeger: + items: + properties: + connection: + description: Empty takes the definition's default connection + of the provider's type + title: Connection + type: string + end: + type: string + every: + description: Re-read the rows written since the last + read at this interval such as 30s + title: Every + type: string + from: + description: Start of the window; a time or date math + such as now-1h + title: From + type: string + limit: + description: Most rows one read keeps; 0 keeps all + format: int32 + title: Limit + type: integer + lookback: + type: string + maxDuration: + type: string + minDuration: + type: string + name: + description: Names the capture's session; empty takes + -
--- + title: Name + type: string + operation: + type: string + params: + description: The query's params by name + title: Params + type: object + x-kubernetes-preserve-unknown-fields: true + query: + title: Query + type: string + service: + type: string + start: + type: string + tags: + type: string + to: + description: End of the window; empty is now + title: To + type: string + untilStopped: + description: Read once when the run stops over the whole + run + title: Until stopped + type: boolean + url: + type: string + type: object + title: Jaeger + type: array + x-kubernetes-list-type: atomic + opensearch: + items: + properties: + address: + type: string + connection: + description: Empty takes the definition's default connection + of the provider's type + title: Connection + type: string + every: + description: Re-read the rows written since the last + read at this interval such as 30s + title: Every + type: string + from: + description: Start of the window; a time or date math + such as now-1h + title: From + type: string + index: + type: string + limit: + description: Most rows one read keeps; 0 keeps all + format: int32 + title: Limit + type: integer + name: + description: Names the capture's session; empty takes + -
--- + title: Name + type: string + pagingMode: + type: string + pagingThreshold: + format: int32 + nullable: true + type: integer + params: + description: The query's params by name + title: Params + type: object + x-kubernetes-preserve-unknown-fields: true + query: + title: Query + type: string + search: + nullable: true + properties: + aggregations: + type: object + x-kubernetes-preserve-unknown-fields: true + fields: + items: + type: string + type: array + x-kubernetes-list-type: atomic + from: + format: int32 + nullable: true + type: integer + query: + nullable: true + properties: + analyzer: + type: string + boost: + format: double + nullable: true + type: number + caseInsensitive: + nullable: true + type: boolean + conditions: + items: + x-kubernetes-preserve-unknown-fields: true + type: array + x-kubernetes-list-type: atomic + escape: + nullable: true + type: boolean + field: + type: string + fields: + items: + type: string + type: array + x-kubernetes-list-type: atomic + format: + type: string + fuzziness: + type: string + gt: + nullable: true + properties: + Literal: + x-kubernetes-preserve-unknown-fields: true + Param: + type: string + required: + - Param + type: object + gte: + nullable: true + properties: + Literal: + x-kubernetes-preserve-unknown-fields: true + Param: + type: string + required: + - Param + type: object + lt: + nullable: true + properties: + Literal: + x-kubernetes-preserve-unknown-fields: true + Param: + type: string + required: + - Param + type: object + lte: + nullable: true + properties: + Literal: + x-kubernetes-preserve-unknown-fields: true + Param: + type: string + required: + - Param + type: object + matchOperator: + type: string + minimumShouldMatch: + type: string + multiMatchType: + type: string + occur: + type: string + op: + type: string + optional: + type: boolean + path: + type: string + scoreMode: + type: string + slop: + format: int32 + nullable: true + type: integer + timeZone: + type: string + value: + nullable: true + properties: + Literal: + x-kubernetes-preserve-unknown-fields: true + Param: + type: string + required: + - Param + type: object + values: + items: + properties: + Literal: + x-kubernetes-preserve-unknown-fields: true + Param: + type: string + required: + - Param + type: object + type: array + x-kubernetes-list-type: atomic + when: + type: string + required: + - op + type: object + size: + format: int32 + nullable: true + type: integer + sort: + items: + properties: + field: + type: string + missing: + type: string + mode: + type: string + order: + type: string + unmappedType: + type: string + required: + - field + type: object + type: array + x-kubernetes-list-type: atomic + source: + nullable: true + properties: + enabled: + nullable: true + type: boolean + excludes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + includes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + storedFields: + items: + type: string + type: array + x-kubernetes-list-type: atomic + timeField: + type: string + timeFieldFormat: + type: string + trackTotalHits: + nullable: true + properties: + enabled: + nullable: true + type: boolean + threshold: + format: int32 + nullable: true + type: integer + type: object + type: object + tailLag: + type: string + tailPoll: + type: string + to: + description: End of the window; empty is now + title: To + type: string + untilStopped: + description: Read once when the run stops over the whole + run + title: Until stopped + type: boolean + type: object + title: OpenSearch + type: array + x-kubernetes-list-type: atomic + opentelemetry: + items: + properties: + connection: + description: Empty takes the definition's default connection + of the provider's type + title: Connection + type: string + dateField: + type: string + every: + description: Re-read the rows written since the last + read at this interval such as 30s + title: Every + type: string + format: + type: string + from: + description: Start of the window; a time or date math + such as now-1h + title: From + type: string + index: + type: string + limit: + description: Most rows one read keeps; 0 keeps all + format: int32 + title: Limit + type: integer + name: + description: Names the capture's session; empty takes + -
--- + title: Name + type: string + pagingMode: + type: string + pagingThreshold: + format: int32 + nullable: true + type: integer + params: + description: The query's params by name + title: Params + type: object + x-kubernetes-preserve-unknown-fields: true + query: + title: Query + type: string + search: + nullable: true + properties: + aggregations: + type: object + x-kubernetes-preserve-unknown-fields: true + fields: + items: + type: string + type: array + x-kubernetes-list-type: atomic + from: + format: int32 + nullable: true + type: integer + query: + nullable: true + properties: + analyzer: + type: string + boost: + format: double + nullable: true + type: number + caseInsensitive: + nullable: true + type: boolean + conditions: + items: + x-kubernetes-preserve-unknown-fields: true + type: array + x-kubernetes-list-type: atomic + escape: + nullable: true + type: boolean + field: + type: string + fields: + items: + type: string + type: array + x-kubernetes-list-type: atomic + format: + type: string + fuzziness: + type: string + gt: + nullable: true + properties: + Literal: + x-kubernetes-preserve-unknown-fields: true + Param: + type: string + required: + - Param + type: object + gte: + nullable: true + properties: + Literal: + x-kubernetes-preserve-unknown-fields: true + Param: + type: string + required: + - Param + type: object + lt: + nullable: true + properties: + Literal: + x-kubernetes-preserve-unknown-fields: true + Param: + type: string + required: + - Param + type: object + lte: + nullable: true + properties: + Literal: + x-kubernetes-preserve-unknown-fields: true + Param: + type: string + required: + - Param + type: object + matchOperator: + type: string + minimumShouldMatch: + type: string + multiMatchType: + type: string + occur: + type: string + op: + type: string + optional: + type: boolean + path: + type: string + scoreMode: + type: string + slop: + format: int32 + nullable: true + type: integer + timeZone: + type: string + value: + nullable: true + properties: + Literal: + x-kubernetes-preserve-unknown-fields: true + Param: + type: string + required: + - Param + type: object + values: + items: + properties: + Literal: + x-kubernetes-preserve-unknown-fields: true + Param: + type: string + required: + - Param + type: object + type: array + x-kubernetes-list-type: atomic + when: + type: string + required: + - op + type: object + size: + format: int32 + nullable: true + type: integer + sort: + items: + properties: + field: + type: string + missing: + type: string + mode: + type: string + order: + type: string + unmappedType: + type: string + required: + - field + type: object + type: array + x-kubernetes-list-type: atomic + source: + nullable: true + properties: + enabled: + nullable: true + type: boolean + excludes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + includes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + storedFields: + items: + type: string + type: array + x-kubernetes-list-type: atomic + timeField: + type: string + timeFieldFormat: + type: string + trackTotalHits: + nullable: true + properties: + enabled: + nullable: true + type: boolean + threshold: + format: int32 + nullable: true + type: integer + type: object + type: object + selectFields: + items: + type: string + type: array + x-kubernetes-list-type: atomic + sourceExcludes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + to: + description: End of the window; empty is now + title: To + type: string + untilStopped: + description: Read once when the run stops over the whole + run + title: Until stopped + type: boolean + type: object + title: OpenTelemetry + type: array + x-kubernetes-list-type: atomic + postgrest: + items: + properties: + connection: + description: Empty takes the definition's default connection + of the provider's type + title: Connection + type: string + every: + description: Re-read the rows written since the last + read at this interval such as 30s + title: Every + type: string + from: + description: Start of the window; a time or date math + such as now-1h + title: From + type: string + jsonpath: + type: string + limit: + description: Most rows one read keeps; 0 keeps all + format: int32 + title: Limit + type: integer + name: + description: Names the capture's session; empty takes + -
--- + title: Name + type: string + params: + description: The query's params by name + title: Params + type: object + x-kubernetes-preserve-unknown-fields: true + query: + title: Query + type: string + to: + description: End of the window; empty is now + title: To + type: string + untilStopped: + description: Read once when the run stops over the whole + run + title: Until stopped + type: boolean + url: + type: string + type: object + title: PostgREST + type: array + x-kubernetes-list-type: atomic + profile: + items: + properties: + every: + description: Re-read the rows written since the last + read at this interval such as 30s + title: Every + type: string + from: + description: Start of the window; a time or date math + such as now-1h + title: From + type: string + limit: + description: Most rows one read keeps; 0 keeps all + format: int32 + title: Limit + type: integer + name: + description: Names the capture's session; empty takes + -
--- + title: Name + type: string + params: + description: The query's params by name + title: Params + type: object + x-kubernetes-preserve-unknown-fields: true + profile: + description: A saved profile of the config dirs' profile + store + title: Profile + type: string + to: + description: End of the window; empty is now + title: To + type: string + untilStopped: + description: Read once when the run stops over the whole + run + title: Until stopped + type: boolean + required: + - profile + type: object + title: Saved profile + type: array + x-kubernetes-list-type: atomic + prometheus: + items: + properties: + connection: + description: Empty takes the definition's default connection + of the provider's type + title: Connection + type: string + every: + description: Re-read the rows written since the last + read at this interval such as 30s + title: Every + type: string + from: + description: Start of the window; a time or date math + such as now-1h + title: From + type: string + limit: + description: Most rows one read keeps; 0 keeps all + format: int32 + title: Limit + type: integer + name: + description: Names the capture's session; empty takes + -
--- + title: Name + type: string + params: + description: The query's params by name + title: Params + type: object + x-kubernetes-preserve-unknown-fields: true + query: + title: Query + type: string + selectLabels: + items: + type: string + title: Select labels + type: array + x-kubernetes-list-type: atomic + step: + description: Range-query step such as 15s; empty reads + instants + title: Step + type: string + to: + description: End of the window; empty is now + title: To + type: string + untilStopped: + description: Read once when the run stops over the whole + run + title: Until stopped + type: boolean + url: + description: A Prometheus URL in place of a connection + title: URL + type: string + type: object + title: Prometheus + type: array + x-kubernetes-list-type: atomic + sql: + items: + properties: + connection: + description: Empty takes the definition's default connection + of the provider's type + title: Connection + type: string + database: + title: Database + type: string + every: + description: Re-read the rows written since the last + read at this interval such as 30s + title: Every + type: string + from: + description: Start of the window; a time or date math + such as now-1h + title: From + type: string + limit: + description: Most rows one read keeps; 0 keeps all + format: int32 + title: Limit + type: integer + name: + description: Names the capture's session; empty takes + -
--- + title: Name + type: string + params: + description: The query's params by name + title: Params + type: object + x-kubernetes-preserve-unknown-fields: true + query: + title: Query + type: string + to: + description: End of the window; empty is now + title: To + type: string + type: + description: postgres mysql sqlserver clickhouse or + sqlite + enum: + - postgres + - mysql + - sqlserver + - clickhouse + - sqlite + title: Type + type: string + untilStopped: + description: Read once when the run stops over the whole + run + title: Until stopped + type: boolean + url: + description: An inline DSN in place of a connection + title: URL + type: string + required: + - type + type: object + title: SQL + type: array + x-kubernetes-list-type: atomic + title: Query + type: object + sqlserver: + properties: + deadlockReport: + items: + properties: + connection: + title: Connection + type: string + databases: + description: Participant database patterns; empty selects + the connected database + items: + type: string + title: Databases + type: array + x-kubernetes-list-type: atomic + from: + description: Historical window start in RFC3339; empty + follows from now + title: From + type: string + name: + description: Names the capture's session; empty takes + -
--- + title: Name + type: string + poll: + description: Follow poll interval + title: Poll + type: string + to: + description: Historical window end in RFC3339; empty + uses server time + title: To + type: string + type: object + title: Deadlock report + type: array + x-kubernetes-list-type: atomic + planCache: + items: + properties: + connection: + description: SQL Server connection reference + title: Connection + type: string + databases: + description: Database patterns with * wildcards and + ! exclusions; empty selects the connected database + items: + type: string + title: Databases + type: array + x-kubernetes-list-type: atomic + minExecutions: + description: Minimum interval executions to record + format: int32 + title: Minimum executions + type: integer + name: + description: Names the capture's session; empty takes + -
--- + title: Name + type: string + poll: + description: Poll interval such as 30s + title: Poll + type: string + text: + description: SQL text patterns with * wildcards and + ! exclusions; bare values match substrings + items: + type: string + title: Statement text + type: array + x-kubernetes-list-type: atomic + top: + description: Maximum cost samples per poll ranked by + interval CPU; defaults to 200 + format: int32 + title: Top + type: integer + type: object + title: Plan cache + type: array + x-kubernetes-list-type: atomic + systemHealth: + description: Attach read-only to the system_health session + items: + properties: + apps: + items: + type: string + type: array + x-kubernetes-list-type: atomic + connection: + title: Connection + type: string + databases: + items: + type: string + type: array + x-kubernetes-list-type: atomic + events: + items: + type: string + type: array + x-kubernetes-list-type: atomic + hosts: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxEvents: + format: int32 + type: integer + maxMemoryKb: + format: int32 + type: integer + minDuration: + type: string + name: + description: Names the capture's session; empty takes + -
--- + title: Name + type: string + poll: + type: string + session: + enum: + - system_health + title: Existing session + type: string + sessionName: + type: string + tables: + items: + type: string + type: array + x-kubernetes-list-type: atomic + types: + items: + type: string + type: array + x-kubernetes-list-type: atomic + users: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + title: system_health + type: array + x-kubernetes-list-type: atomic + top: + description: Samples sys.dm_exec_requests top style + items: + properties: + connection: + description: Postgres or SQL Server connection reference + for the database seam + title: Connection + type: string + databases: + description: Database patterns with * wildcards and + ! exclusions; empty keeps every database + items: + type: string + title: Databases + type: array + x-kubernetes-list-type: atomic + idle: + description: Also record idle sessions; sessions holding + an open transaction are always recorded + title: Idle sessions + type: boolean + minElapsed: + description: Only record requests running at least this + long such as 500ms + title: Minimum elapsed + type: string + name: + description: Names the capture's session; empty takes + -
--- + title: Name + type: string + poll: + description: Poll interval such as 5s + title: Poll + type: string + text: + description: SQL text patterns with * wildcards and + ! exclusions; bare values match substrings + items: + type: string + title: Statement text + type: array + x-kubernetes-list-type: atomic + top: + description: Maximum requests per poll ranked by elapsed + time; defaults to 50 + format: int32 + title: Top + type: integer + type: object + title: Running requests + type: array + x-kubernetes-list-type: atomic + title: SQL Server + type: object + telltale: + properties: + exec: + items: + properties: + agent: + description: Read through a connected agent instead + of the execution's process + nullable: true + properties: + id: + title: Agent ID + type: string + selector: + additionalProperties: + type: string + description: Agent resource attributes to match + title: Selector + type: object + wait: + description: Wait for a matching agent to connect + title: Wait + type: boolean + title: Agent + type: object + executable: + description: Executable name glob + title: Executable + type: string + name: + description: Names the capture's session; empty takes + -
--- + title: Name + type: string + tailBytes: + description: Maximum bytes of each output tail + format: int32 + title: Tail bytes + type: integer + type: object + title: Executions + type: array + x-kubernetes-list-type: atomic + http: + items: + properties: + agent: + description: Read through a connected agent instead + of the execution's process + nullable: true + properties: + id: + title: Agent ID type: string - trackTotalHits: - nullable: true - properties: - enabled: - nullable: true - type: boolean - threshold: - format: int32 - nullable: true - type: integer + selector: + additionalProperties: + type: string + description: Agent resource attributes to match + title: Selector type: object + wait: + description: Wait for a matching agent to connect + title: Wait + type: boolean + title: Agent type: object - selectFields: + bodies: + description: Retained bytes of each body default 256KiB + maximum 4MiB; zero keeps none + title: Bodies + type: string + dedupWindow: + description: Store an identical exchange only once within + this duration such as 1h + title: Dedup window + type: string + direction: + default: both + description: Incoming requests outgoing clients or both + enum: + - incoming + - outgoing + - both + title: Direction + type: string + features: + description: 'Telltale''s transports to observe by their + HAR level: http prometheus loki opensearch git ...' items: type: string + title: Features type: array x-kubernetes-list-type: atomic - serviceField: - type: string - sourceExcludes: + hosts: + description: Host globs to record items: type: string + title: Hosts type: array x-kubernetes-list-type: atomic - spanIdField: + name: + description: Names the capture's session; empty takes + -
--- + title: Name type: string - statusFields: + paths: + description: URL path globs to record items: type: string + title: Paths type: array x-kubernetes-list-type: atomic - to: - description: End of the window; empty is now - title: To - type: string - traceIdField: - type: string - untilStopped: - description: Read once when the run stops over the whole - run - title: Until stopped - type: boolean + redact: + description: Additional header names to redact + items: + type: string + title: Redact headers + type: array + x-kubernetes-list-type: atomic + required: + - features type: object - title: OpenTelemetry + title: HTTP type: array x-kubernetes-list-type: atomic - postgrest: + llm: items: properties: - connection: - description: Empty takes the definition's default connection - of the provider's type - title: Connection - type: string - every: - description: Re-read the rows written since the last - read at this interval such as 30s - title: Every - type: string - from: - description: Start of the window; a time or date math - such as now-1h - title: From - type: string - jsonpath: + agent: + description: Read through a connected agent instead + of the execution's process + nullable: true + properties: + id: + title: Agent ID + type: string + selector: + additionalProperties: + type: string + description: Agent resource attributes to match + title: Selector + type: object + wait: + description: Wait for a matching agent to connect + title: Wait + type: boolean + title: Agent + type: object + includePrompts: + description: Include prompts + type: boolean + includeResponses: + description: Include responses + type: boolean + models: + description: Model glob type: string - limit: - description: Most rows one read keeps; 0 keeps all - format: int32 - title: Limit - type: integer name: description: Names the capture's session; empty takes -
--- title: Name type: string - params: - description: The query's params by name - title: Params + type: object + title: LLM calls + type: array + x-kubernetes-list-type: atomic + logs: + items: + properties: + agent: + description: Read through a connected agent instead + of the execution's process + nullable: true + properties: + id: + title: Agent ID + type: string + selector: + additionalProperties: + type: string + description: Agent resource attributes to match + title: Selector + type: object + wait: + description: Wait for a matching agent to connect + title: Wait + type: boolean + title: Agent type: object - x-kubernetes-preserve-unknown-fields: true - query: - title: Query + levels: + additionalProperties: + type: string + description: 'Refused: setting log levels changes the + agent' + title: Levels + type: object + match: + title: Match type: string - to: - description: End of the window; empty is now - title: To + minLevel: + title: Minimum level type: string - untilStopped: - description: Read once when the run stops over the whole - run - title: Until stopped - type: boolean - url: + name: + description: Names the capture's session; empty takes + -
--- + title: Name type: string type: object - title: PostgREST + title: Logs type: array x-kubernetes-list-type: atomic - profile: + spans: items: properties: - every: - description: Re-read the rows written since the last - read at this interval such as 30s - title: Every - type: string - from: - description: Start of the window; a time or date math - such as now-1h - title: From - type: string - limit: - description: Most rows one read keeps; 0 keeps all - format: int32 - title: Limit - type: integer + agent: + description: Read through a connected agent instead + of the execution's process + nullable: true + properties: + id: + title: Agent ID + type: string + selector: + additionalProperties: + type: string + description: Agent resource attributes to match + title: Selector + type: object + wait: + description: Wait for a matching agent to connect + title: Wait + type: boolean + title: Agent + type: object name: description: Names the capture's session; empty takes -
--- title: Name type: string - params: - description: The query's params by name - title: Params + type: object + title: Spans + type: array + x-kubernetes-list-type: atomic + sql: + items: + properties: + agent: + description: Read through a connected agent instead + of the execution's process + nullable: true + properties: + id: + title: Agent ID + type: string + selector: + additionalProperties: + type: string + description: Agent resource attributes to match + title: Selector + type: object + wait: + description: Wait for a matching agent to connect + title: Wait + type: boolean + title: Agent type: object - x-kubernetes-preserve-unknown-fields: true - profile: - description: A saved profile of the config dirs' profile - store - title: Profile + connections: + description: 'Connections to observe: names; * for every + one; or self for telltale''s own database' + items: + type: string + title: Connections + type: array + x-kubernetes-list-type: atomic + minDuration: + description: Record only statements that ran at least + this long such as 100ms + title: Minimum duration type: string - to: - description: End of the window; empty is now - title: To + name: + description: Names the capture's session; empty takes + -
--- + title: Name type: string - untilStopped: - description: Read once when the run stops over the whole - run - title: Until stopped - type: boolean required: - - profile + - connections type: object - title: Saved profile + title: SQL type: array x-kubernetes-list-type: atomic - prometheus: + terminal: items: properties: - connection: - description: Empty takes the definition's default connection - of the provider's type - title: Connection + command: + description: Command line named in the cast's header + items: + type: string + title: Command + type: array + x-kubernetes-list-type: atomic + height: + description: Terminal height in cells + format: int32 + title: Height + type: integer + maxBytes: + description: Most output recorded as events such as + 4MiB + title: Max bytes type: string - every: - description: Re-read the rows written since the last - read at this interval such as 30s - title: Every + name: + description: Names the capture's session; empty takes + -
--- + title: Name type: string - from: - description: Start of the window; a time or date math - such as now-1h - title: From + snapshotInterval: + description: How often the settled screen is recorded + such as 100ms + title: Snapshot interval type: string - limit: - description: Most rows one read keeps; 0 keeps all + width: + description: Terminal width in cells format: int32 - title: Limit + title: Width type: integer + type: object + title: Terminal + type: array + x-kubernetes-list-type: atomic + title: Telltale + type: object + title: Collect + type: object + connections: + description: 'Default connections: connection:// references or inline + connections' + items: + description: connection://, or an inline connection {name, + type, url, username, password, properties} + x-kubernetes-preserve-unknown-fields: true + title: Connections + type: array + x-kubernetes-list-type: atomic + description: + title: Description + type: string + every: + description: How often checkpointed captures are read such as 30s + title: Every + type: string + instrument: + description: Captures that change the target's runtime + properties: + go: + properties: + pprof: + description: 'Mutex and block profiles: sets the Go runtime''s + profile rate' + items: + properties: + agent: + description: Read through a connected agent instead + of the execution's process + nullable: true + properties: + id: + title: Agent ID + type: string + selector: + additionalProperties: + type: string + description: Agent resource attributes to match + title: Selector + type: object + wait: + description: Wait for a matching agent to connect + title: Wait + type: boolean + title: Agent + type: object name: description: Names the capture's session; empty takes -
--- title: Name type: string - params: - description: The query's params by name - title: Params - type: object - x-kubernetes-preserve-unknown-fields: true - query: - title: Query - type: string - selectLabels: - items: - type: string - title: Select labels - type: array - x-kubernetes-list-type: atomic - step: - description: Range-query step such as 15s; empty reads - instants - title: Step - type: string - to: - description: End of the window; empty is now - title: To + profile: + enum: + - mutex + - block + title: Profile type: string - untilStopped: - description: Read once when the run stops over the whole - run - title: Until stopped - type: boolean - url: - description: A Prometheus URL in place of a connection - title: URL + source: + description: gops (default) or pprof + enum: + - gops + - pprof + title: Source type: string + required: + - profile type: object - title: Prometheus + title: pprof type: array x-kubernetes-list-type: atomic - sql: + title: Go + type: object + jvm: + properties: + asyncProfiler: + description: Attaches async-profiler items: properties: - connection: - description: Empty takes the definition's default connection - of the provider's type - title: Connection - type: string - database: - title: Database + agent: + description: Read through a connected agent instead + of the execution's process + nullable: true + properties: + id: + title: Agent ID + type: string + selector: + additionalProperties: + type: string + description: Agent resource attributes to match + title: Selector + type: object + wait: + description: Wait for a matching agent to connect + title: Wait + type: boolean + title: Agent + type: object + asyncProfiler: + description: Local asprof executable for process captures + title: Async-profiler executable type: string - every: - description: Re-read the rows written since the last - read at this interval such as 30s - title: Every + event: + enum: + - cpu + - wall + - alloc + - lock + title: Event type: string - from: - description: Start of the window; a time or date math - such as now-1h - title: From + interval: + title: Sampling interval type: string - limit: - description: Most rows one read keeps; 0 keeps all + maxStackDepth: format: int32 - title: Limit + title: Maximum stack depth type: integer name: description: Names the capture's session; empty takes -
--- title: Name type: string - params: - description: The query's params by name - title: Params - type: object - x-kubernetes-preserve-unknown-fields: true - query: - title: Query - type: string - to: - description: End of the window; empty is now - title: To - type: string - type: - description: postgres mysql sqlserver clickhouse or - sqlite - enum: - - postgres - - mysql - - sqlserver - - clickhouse - - sqlite - title: Type - type: string - untilStopped: - description: Read once when the run stops over the whole - run - title: Until stopped + threads: + title: Split by thread type: boolean - url: - description: An inline DSN in place of a connection - title: URL - type: string - required: - - type type: object - title: SQL + title: async-profiler type: array x-kubernetes-list-type: atomic - title: Query - type: object - sqlserver: - properties: - deadlockReport: + expressions: items: properties: - connection: - title: Connection + agent: + description: Read through a connected agent instead + of the execution's process + nullable: true + properties: + id: + title: Agent ID + type: string + selector: + additionalProperties: + type: string + description: Agent resource attributes to match + title: Selector + type: object + wait: + description: Wait for a matching agent to connect + title: Wait + type: boolean + title: Agent + type: object + expression: + title: OGNL expression type: string - databases: - description: Participant database patterns; empty selects - the connected database - items: - type: string - title: Databases - type: array - x-kubernetes-list-type: atomic - from: - description: Historical window start in RFC3339; empty - follows from now - title: From + intervalMs: + format: int32 + title: Interval in milliseconds + type: integer + loader: + title: Application loader hash type: string name: description: Names the capture's session; empty takes -
--- title: Name type: string - poll: - description: Follow poll interval - title: Poll - type: string - to: - description: Historical window end in RFC3339; empty - uses server time - title: To - type: string + required: + - expression type: object - title: Deadlock report + title: Periodic OGNL type: array x-kubernetes-list-type: atomic - planCache: + flightRecorder: + description: Starts a JFR recording items: properties: - connection: - description: SQL Server connection reference - title: Connection - type: string - databases: - description: Database patterns with * wildcards and - ! exclusions; empty selects the connected database - items: - type: string - title: Databases - type: array - x-kubernetes-list-type: atomic - minExecutions: - description: Minimum interval executions to record + agent: + description: Read through a connected agent instead + of the execution's process + nullable: true + properties: + id: + title: Agent ID + type: string + selector: + additionalProperties: + type: string + description: Agent resource attributes to match + title: Selector + type: object + wait: + description: Wait for a matching agent to connect + title: Wait + type: boolean + title: Agent + type: object + maxStackDepth: format: int32 - title: Minimum executions + title: Maximum stack depth type: integer name: description: Names the capture's session; empty takes -
--- title: Name type: string - poll: - description: Poll interval such as 30s - title: Poll + settings: + description: profile (default) or default or a .jfc + file + title: JFR settings type: string - text: - description: SQL text patterns with * wildcards and - ! exclusions; bare values match substrings - items: - type: string - title: Statement text - type: array - x-kubernetes-list-type: atomic - top: - description: Maximum cost samples per poll ranked by - interval CPU; defaults to 200 - format: int32 - title: Top - type: integer + threads: + title: Split by thread + type: boolean type: object - title: Plan cache + title: Flight Recorder type: array x-kubernetes-list-type: atomic - systemHealth: - description: Attach read-only to the system_health session + http: items: properties: - apps: - items: - type: string - type: array - x-kubernetes-list-type: atomic - connection: - title: Connection + agent: + description: Read through a connected agent instead + of the execution's process + nullable: true + properties: + id: + title: Agent ID + type: string + selector: + additionalProperties: + type: string + description: Agent resource attributes to match + title: Selector + type: object + wait: + description: Wait for a matching agent to connect + title: Wait + type: boolean + title: Agent + type: object + bodies: + description: Retained bytes of each body default 256KiB + maximum 4MiB; zero keeps none + title: Bodies type: string - databases: - items: - type: string - type: array - x-kubernetes-list-type: atomic - events: + dedupWindow: + description: Store an identical exchange only once within + this duration such as 1h + title: Dedup window + type: string + direction: + default: both + description: Incoming requests outgoing clients or both + enum: + - incoming + - outgoing + - both + title: Direction + type: string + features: + description: 'Telltale''s transports to observe by their + HAR level: http prometheus loki opensearch git ...' items: type: string + title: Features type: array x-kubernetes-list-type: atomic hosts: + description: Host globs to record items: type: string + title: Hosts type: array x-kubernetes-list-type: atomic - maxEvents: - format: int32 - type: integer - maxMemoryKb: - format: int32 - type: integer - minDuration: - type: string name: description: Names the capture's session; empty takes -
--- title: Name type: string - poll: - type: string - session: - enum: - - system_health - title: Existing session - type: string - sessionName: - type: string - tables: + paths: + description: URL path globs to record items: type: string + title: Paths type: array x-kubernetes-list-type: atomic - types: - items: - type: string - type: array - x-kubernetes-list-type: atomic - users: + redact: + description: Additional header names to redact items: type: string + title: Redact headers type: array x-kubernetes-list-type: atomic + required: + - features type: object - title: system_health + title: HTTP exchanges type: array x-kubernetes-list-type: atomic - title: SQL Server - type: object - telltale: - properties: - exec: + httpTop: items: properties: agent: @@ -2257,25 +3368,54 @@ spec: type: boolean title: Agent type: object - executable: - description: Executable name glob - title: Executable + direction: + default: both + description: Incoming requests outgoing clients or both + enum: + - incoming + - outgoing + - both + title: Direction + type: string + hosts: + description: Host globs to record + items: + type: string + title: Hosts + type: array + x-kubernetes-list-type: atomic + minElapsed: + description: Only show requests running at least this + long + title: Minimum elapsed type: string name: description: Names the capture's session; empty takes -
--- title: Name type: string - tailBytes: - description: Maximum bytes of each output tail + paths: + description: URL path globs to record + items: + type: string + title: Paths + type: array + x-kubernetes-list-type: atomic + poll: + description: Sampling interval such as 5s + title: Poll + type: string + top: + description: Maximum active requests per sample; default + 50 format: int32 - title: Tail bytes + title: Top type: integer type: object - title: Executions + title: Active HTTP exchanges type: array x-kubernetes-list-type: atomic - http: + jms: items: properties: agent: @@ -2298,31 +3438,40 @@ spec: type: boolean title: Agent type: object - dedupWindow: - description: Store an identical exchange only once within - this duration such as 1h - title: Dedup window - type: string - features: - description: 'Telltale''s transports to observe by their - HAR level: http prometheus loki opensearch git ...' + body: + description: Capture bounded text bodies without consuming + stream cursors or deserializing objects + title: Message bodies + type: boolean + capacity: + format: int32 + title: Buffer capacity + type: integer + destinations: + description: Exact queue or topic names to observe; + empty observes all items: type: string - title: Features + title: Destinations type: array x-kubernetes-list-type: atomic + direction: + title: Direction + type: string + maxBodyBytes: + format: int32 + title: Body byte limit + type: integer name: description: Names the capture's session; empty takes -
--- title: Name type: string - required: - - features type: object - title: HTTP + title: JMS operations type: array x-kubernetes-list-type: atomic - llm: + jmsTop: items: properties: agent: @@ -2340,30 +3489,31 @@ spec: title: Selector type: object wait: - description: Wait for a matching agent to connect - title: Wait - type: boolean - title: Agent - type: object - includePrompts: - description: Include prompts - type: boolean - includeResponses: - description: Include responses - type: boolean - models: - description: Model glob - type: string + description: Wait for a matching agent to connect + title: Wait + type: boolean + title: Agent + type: object + destinations: + items: + type: string + type: array + x-kubernetes-list-type: atomic name: description: Names the capture's session; empty takes -
--- title: Name type: string + poll: + type: string + top: + format: int32 + type: integer type: object - title: LLM calls + title: JMS receive handlers type: array x-kubernetes-list-type: atomic - logs: + methods: items: properties: agent: @@ -2386,29 +3536,119 @@ spec: type: boolean title: Agent type: object - levels: - additionalProperties: - type: string - description: 'Refused: setting log levels changes the - agent' - title: Levels - type: object + alloc: + title: Allocation counters + type: boolean + arg-chars: + format: int32 + title: String character limit + type: integer + args: + title: Arguments + type: boolean + calls: + title: Nested calls + type: boolean + events: + format: int32 + title: Event limit + type: integer + exception: + title: Exceptions + type: boolean + exceptionsOnly: + description: Keep only root calls that threw + title: Exceptions only + type: boolean + expand: + format: int32 + title: Object graph depth + type: integer + gc: + title: GC counters + type: boolean match: - title: Match - type: string - minLevel: - title: Minimum level + description: Keep only root calls whose rendered argument + or return value matches every predicate + items: + properties: + index: + format: int32 + title: Argument index + type: integer + "on": + enum: + - arg + - return + title: Subject + type: string + op: + description: regex is a Java regular expression + found anywhere in the value + enum: + - equals + - contains + - regex + title: Operator + type: string + path: + description: Dotted child labels inside the value + such as customer.name + title: Path + type: string + value: + title: Value + type: string + required: + - "on" + - op + - value + type: object + title: Value matches + type: array + x-kubernetes-list-type: atomic + minCost: + description: Keep only root calls that ran at least + this long such as 250ms + title: Minimum cost type: string name: description: Names the capture's session; empty takes -
--- title: Name type: string + platform: + title: Platform calls + type: boolean + return: + title: Return values + type: boolean + targets: + items: + properties: + class: + title: Class + type: string + loader: + title: Class loader + type: string + method: + title: Method + type: string + required: + - class + - method + type: object + title: Methods + type: array + x-kubernetes-list-type: atomic + required: + - targets type: object - title: Logs + title: Watch or trace methods type: array x-kubernetes-list-type: atomic - spans: + notifications: items: properties: agent: @@ -2431,16 +3671,31 @@ spec: type: boolean title: Agent type: object + capacity: + format: int32 + title: Buffer capacity + type: integer + mbean: + title: MBean or ObjectName pattern + type: string name: description: Names the capture's session; empty takes -
--- title: Name type: string + types: + items: + type: string + title: Notification type prefixes + type: array + x-kubernetes-list-type: atomic + required: + - mbean type: object - title: Spans + title: JMX notifications and GC type: array x-kubernetes-list-type: atomic - sql: + redefine: items: properties: agent: @@ -2463,98 +3718,35 @@ spec: type: boolean title: Agent type: object - connections: - description: 'Connections to observe: names; * for every - one; or self for telltale''s own database' - items: - type: string - title: Connections - type: array - x-kubernetes-list-type: atomic - minDuration: - description: Record only statements that ran at least - this long such as 100ms - title: Minimum duration + class: + title: Class type: string - name: - description: Names the capture's session; empty takes - -
--- - title: Name + classpath: + title: Compile classpath type: string - required: - - connections - type: object - title: SQL - type: array - x-kubernetes-list-type: atomic - terminal: - items: - properties: - command: - description: Command line named in the cast's header - items: - type: string - title: Command - type: array - x-kubernetes-list-type: atomic - height: - description: Terminal height in cells - format: int32 - title: Height - type: integer - maxBytes: - description: Most output recorded as events such as - 4MiB - title: Max bytes + loader: + title: Class loader + type: string + method: + description: Empty replaces the whole class + title: Method type: string name: description: Names the capture's session; empty takes -
--- title: Name type: string - snapshotInterval: - description: How often the settled screen is recorded - such as 100ms - title: Snapshot interval + source: + title: Source type: string - width: - description: Terminal width in cells - format: int32 - title: Width - type: integer + required: + - class + - source type: object - title: Terminal + title: Redefine Java code type: array x-kubernetes-list-type: atomic - title: Telltale - type: object - title: Collect - type: object - connections: - description: 'Default connections: connection:// references or inline - connections' - items: - description: connection://, or an inline connection {name, - type, url, username, password, properties} - x-kubernetes-preserve-unknown-fields: true - title: Connections - type: array - x-kubernetes-list-type: atomic - description: - title: Description - type: string - every: - description: How often checkpointed captures are read such as 30s - title: Every - type: string - instrument: - description: Captures that change the target's runtime - properties: - go: - properties: - pprof: - description: 'Mutex and block profiles: sets the Go runtime''s - profile rate' + spans: items: properties: agent: @@ -2582,31 +3774,11 @@ spec: -
--- title: Name type: string - profile: - enum: - - mutex - - block - title: Profile - type: string - source: - description: gops (default) or pprof - enum: - - gops - - pprof - title: Source - type: string - required: - - profile type: object - title: pprof + title: Existing OTel spans type: array x-kubernetes-list-type: atomic - title: Go - type: object - jvm: - properties: - asyncProfiler: - description: Attaches async-profiler + sql: items: properties: agent: @@ -2629,39 +3801,31 @@ spec: type: boolean title: Agent type: object - asyncProfiler: - description: Local asprof executable for process captures - title: Async-profiler executable - type: string - event: - enum: - - cpu - - wall - - alloc - - lock - title: Event - type: string - interval: - title: Sampling interval + connections: + default: + - '*' + description: JVM connection identities to observe; * + selects every observed connection + items: + type: string + title: Connections + type: array + x-kubernetes-list-type: atomic + minDuration: + description: Record only statements that ran at least + this long such as 100ms + title: Minimum duration type: string - maxStackDepth: - format: int32 - title: Maximum stack depth - type: integer name: description: Names the capture's session; empty takes -
--- title: Name type: string - threads: - title: Split by thread - type: boolean type: object - title: async-profiler + title: SQL executions type: array x-kubernetes-list-type: atomic - flightRecorder: - description: Starts a JFR recording + sqlTop: items: properties: agent: @@ -2684,25 +3848,54 @@ spec: type: boolean title: Agent type: object - maxStackDepth: - format: int32 - title: Maximum stack depth - type: integer + connection: + description: Postgres or SQL Server connection reference + for the database seam + title: Connection + type: string + databases: + description: Database patterns with * wildcards and + ! exclusions; empty keeps every database + items: + type: string + title: Databases + type: array + x-kubernetes-list-type: atomic + idle: + description: Also record idle sessions; sessions holding + an open transaction are always recorded + title: Idle sessions + type: boolean + minElapsed: + description: Only record requests running at least this + long such as 500ms + title: Minimum elapsed + type: string name: description: Names the capture's session; empty takes -
--- title: Name type: string - settings: - description: profile (default) or default or a .jfc - file - title: JFR settings + poll: + description: Poll interval such as 5s + title: Poll type: string - threads: - title: Split by thread - type: boolean + text: + description: SQL text patterns with * wildcards and + ! exclusions; bare values match substrings + items: + type: string + title: Statement text + type: array + x-kubernetes-list-type: atomic + top: + description: Maximum requests per poll ranked by elapsed + time; defaults to 50 + format: int32 + title: Top + type: integer type: object - title: Flight Recorder + title: Active SQL executions type: array x-kubernetes-list-type: atomic title: JVM diff --git a/docs/_components/KindReference.test.tsx b/docs/_components/KindReference.test.tsx new file mode 100644 index 0000000..2a18be9 --- /dev/null +++ b/docs/_components/KindReference.test.tsx @@ -0,0 +1,30 @@ +import { renderToStaticMarkup } from 'react-dom/server'; +import { describe, expect, it } from 'vitest'; +import { KindReference } from './KindReference'; + +describe('trace kind enum choices', () => { + it('renders the unified process identity and both named result schemas', () => { + const html = renderToStaticMarkup(); + expect(html).toContain('process_facets'); + expect(html).toContain('process_metrics'); + expect(html).toContain('rssBytes'); + expect(html).toContain('server'); + }); + + it('lists defaults first in bold without repeating options in details', () => { + const html = renderToStaticMarkup(); + expect(html).not.toContain('Allowed values'); + expect(html).not.toContain('(one of connect, mitm)'); + expect(html).toContain('--modeconnect | mitm'); + expect(html).toContain('modeconnect | mitm'); + expect(html).toContain('--bodiesstring'); + const profile = renderToStaticMarkup(); + expect(profile).toContain('heap | cpu | allocs | mutex | block | goroutine'); + }); + + it('marks required flags and parameters with an accessible asterisk', () => { + const html = renderToStaticMarkup(); + expect(html).toContain('--connection*'); + expect(html).toContain('connection*'); + }); +}); diff --git a/docs/_components/KindReference.tsx b/docs/_components/KindReference.tsx new file mode 100644 index 0000000..7e5e65d --- /dev/null +++ b/docs/_components/KindReference.tsx @@ -0,0 +1,122 @@ +import { Fragment, useState } from 'react'; +import catalog from '../_data/kinds.json'; +import './kind-reference.css'; + +type Schema = { + type?: string; + description?: string; + properties?: Record; + required?: string[]; + enum?: unknown[]; + default?: unknown; + items?: Schema; + additionalProperties?: unknown; +}; +type Column = { name: string; type: string; label?: string; description?: string }; +type Argument = { name: string; type: string; description: string; default: string; required: boolean }; +type Kind = { name: string; params: Schema; columns: Column[]; flags: Argument[]; results?: { name: string; columns: Column[] }[] }; + +const examples: Record> = { + exec: { name: 'git', pid: 1234, durationMs: 35, exitCode: 0, stdoutTail: 'main\n' }, + goprof: { takenAt: '2026-01-01T10:00:00Z', pid: 1234, profile: 'heap', sampleType: 'inuse_space', value: 1048576, unit: 'bytes' }, + http: { startedAt: '2026-01-01T10:00:00Z', method: 'GET', host: 'example.com', url: 'https://example.com/', status: 200, durationMs: 34 }, + jvm_jfr: { timestamp: '2026-01-01T10:00:00Z', event: 'cpu', thread: 'main', frame: 'example.Service.run', samples: 3 }, + llm: { phase: 'completed', model: 'example-model', backend: 'example', durationMs: 1200, cost: 0.001 }, + logs: { firstObserved: '2026-01-01T10:00:00Z', count: 1, severity: 'INFO', message: 'Server started', source: 'application' }, + process: { key: '1234@1767261600', pid: 1234, ppid: 1200, name: 'server', command: './server', startedAt: '2026-01-01T10:00:00Z', cwd: '/work/checkout', tags: { owner: 'example' } }, + query: { time: '2026-01-01T10:00:00Z', profile: 'pod-cpu-by-pod', provider: 'prometheus', row: { pod: 'api-0', value: 0.2 } }, + spans: { traceId: '0123456789abcdef0123456789abcdef', spanId: '0123456789abcdef', service: 'checkout', operation: 'GET /orders', durationMs: 34, status: 'ok' }, + sql: { startedAt: '2026-01-01T10:00:00Z', connection: 'proxy', op: 'SELECT', sql: 'SELECT id FROM orders', durationMs: 8, rows: 2 }, + sql_deadlock: { timestamp: '2026-01-01T10:00:00Z', database: 'orders', victims: 1, participants: 2, victimStatement: 'UPDATE orders SET status = 1' }, + sql_plancache: { sampledAt: '2026-01-01T10:00:00Z', database: 'orders', executions: 5, cpuMs: 25, logicalReads: 100, sql: 'SELECT id FROM orders' }, + sql_top: { sampledAt: '2026-01-01T10:00:00Z', engine: 'postgres', database: 'orders', sessionId: 41, state: 'active', waitType: 'Lock:transactionid', blockedBy: 40, elapsedMs: 1500, sql: 'UPDATE orders SET paid = true' }, + sql_xevent: { timestamp: '2026-01-01T10:00:00Z', name: 'sql_batch_completed', database: 'orders', durationMs: 12, sql: 'SELECT id FROM orders' }, + stacks: { takenAt: '2026-01-01T10:00:00Z', pid: 1234, format: 'goroutine', goroutineId: 1, state: 'running' }, + terminal: { at: '2026-01-01T10:00:00Z', offset: 0, kind: 'output', data: 'Build complete\n', width: 120, height: 40 }, +}; + +function fields(schema: Schema, prefix = ''): Array<{ path: string; schema: Schema; required: boolean }> { + return Object.entries(schema.properties ?? {}).flatMap(([name, value]) => { + const path = prefix ? `${prefix}.${name}` : name; + if (value.properties && Object.keys(value.properties).length) return fields(value, path); + return [{ path, schema: value, required: (schema.required ?? []).includes(name) }]; + }); +} + +function valueType(schema: Schema): string { + if (schema.type === 'array') return `${schema.items?.type ?? 'object'}[]`; + return schema.type ?? (schema.additionalProperties ? 'object' : 'unknown'); +} + +function EnumOptions({ values, defaultValue }: { values: unknown[]; defaultValue?: unknown }) { + const isDefault = (value: unknown) => defaultValue !== undefined && JSON.stringify(value) === JSON.stringify(defaultValue); + const ordered = [...values.filter(isDefault), ...values.filter((value) => !isDefault(value))]; + return <>{ordered.map((value, index) => {index > 0 && ' | '}{isDefault(value) ? {String(value)} : String(value)})}; +} + +function tabsFor(kind: string) { + const record = examples[kind]; + if (!record) throw new Error(`Missing output example for ${kind}`); + return record; +} + +export function KindReference({ kind }: { kind: string }) { + const entry = (catalog as Kind[]).find((item) => item.name === kind); + if (!entry) throw new Error(`Unknown trace kind: ${kind}`); + const params = fields(entry.params); + return <> +

Arguments

+

Options for telltale run {kind}, including common run controls. Enum defaults appear first in bold; * marks required fields. Global formatting and logging options are listed in --help.

+
+ {entry.flags.map((flag) => { + const match = flag.description.match(/\s*\(one of ([^)]+)\)$/); + const options = match?.[1].split(',').map((value) => value.trim()); + const description = flag.description.replace(/\s*\(one of ([^)]+)\)$/, '').replace(/\s*\(supported seams: \[[^\]]+\]\)$/, ''); + return ; + })} +
FlagTypeDetails
--{flag.name}{flag.required && *}{options ? : flag.type}{description || '—'}{flag.default !== '' && !options?.includes(flag.default) && <> Default: {flag.default}.}
+
+ Capture parameter schema +

These JSON parameter paths are used when configuring captures and can differ from CLI flag names.

+
+ {params.map(({ path, schema, required }) => )} +
ParameterTypeDetails
{path}{required && *}{schema.enum ? : valueType(schema)}{schema.description?.replace(/\s*\(supported seams: \[[^\]]+\]\)$/, '') || '—'}{schema.default !== undefined && !schema.enum?.some((value) => JSON.stringify(value) === JSON.stringify(schema.default)) && <> Default: {JSON.stringify(schema.default)}.}
+
+

Record schema

+

Columns exposed by the kind's record query profile. Fields may be absent or empty depending on the capture.

+
+ {entry.columns.map((column) => )} +
FieldTypeDescription
{column.name}{column.type}{column.description || column.label || '—'}
+ {entry.results?.map((result) =>
+

{kind}_{result.name}

+

Read with telltale records {kind} --run <run-id> --result {result.name}.

+
+ {result.columns.map((column) => )} +
FieldTypeDescription
{column.name}{column.type}{column.description || column.label || '—'}
+
)} + + ; +} + +export function OutputExamples({ kind }: { kind: string }) { + const [tab, setTab] = useState<'JSON' | 'ANSI' | 'Web'>('JSON'); + const record = tabsFor(kind); + const entries = Object.entries(record); + return
+

Example output

+

Illustrative values and selected fields. The ANSI and Web tabs show simplified layouts; actual presentation depends on the capture and formatting options.

+
+ {(['JSON', 'ANSI', 'Web'] as const).map((name) => )} +
+
+ {tab === 'JSON' &&
{JSON.stringify(record, null, 2)}
} + {tab === 'ANSI' &&
{entries.map(([key, value]) => `${key.padEnd(16)} ${typeof value === 'object' ? JSON.stringify(value) : String(value)}`).join('\n')}
} + {tab === 'Web' &&
{entries.map(([key, value]) => )}
FieldValue
{key}{typeof value === 'object' ? JSON.stringify(value) : String(value)}
} +
+
; +} diff --git a/docs/_components/kind-reference.css b/docs/_components/kind-reference.css new file mode 100644 index 0000000..24ec6c3 --- /dev/null +++ b/docs/_components/kind-reference.css @@ -0,0 +1,6 @@ +.kind-table { overflow-x: auto; } +.kind-required { color: #c42b1c; font-weight: 700; margin-left: 0.2rem; } +.kind-output [role='tablist'] { display: flex; gap: 0.25rem; border-bottom: 1px solid var(--rp-c-divider); } +.kind-output [role='tab'] { padding: 0.5rem 1rem; border-bottom: 2px solid transparent; cursor: pointer; } +.kind-output [role='tab'][aria-selected='true'] { color: var(--rp-c-brand); border-color: var(--rp-c-brand); } +.kind-output [role='tab']:focus-visible { outline: 2px solid var(--rp-c-brand); outline-offset: -2px; } diff --git a/docs/_data/kinds.json b/docs/_data/kinds.json new file mode 100644 index 0000000..4b3f6a3 --- /dev/null +++ b/docs/_data/kinds.json @@ -0,0 +1,7528 @@ +[ + { + "name": "entity", + "title": "Entity operations", + "capabilities": { + "live": true + }, + "params": { + "type": "object", + "properties": { + "entity": { + "type": "string", + "description": "Entity name glob" + }, + "errorsOnly": { + "type": "boolean", + "description": "Record failed operations only" + }, + "target": { + "properties": { + "agent": { + "type": "string", + "title": "Agent", + "description": "Agent ID to observe" + }, + "allowMutation": { + "type": "boolean", + "title": "Allow mutation", + "description": "Allow probes to change runtime settings" + }, + "pid": { + "type": "integer", + "title": "PID", + "format": "int32", + "description": "Process to observe" + }, + "seam": { + "type": "string", + "title": "Seam", + "description": "How the capture is armed (supported seams: [agent])", + "enum": [ + "agent" + ], + "default": "agent" + }, + "selector": { + "type": "object", + "title": "Selector", + "description": "Agent resource attributes to match", + "additionalProperties": { + "type": "string" + } + }, + "wait": { + "type": "boolean", + "title": "Wait for agents", + "description": "Wait for a matching agent to connect" + } + }, + "type": "object", + "x-telltale-seams": [ + "agent" + ] + }, + "verb": { + "type": "string", + "description": "Verb glob" + } + } + }, + "columns": [ + { + "name": "truncated", + "label": "Truncated", + "type": "json", + "filter": { + "kind": "terms", + "array": true + } + }, + { + "name": "entity", + "type": "string" + }, + { + "name": "verb", + "type": "string" + }, + { + "name": "surface", + "type": "string" + }, + { + "name": "targetId", + "type": "string" + }, + { + "name": "admin", + "type": "boolean" + }, + { + "name": "readOnly", + "type": "boolean" + }, + { + "name": "parameters", + "type": "json" + }, + { + "name": "args", + "type": "json", + "filter": { + "kind": "terms", + "array": true + } + }, + { + "name": "startedAt", + "type": "datetime", + "kind": "timestamp" + }, + { + "name": "durationMs", + "type": "number" + }, + { + "name": "error", + "type": "string" + }, + { + "name": "resource", + "type": "json" + } + ], + "exportFormats": [ + { + "name": "ndjson", + "ext": "ndjson", + "mime": "application/x-ndjson" + } + ], + "flags": [ + { + "name": "agent", + "type": "string", + "description": "Agent ID to observe", + "default": "", + "required": false + }, + { + "name": "allow-mutation", + "type": "bool", + "description": "Allow probes to change runtime settings", + "default": "false", + "required": false + }, + { + "name": "data-dir", + "type": "string", + "description": "Directory holding telltale's trace data (default: $XDG_CACHE_HOME/telltale or the OS cache dir)", + "default": "", + "required": false + }, + { + "name": "duration", + "type": "duration", + "description": "Stop the run after this long, e.g. 30s (0 runs until the command exits or Ctrl+C)", + "default": "0s", + "required": false + }, + { + "name": "entity", + "type": "string", + "description": "Entity name glob", + "default": "", + "required": false + }, + { + "name": "errors-only", + "type": "bool", + "description": "Record failed operations only", + "default": "false", + "required": false + }, + { + "name": "follow", + "type": "bool", + "description": "Print every armed capture's records while recording", + "default": "false", + "required": false + }, + { + "name": "pid", + "type": "int", + "description": "Process to observe", + "default": "0", + "required": false + }, + { + "name": "seam", + "type": "string", + "description": "How the capture is armed (supported seams: [agent]) (one of agent)", + "default": "agent", + "required": false + }, + { + "name": "selector", + "type": "string", + "description": "Agent resource attributes to match", + "default": "", + "required": false + }, + { + "name": "sort", + "type": "string", + "description": "Record order: seq or timestamp", + "default": "seq", + "required": false + }, + { + "name": "tui", + "type": "bool", + "description": "Follow records in an interactive terminal", + "default": "false", + "required": false + }, + { + "name": "ui", + "type": "bool", + "description": "Open this run's traces in a local web viewer; keep serving until Ctrl+C", + "default": "false", + "required": false + }, + { + "name": "verb", + "type": "string", + "description": "Verb glob", + "default": "", + "required": false + }, + { + "name": "wait", + "type": "bool", + "description": "Wait for a matching agent to connect", + "default": "false", + "required": false + } + ] + }, + { + "name": "exec", + "title": "Executions", + "capabilities": { + "live": true + }, + "params": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Executable name glob" + }, + "tailBytes": { + "type": "integer", + "format": "int32", + "description": "Maximum bytes of each output tail", + "default": 4096 + }, + "target": { + "properties": { + "agent": { + "type": "string", + "title": "Agent", + "description": "Agent ID to observe" + }, + "allowMutation": { + "type": "boolean", + "title": "Allow mutation", + "description": "Allow probes to change runtime settings" + }, + "pid": { + "type": "integer", + "title": "PID", + "format": "int32", + "description": "Process to observe" + }, + "seam": { + "type": "string", + "title": "Seam", + "description": "How the capture is armed (supported seams: [self agent])", + "enum": [ + "self", + "agent" + ], + "default": "self" + }, + "selector": { + "type": "object", + "title": "Selector", + "description": "Agent resource attributes to match", + "additionalProperties": { + "type": "string" + } + }, + "wait": { + "type": "boolean", + "title": "Wait for agents", + "description": "Wait for a matching agent to connect" + } + }, + "type": "object", + "x-telltale-seams": [ + "self", + "agent" + ] + } + } + }, + "columns": [ + { + "name": "truncated", + "label": "Truncated", + "type": "json", + "filter": { + "kind": "terms", + "array": true + } + }, + { + "name": "name", + "type": "string" + }, + { + "name": "args", + "type": "json", + "filter": { + "kind": "terms", + "array": true + } + }, + { + "name": "dir", + "type": "string" + }, + { + "name": "envKeys", + "type": "json", + "filter": { + "kind": "terms", + "array": true + } + }, + { + "name": "pid", + "type": "number", + "format": "integer" + }, + { + "name": "startedAt", + "type": "datetime", + "kind": "timestamp" + }, + { + "name": "durationMs", + "type": "number" + }, + { + "name": "exitCode", + "type": "number", + "format": "integer" + }, + { + "name": "error", + "type": "string" + }, + { + "name": "stdoutTail", + "type": "string" + }, + { + "name": "stderrTail", + "type": "string" + }, + { + "name": "resource", + "type": "json" + } + ], + "exportFormats": [ + { + "name": "ndjson", + "ext": "ndjson", + "mime": "application/x-ndjson" + } + ], + "flags": [ + { + "name": "agent", + "type": "string", + "description": "Agent ID to observe", + "default": "", + "required": false + }, + { + "name": "allow-mutation", + "type": "bool", + "description": "Allow probes to change runtime settings", + "default": "false", + "required": false + }, + { + "name": "data-dir", + "type": "string", + "description": "Directory holding telltale's trace data (default: $XDG_CACHE_HOME/telltale or the OS cache dir)", + "default": "", + "required": false + }, + { + "name": "duration", + "type": "duration", + "description": "Stop the run after this long, e.g. 30s (0 runs until the command exits or Ctrl+C)", + "default": "0s", + "required": false + }, + { + "name": "follow", + "type": "bool", + "description": "Print every armed capture's records while recording", + "default": "false", + "required": false + }, + { + "name": "name", + "type": "string", + "description": "Executable name glob", + "default": "", + "required": false + }, + { + "name": "pid", + "type": "int", + "description": "Process to observe", + "default": "0", + "required": false + }, + { + "name": "seam", + "type": "string", + "description": "How the capture is armed (supported seams: [self agent]) (one of self, agent)", + "default": "self", + "required": false + }, + { + "name": "selector", + "type": "string", + "description": "Agent resource attributes to match", + "default": "", + "required": false + }, + { + "name": "sort", + "type": "string", + "description": "Record order: seq or timestamp", + "default": "seq", + "required": false + }, + { + "name": "tail-bytes", + "type": "int", + "description": "Maximum bytes of each output tail", + "default": "4096", + "required": false + }, + { + "name": "tui", + "type": "bool", + "description": "Follow records in an interactive terminal", + "default": "false", + "required": false + }, + { + "name": "ui", + "type": "bool", + "description": "Open this run's traces in a local web viewer; keep serving until Ctrl+C", + "default": "false", + "required": false + }, + { + "name": "wait", + "type": "bool", + "description": "Wait for a matching agent to connect", + "default": "false", + "required": false + } + ] + }, + { + "name": "goprof", + "title": "Go profiles", + "capabilities": { + "live": true + }, + "params": { + "type": "object", + "properties": { + "target": { + "properties": { + "agent": { + "type": "string", + "title": "Agent", + "description": "Agent ID to observe" + }, + "allowMutation": { + "type": "boolean", + "title": "Allow mutation", + "description": "Allow probes to change runtime settings" + }, + "pid": { + "type": "integer", + "title": "PID", + "format": "int32", + "description": "Process to observe" + }, + "seam": { + "type": "string", + "title": "Seam", + "description": "How the capture is armed (supported seams: [process agent self])", + "enum": [ + "process", + "agent", + "self" + ], + "default": "process" + }, + "selector": { + "type": "object", + "title": "Selector", + "description": "Agent resource attributes to match", + "additionalProperties": { + "type": "string" + } + }, + "wait": { + "type": "boolean", + "title": "Wait for agents", + "description": "Wait for a matching agent to connect" + } + }, + "type": "object", + "x-telltale-seams": [ + "process", + "agent", + "self" + ] + } + } + }, + "columns": [ + { + "name": "checkpointId", + "type": "string", + "filter": { + "kind": "exact" + }, + "hidden": true + }, + { + "name": "takenAt", + "label": "Taken", + "type": "datetime", + "kind": "timestamp" + }, + { + "name": "reason", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "source", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "key", + "type": "string", + "filter": { + "kind": "exact" + }, + "hidden": true + }, + { + "name": "pid", + "label": "PID", + "type": "number", + "format": "integer", + "filter": { + "kind": "terms" + } + }, + { + "name": "profile", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "sampleType", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "unit", + "type": "string" + }, + { + "name": "value", + "type": "number", + "format": "integer" + }, + { + "name": "frames", + "type": "json", + "hidden": true + }, + { + "name": "stackHash", + "type": "string", + "filter": { + "kind": "exact" + }, + "hidden": true + } + ], + "exportFormats": [ + { + "name": "ndjson", + "ext": "ndjson", + "mime": "application/x-ndjson" + }, + { + "name": "pprof", + "ext": "pb.gz", + "mime": "application/octet-stream" + } + ], + "flags": [ + { + "name": "agent", + "type": "string", + "description": "Agent ID to observe", + "default": "", + "required": false + }, + { + "name": "allow-mutation", + "type": "bool", + "description": "Allow probes to change runtime settings", + "default": "false", + "required": false + }, + { + "name": "data-dir", + "type": "string", + "description": "Directory holding telltale's trace data (default: $XDG_CACHE_HOME/telltale or the OS cache dir)", + "default": "", + "required": false + }, + { + "name": "duration", + "type": "duration", + "description": "Stop the run after this long, e.g. 30s (0 runs until the command exits or Ctrl+C)", + "default": "0s", + "required": false + }, + { + "name": "every", + "type": "duration", + "description": "Read the checkpointed captures this often, e.g. 10s", + "default": "0s", + "required": false + }, + { + "name": "follow", + "type": "bool", + "description": "Print every armed capture's records while recording", + "default": "false", + "required": false + }, + { + "name": "pid", + "type": "int", + "description": "Process to observe", + "default": "0", + "required": false + }, + { + "name": "profile", + "type": "string", + "description": "Go profile read at each checkpoint (one of cpu, heap, allocs, mutex, block, goroutine)", + "default": "heap", + "required": false + }, + { + "name": "seam", + "type": "string", + "description": "How the capture is armed (supported seams: [process agent self]) (one of process, agent, self)", + "default": "process", + "required": false + }, + { + "name": "seconds", + "type": "int", + "description": "Duration of a CPU profile window in seconds", + "default": "0", + "required": false + }, + { + "name": "selector", + "type": "string", + "description": "Agent resource attributes to match", + "default": "", + "required": false + }, + { + "name": "sort", + "type": "string", + "description": "Record order: seq or timestamp", + "default": "seq", + "required": false + }, + { + "name": "source", + "type": "string", + "description": "Process profile source (one of pprof, gops, file)", + "default": "gops", + "required": false + }, + { + "name": "tui", + "type": "bool", + "description": "Follow records in an interactive terminal", + "default": "false", + "required": false + }, + { + "name": "ui", + "type": "bool", + "description": "Open this run's traces in a local web viewer; keep serving until Ctrl+C", + "default": "false", + "required": false + }, + { + "name": "wait", + "type": "bool", + "description": "Wait for a matching agent to connect", + "default": "false", + "required": false + } + ] + }, + { + "name": "http", + "title": "HTTP proxy traffic", + "capabilities": { + "live": true + }, + "params": { + "type": "object", + "properties": { + "bodies": { + "type": "string", + "title": "Bodies", + "description": "How much of each body to keep such as 64KiB (at most 4MiB; 0 keeps none)", + "default": "256KiB" + }, + "dedupWindow": { + "type": "string", + "title": "Dedup window", + "description": "Store identical exchanges only once within this duration" + }, + "export": { + "type": "object", + "title": "Export", + "description": "Variable names the proxy and its CA are exported under", + "properties": { + "ca": { + "type": "string", + "title": "CA variable", + "description": "Variable the mitm CA file is exported under instead of SSL_CERT_FILE and the other trust variables" + }, + "noProxy": { + "type": "string", + "title": "No-proxy variable", + "description": "Variable the bypass list is exported under instead of NO_PROXY" + }, + "proxy": { + "type": "string", + "title": "Proxy variable", + "description": "Variable the proxy URL is exported under instead of HTTP(S)_PROXY" + } + } + }, + "features": { + "type": "array", + "title": "Features", + "description": "HTTP features to observe", + "default": [ + "telltale" + ], + "items": { + "type": "string" + } + }, + "hosts": { + "type": "array", + "title": "Hosts", + "description": "Host globs without a port to record and decrypt; empty records every host", + "items": { + "type": "string" + } + }, + "listen": { + "type": "string", + "title": "Listen", + "description": "host:port the proxy listens on such as 127.0.0.1:18080; empty takes a free loopback port" + }, + "mode": { + "default": "connect", + "description": "connect records HTTPS tunnels by host; mitm decrypts and records each exchange", + "enum": [ + "connect", + "mitm" + ], + "title": "Mode", + "type": "string" + }, + "noProxy": { + "type": "array", + "title": "No proxy", + "description": "host:port addresses the command reaches directly", + "items": { + "type": "string" + } + }, + "paths": { + "type": "array", + "title": "Paths", + "description": "URL path globs to record; empty records every path", + "items": { + "type": "string" + } + }, + "redact": { + "type": "array", + "title": "Redact headers", + "description": "Headers to blank beside the credential headers that are always blanked", + "items": { + "type": "string" + } + }, + "target": { + "properties": { + "agent": { + "type": "string", + "title": "Agent", + "description": "Agent ID to observe" + }, + "allowMutation": { + "type": "boolean", + "title": "Allow mutation", + "description": "Allow probes to change runtime settings" + }, + "pid": { + "type": "integer", + "title": "PID", + "format": "int32", + "description": "Process to observe" + }, + "seam": { + "type": "string", + "title": "Seam", + "description": "How the capture is armed (supported seams: [self agent proxy])", + "enum": [ + "self", + "agent", + "proxy" + ], + "default": "self" + }, + "selector": { + "type": "object", + "title": "Selector", + "description": "Agent resource attributes to match", + "additionalProperties": { + "type": "string" + } + }, + "wait": { + "type": "boolean", + "title": "Wait for agents", + "description": "Wait for a matching agent to connect" + } + }, + "type": "object", + "x-telltale-seams": [ + "self", + "agent", + "proxy" + ] + }, + "upstreamCA": { + "type": "string", + "title": "Upstream CA file", + "description": "PEM file of authorities the proxy trusts beside the system roots under mitm" + } + } + }, + "columns": [ + { + "name": "truncated", + "label": "Truncated", + "type": "json", + "filter": { + "kind": "terms", + "array": true + } + }, + { + "name": "startedDateTime", + "type": "string" + }, + { + "name": "time", + "type": "number" + }, + { + "name": "request", + "type": "json" + }, + { + "name": "response", + "type": "json" + }, + { + "name": "cache", + "type": "json" + }, + { + "name": "timings", + "type": "json" + }, + { + "name": "_id", + "type": "string" + }, + { + "name": "_pending", + "type": "boolean" + }, + { + "name": "_error", + "type": "string" + }, + { + "name": "startedAt", + "label": "Started", + "type": "datetime", + "kind": "timestamp" + }, + { + "name": "feature", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "method", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "host", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "url", + "label": "URL", + "type": "string", + "filter": { + "kind": "text" + } + }, + { + "name": "status", + "type": "number", + "format": "integer", + "filter": { + "kind": "terms" + } + }, + { + "name": "durationMs", + "label": "Duration", + "type": "duration", + "unit": "ms" + }, + { + "name": "resource", + "type": "json" + }, + { + "name": "error", + "type": "string", + "filter": { + "kind": "text" + } + }, + { + "name": "serverIPAddress", + "label": "Server IP", + "type": "string" + }, + { + "name": "tunnelled", + "type": "boolean" + }, + { + "name": "bytesIn", + "label": "Bytes in", + "type": "number", + "format": "integer" + }, + { + "name": "bytesOut", + "label": "Bytes out", + "type": "number", + "format": "integer" + }, + { + "name": "encoding", + "type": "string", + "hidden": true + }, + { + "name": "comment", + "type": "string", + "filter": { + "kind": "text" + } + } + ], + "exportFormats": [ + { + "name": "ndjson", + "ext": "ndjson", + "mime": "application/x-ndjson" + }, + { + "name": "har", + "ext": "har", + "mime": "application/json" + } + ], + "flags": [ + { + "name": "agent", + "type": "string", + "description": "Agent ID to observe", + "default": "", + "required": false + }, + { + "name": "allow-mutation", + "type": "bool", + "description": "Allow probes to change runtime settings", + "default": "false", + "required": false + }, + { + "name": "bodies", + "type": "string", + "description": "How much of each body to keep such as 64KiB (at most 4MiB; 0 keeps none)", + "default": "256KiB", + "required": false + }, + { + "name": "data-dir", + "type": "string", + "description": "Directory holding telltale's trace data (default: $XDG_CACHE_HOME/telltale or the OS cache dir)", + "default": "", + "required": false + }, + { + "name": "dedup-window", + "type": "string", + "description": "Store identical exchanges only once within this duration", + "default": "", + "required": false + }, + { + "name": "duration", + "type": "duration", + "description": "Stop the run after this long, e.g. 30s (0 runs until the command exits or Ctrl+C)", + "default": "0s", + "required": false + }, + { + "name": "export-ca", + "type": "string", + "description": "Variable the mitm CA file is exported under instead of SSL_CERT_FILE and the other trust variables", + "default": "", + "required": false + }, + { + "name": "export-no-proxy", + "type": "string", + "description": "Variable the bypass list is exported under instead of NO_PROXY", + "default": "", + "required": false + }, + { + "name": "export-proxy", + "type": "string", + "description": "Variable the proxy URL is exported under instead of HTTP(S)_PROXY", + "default": "", + "required": false + }, + { + "name": "features", + "type": "stringSlice", + "description": "HTTP features to observe", + "default": "[telltale]", + "required": false + }, + { + "name": "follow", + "type": "bool", + "description": "Print every armed capture's records while recording", + "default": "false", + "required": false + }, + { + "name": "hosts", + "type": "stringSlice", + "description": "Host globs without a port to record and decrypt; empty records every host", + "default": "[]", + "required": false + }, + { + "name": "listen", + "type": "string", + "description": "host:port the proxy listens on such as 127.0.0.1:18080; empty takes a free loopback port", + "default": "", + "required": false + }, + { + "name": "mode", + "type": "string", + "description": "connect records HTTPS tunnels by host; mitm decrypts and records each exchange (one of connect, mitm)", + "default": "connect", + "required": false + }, + { + "name": "no-proxy", + "type": "stringSlice", + "description": "host:port addresses the command reaches directly", + "default": "[]", + "required": false + }, + { + "name": "paths", + "type": "stringSlice", + "description": "URL path globs to record; empty records every path", + "default": "[]", + "required": false + }, + { + "name": "pid", + "type": "int", + "description": "Process to observe", + "default": "0", + "required": false + }, + { + "name": "redact", + "type": "stringSlice", + "description": "Headers to blank beside the credential headers that are always blanked", + "default": "[]", + "required": false + }, + { + "name": "seam", + "type": "string", + "description": "How the capture is armed (supported seams: [self agent proxy]) (one of self, agent, proxy)", + "default": "self", + "required": false + }, + { + "name": "selector", + "type": "string", + "description": "Agent resource attributes to match", + "default": "", + "required": false + }, + { + "name": "sort", + "type": "string", + "description": "Record order: seq or timestamp", + "default": "seq", + "required": false + }, + { + "name": "tui", + "type": "bool", + "description": "Follow records in an interactive terminal", + "default": "false", + "required": false + }, + { + "name": "ui", + "type": "bool", + "description": "Open this run's traces in a local web viewer; keep serving until Ctrl+C", + "default": "false", + "required": false + }, + { + "name": "upstream-ca", + "type": "string", + "description": "PEM file of authorities the proxy trusts beside the system roots under mitm", + "default": "", + "required": false + }, + { + "name": "wait", + "type": "bool", + "description": "Wait for a matching agent to connect", + "default": "false", + "required": false + } + ] + }, + { + "name": "jvm_jfr", + "title": "JFR profile samples", + "capabilities": { + "live": true + }, + "params": { + "type": "object", + "properties": { + "asyncProfiler": { + "type": "string", + "title": "Async-profiler executable", + "description": "Local asprof executable for process captures" + }, + "engine": { + "default": "jfr", + "enum": [ + "jfr", + "async-profiler" + ], + "title": "Engine", + "type": "string" + }, + "event": { + "enum": [ + "cpu", + "wall", + "alloc", + "lock" + ], + "title": "Async-profiler event", + "type": "string" + }, + "interval": { + "type": "string", + "title": "Sampling interval" + }, + "maxStackDepth": { + "type": "integer", + "title": "Maximum stack depth", + "format": "int32", + "description": "Keep this many leaf frames; zero keeps all recorded frames" + }, + "settings": { + "type": "string", + "title": "JFR settings" + }, + "target": { + "properties": { + "agent": { + "type": "string", + "title": "Agent", + "description": "Agent ID to observe" + }, + "allowMutation": { + "type": "boolean", + "title": "Allow mutation", + "description": "Allow probes to change runtime settings" + }, + "pid": { + "type": "integer", + "title": "PID", + "format": "int32", + "description": "Process to observe" + }, + "seam": { + "type": "string", + "title": "Seam", + "description": "How the capture is armed (supported seams: [process agent])", + "enum": [ + "process", + "agent" + ], + "default": "process" + }, + "selector": { + "type": "object", + "title": "Selector", + "description": "Agent resource attributes to match", + "additionalProperties": { + "type": "string" + } + }, + "wait": { + "type": "boolean", + "title": "Wait for agents", + "description": "Wait for a matching agent to connect" + } + }, + "type": "object", + "x-telltale-seams": [ + "process", + "agent" + ] + }, + "threads": { + "type": "boolean", + "title": "Split by thread", + "description": "Retain thread identities in decoded samples" + } + } + }, + "columns": [ + { + "name": "resource", + "type": "json" + }, + { + "name": "truncated", + "label": "Truncated", + "type": "json", + "filter": { + "kind": "terms", + "array": true + } + }, + { + "name": "timestamp", + "label": "At", + "type": "datetime", + "kind": "timestamp" + }, + { + "name": "from", + "type": "datetime", + "hidden": true + }, + { + "name": "event", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "thread", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "frame", + "label": "Top frame", + "type": "string", + "filter": { + "kind": "text" + } + }, + { + "name": "stack", + "type": "string", + "filter": { + "kind": "text" + }, + "hidden": true + }, + { + "name": "samples", + "type": "number", + "format": "integer" + }, + { + "name": "value", + "type": "number", + "format": "integer" + }, + { + "name": "unit", + "type": "string" + } + ], + "exportFormats": [ + { + "name": "ndjson", + "ext": "ndjson", + "mime": "application/x-ndjson" + } + ], + "flags": [ + { + "name": "agent", + "type": "string", + "description": "Agent ID to observe", + "default": "", + "required": false + }, + { + "name": "allow-mutation", + "type": "bool", + "description": "Allow probes to change runtime settings", + "default": "false", + "required": false + }, + { + "name": "async-profiler", + "type": "string", + "description": "Local asprof executable for process captures", + "default": "", + "required": false + }, + { + "name": "data-dir", + "type": "string", + "description": "Directory holding telltale's trace data (default: $XDG_CACHE_HOME/telltale or the OS cache dir)", + "default": "", + "required": false + }, + { + "name": "duration", + "type": "duration", + "description": "Stop the run after this long, e.g. 30s (0 runs until the command exits or Ctrl+C)", + "default": "0s", + "required": false + }, + { + "name": "engine", + "type": "string", + "description": "Engine (one of jfr, async-profiler)", + "default": "jfr", + "required": false + }, + { + "name": "event", + "type": "string", + "description": "Async-profiler event (one of cpu, wall, alloc, lock)", + "default": "", + "required": false + }, + { + "name": "every", + "type": "duration", + "description": "Read the checkpointed captures this often, e.g. 10s", + "default": "0s", + "required": false + }, + { + "name": "follow", + "type": "bool", + "description": "Print every armed capture's records while recording", + "default": "false", + "required": false + }, + { + "name": "interval", + "type": "string", + "description": "Sampling interval", + "default": "", + "required": false + }, + { + "name": "max-stack-depth", + "type": "int", + "description": "Keep this many leaf frames; zero keeps all recorded frames", + "default": "0", + "required": false + }, + { + "name": "pid", + "type": "int", + "description": "Process to observe", + "default": "0", + "required": false + }, + { + "name": "seam", + "type": "string", + "description": "How the capture is armed (supported seams: [process agent]) (one of process, agent)", + "default": "process", + "required": false + }, + { + "name": "selector", + "type": "string", + "description": "Agent resource attributes to match", + "default": "", + "required": false + }, + { + "name": "settings", + "type": "string", + "description": "JFR settings", + "default": "", + "required": false + }, + { + "name": "sort", + "type": "string", + "description": "Record order: seq or timestamp", + "default": "seq", + "required": false + }, + { + "name": "threads", + "type": "bool", + "description": "Retain thread identities in decoded samples", + "default": "false", + "required": false + }, + { + "name": "tui", + "type": "bool", + "description": "Follow records in an interactive terminal", + "default": "false", + "required": false + }, + { + "name": "ui", + "type": "bool", + "description": "Open this run's traces in a local web viewer; keep serving until Ctrl+C", + "default": "false", + "required": false + }, + { + "name": "wait", + "type": "bool", + "description": "Wait for a matching agent to connect", + "default": "false", + "required": false + } + ] + }, + { + "name": "llm", + "title": "LLM calls", + "capabilities": { + "live": true + }, + "params": { + "type": "object", + "properties": { + "includePrompts": { + "type": "boolean", + "description": "Include prompts" + }, + "includeResponses": { + "type": "boolean", + "description": "Include responses" + }, + "models": { + "type": "string", + "description": "Model glob" + }, + "target": { + "properties": { + "agent": { + "type": "string", + "title": "Agent", + "description": "Agent ID to observe" + }, + "allowMutation": { + "type": "boolean", + "title": "Allow mutation", + "description": "Allow probes to change runtime settings" + }, + "pid": { + "type": "integer", + "title": "PID", + "format": "int32", + "description": "Process to observe" + }, + "seam": { + "type": "string", + "title": "Seam", + "description": "How the capture is armed (supported seams: [self agent])", + "enum": [ + "self", + "agent" + ], + "default": "self" + }, + "selector": { + "type": "object", + "title": "Selector", + "description": "Agent resource attributes to match", + "additionalProperties": { + "type": "string" + } + }, + "wait": { + "type": "boolean", + "title": "Wait for agents", + "description": "Wait for a matching agent to connect" + } + }, + "type": "object", + "x-telltale-seams": [ + "self", + "agent" + ] + } + } + }, + "columns": [ + { + "name": "truncated", + "label": "Truncated", + "type": "json", + "filter": { + "kind": "terms", + "array": true + } + }, + { + "name": "phase", + "type": "string" + }, + { + "name": "model", + "type": "string" + }, + { + "name": "backend", + "type": "string" + }, + { + "name": "iter", + "type": "number", + "format": "integer" + }, + { + "name": "prompt", + "type": "string" + }, + { + "name": "tools", + "type": "json" + }, + { + "name": "response", + "type": "string" + }, + { + "name": "usage", + "type": "json" + }, + { + "name": "cost", + "type": "number" + }, + { + "name": "durationMs", + "type": "number" + }, + { + "name": "error", + "type": "string" + }, + { + "name": "resource", + "type": "json" + } + ], + "exportFormats": [ + { + "name": "ndjson", + "ext": "ndjson", + "mime": "application/x-ndjson" + } + ], + "flags": [ + { + "name": "agent", + "type": "string", + "description": "Agent ID to observe", + "default": "", + "required": false + }, + { + "name": "allow-mutation", + "type": "bool", + "description": "Allow probes to change runtime settings", + "default": "false", + "required": false + }, + { + "name": "data-dir", + "type": "string", + "description": "Directory holding telltale's trace data (default: $XDG_CACHE_HOME/telltale or the OS cache dir)", + "default": "", + "required": false + }, + { + "name": "duration", + "type": "duration", + "description": "Stop the run after this long, e.g. 30s (0 runs until the command exits or Ctrl+C)", + "default": "0s", + "required": false + }, + { + "name": "follow", + "type": "bool", + "description": "Print every armed capture's records while recording", + "default": "false", + "required": false + }, + { + "name": "include-prompts", + "type": "bool", + "description": "Include prompts", + "default": "false", + "required": false + }, + { + "name": "include-responses", + "type": "bool", + "description": "Include responses", + "default": "false", + "required": false + }, + { + "name": "models", + "type": "string", + "description": "Model glob", + "default": "", + "required": false + }, + { + "name": "pid", + "type": "int", + "description": "Process to observe", + "default": "0", + "required": false + }, + { + "name": "seam", + "type": "string", + "description": "How the capture is armed (supported seams: [self agent]) (one of self, agent)", + "default": "self", + "required": false + }, + { + "name": "selector", + "type": "string", + "description": "Agent resource attributes to match", + "default": "", + "required": false + }, + { + "name": "sort", + "type": "string", + "description": "Record order: seq or timestamp", + "default": "seq", + "required": false + }, + { + "name": "tui", + "type": "bool", + "description": "Follow records in an interactive terminal", + "default": "false", + "required": false + }, + { + "name": "ui", + "type": "bool", + "description": "Open this run's traces in a local web viewer; keep serving until Ctrl+C", + "default": "false", + "required": false + }, + { + "name": "wait", + "type": "bool", + "description": "Wait for a matching agent to connect", + "default": "false", + "required": false + } + ] + }, + { + "name": "logs", + "title": "Logs", + "capabilities": { + "live": true + }, + "params": { + "type": "object", + "properties": { + "backend": { + "type": "string" + }, + "connection": { + "type": "string" + }, + "container": { + "type": "string" + }, + "levels": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "match": { + "type": "string" + }, + "minLevel": { + "type": "string" + }, + "namespace": { + "type": "string" + }, + "options": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Dynamic response value" + } + }, + "podSelector": { + "type": "string" + }, + "query": { + "type": "string" + }, + "source": { + "type": "string" + }, + "target": { + "properties": { + "agent": { + "type": "string", + "title": "Agent", + "description": "Agent ID to observe" + }, + "allowMutation": { + "type": "boolean", + "title": "Allow mutation", + "description": "Allow probes to change runtime settings" + }, + "pid": { + "type": "integer", + "title": "PID", + "format": "int32", + "description": "Process to observe" + }, + "seam": { + "type": "string", + "title": "Seam", + "description": "How the capture is armed (supported seams: [self agent host])", + "enum": [ + "self", + "agent", + "host" + ], + "default": "self" + }, + "selector": { + "type": "object", + "title": "Selector", + "description": "Agent resource attributes to match", + "additionalProperties": { + "type": "string" + } + }, + "wait": { + "type": "boolean", + "title": "Wait for agents", + "description": "Wait for a matching agent to connect" + } + }, + "type": "object", + "x-telltale-seams": [ + "self", + "agent", + "host" + ] + }, + "timeField": { + "type": "string" + } + } + }, + "columns": [ + { + "name": "id", + "type": "string" + }, + { + "name": "firstObserved", + "type": "datetime", + "kind": "timestamp" + }, + { + "name": "lastObserved", + "type": "datetime" + }, + { + "name": "count", + "type": "number", + "format": "integer" + }, + { + "name": "message", + "type": "string" + }, + { + "name": "hash", + "type": "string" + }, + { + "name": "severity", + "type": "string" + }, + { + "name": "source", + "type": "string" + }, + { + "name": "host", + "type": "string" + }, + { + "name": "labels", + "type": "json" + }, + { + "name": "logger", + "type": "string" + }, + { + "name": "traceId", + "type": "string" + }, + { + "name": "resource", + "type": "json" + } + ], + "exportFormats": [ + { + "name": "ndjson", + "ext": "ndjson", + "mime": "application/x-ndjson" + } + ], + "flags": [ + { + "name": "agent", + "type": "string", + "description": "Agent ID to observe", + "default": "", + "required": false + }, + { + "name": "allow-mutation", + "type": "bool", + "description": "Allow probes to change runtime settings", + "default": "false", + "required": false + }, + { + "name": "backend", + "type": "string", + "description": "", + "default": "", + "required": false + }, + { + "name": "connection", + "type": "string", + "description": "", + "default": "", + "required": false + }, + { + "name": "container", + "type": "string", + "description": "", + "default": "", + "required": false + }, + { + "name": "data-dir", + "type": "string", + "description": "Directory holding telltale's trace data (default: $XDG_CACHE_HOME/telltale or the OS cache dir)", + "default": "", + "required": false + }, + { + "name": "duration", + "type": "duration", + "description": "Stop the run after this long, e.g. 30s (0 runs until the command exits or Ctrl+C)", + "default": "0s", + "required": false + }, + { + "name": "follow", + "type": "bool", + "description": "Print every armed capture's records while recording", + "default": "false", + "required": false + }, + { + "name": "levels", + "type": "stringSlice", + "description": " (key=value pairs)", + "default": "[]", + "required": false + }, + { + "name": "match", + "type": "string", + "description": "", + "default": "", + "required": false + }, + { + "name": "min-level", + "type": "string", + "description": "", + "default": "", + "required": false + }, + { + "name": "namespace", + "type": "string", + "description": "", + "default": "", + "required": false + }, + { + "name": "options", + "type": "string", + "description": " (JSON object)", + "default": "", + "required": false + }, + { + "name": "pid", + "type": "int", + "description": "Process to observe", + "default": "0", + "required": false + }, + { + "name": "pod-selector", + "type": "string", + "description": "", + "default": "", + "required": false + }, + { + "name": "query", + "type": "string", + "description": "", + "default": "", + "required": false + }, + { + "name": "seam", + "type": "string", + "description": "How the capture is armed (supported seams: [self agent host]) (one of self, agent, host)", + "default": "self", + "required": false + }, + { + "name": "selector", + "type": "string", + "description": "Agent resource attributes to match", + "default": "", + "required": false + }, + { + "name": "sort", + "type": "string", + "description": "Record order: seq or timestamp", + "default": "seq", + "required": false + }, + { + "name": "source", + "type": "string", + "description": "", + "default": "", + "required": false + }, + { + "name": "time-field", + "type": "string", + "description": "", + "default": "", + "required": false + }, + { + "name": "tui", + "type": "bool", + "description": "Follow records in an interactive terminal", + "default": "false", + "required": false + }, + { + "name": "ui", + "type": "bool", + "description": "Open this run's traces in a local web viewer; keep serving until Ctrl+C", + "default": "false", + "required": false + }, + { + "name": "wait", + "type": "bool", + "description": "Wait for a matching agent to connect", + "default": "false", + "required": false + } + ] + }, + { + "name": "process", + "title": "Processes", + "capabilities": { + "live": true + }, + "params": { + "type": "object", + "properties": { + "catalog": { + "type": "string" + }, + "environment": { + "type": "object", + "properties": { + "keys": { + "type": "array", + "title": "Keys", + "description": "Environment variables to read onto each process", + "items": { + "type": "string" + } + }, + "prefixes": { + "type": "array", + "title": "Prefixes", + "description": "Prefixes of the environment variables to read onto each process", + "items": { + "type": "string" + } + } + }, + "nullable": true + }, + "facets": { + "type": "array", + "items": { + "type": "string" + } + }, + "interval": { + "type": "string", + "title": "Interval", + "description": "How often the tree is sampled such as 1s (at least 250ms)" + }, + "net": { + "type": "boolean" + }, + "target": { + "properties": { + "agent": { + "type": "string", + "title": "Agent", + "description": "Agent ID to observe" + }, + "allowMutation": { + "type": "boolean", + "title": "Allow mutation", + "description": "Allow probes to change runtime settings" + }, + "pid": { + "type": "integer", + "title": "PID", + "format": "int32", + "description": "Process to observe" + }, + "seam": { + "type": "string", + "title": "Seam", + "description": "How the capture is armed (supported seams: [process host])", + "enum": [ + "process", + "host" + ], + "default": "process" + }, + "selector": { + "type": "object", + "title": "Selector", + "description": "Agent resource attributes to match", + "additionalProperties": { + "type": "string" + } + }, + "wait": { + "type": "boolean", + "title": "Wait for agents", + "description": "Wait for a matching agent to connect" + } + }, + "type": "object", + "x-telltale-seams": [ + "process", + "host" + ] + } + } + }, + "columns": [ + { + "name": "key", + "type": "string" + }, + { + "name": "parentKey", + "type": "string" + }, + { + "name": "pid", + "type": "number", + "format": "integer" + }, + { + "name": "ppid", + "type": "number", + "format": "integer" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "command", + "type": "string" + }, + { + "name": "cwd", + "type": "string" + }, + { + "name": "startedAt", + "type": "datetime" + }, + { + "name": "firstObservedAt", + "type": "datetime", + "kind": "timestamp" + }, + { + "name": "exitedAt", + "type": "datetime" + }, + { + "name": "environment", + "type": "json" + }, + { + "name": "tags", + "type": "json" + }, + { + "name": "environmentError", + "type": "string" + }, + { + "name": "groups", + "type": "json" + }, + { + "name": "stackTransports", + "type": "json" + } + ], + "results": [ + { + "name": "facets", + "columns": [ + { + "name": "process", + "type": "string" + }, + { + "name": "observedAt", + "type": "datetime", + "kind": "timestamp" + }, + { + "name": "tick", + "type": "number", + "format": "integer" + }, + { + "name": "facet", + "type": "string" + }, + { + "name": "entity", + "type": "string" + }, + { + "name": "operation", + "type": "string" + }, + { + "name": "directionSource", + "type": "string" + }, + { + "name": "reason", + "type": "string" + }, + { + "name": "resource", + "type": "json" + } + ] + }, + { + "name": "metrics", + "columns": [ + { + "name": "entity", + "type": "string", + "filter": { + "kind": "exact" + } + }, + { + "name": "observedAt", + "type": "datetime", + "kind": "timestamp" + }, + { + "name": "tick", + "type": "number", + "format": "integer" + }, + { + "name": "cpuPercent", + "type": "number", + "unit": "percent" + }, + { + "name": "rssBytes", + "type": "number", + "format": "integer", + "unit": "bytes" + }, + { + "name": "memoryPercent", + "type": "number", + "unit": "percent" + }, + { + "name": "diskReadBytes", + "type": "number", + "format": "integer", + "unit": "bytes" + }, + { + "name": "diskWriteBytes", + "type": "number", + "format": "integer", + "unit": "bytes" + }, + { + "name": "openFiles", + "type": "number", + "format": "integer" + }, + { + "name": "receivingSockets", + "type": "number", + "format": "integer" + }, + { + "name": "connectionsIn", + "type": "number", + "format": "integer" + }, + { + "name": "connectionsOut", + "type": "number", + "format": "integer" + }, + { + "name": "connectionsUnknown", + "type": "number", + "format": "integer" + }, + { + "name": "diskReadBps", + "type": "number", + "unit": "Bps" + }, + { + "name": "diskWriteBps", + "type": "number", + "unit": "Bps" + }, + { + "name": "netRxBps", + "type": "number", + "unit": "Bps" + }, + { + "name": "netTxBps", + "type": "number", + "unit": "Bps" + }, + { + "name": "swapBytes", + "type": "number", + "format": "integer", + "unit": "bytes" + }, + { + "name": "compressedBytes", + "type": "number", + "format": "integer", + "unit": "bytes" + } + ] + } + ], + "exportFormats": [ + { + "name": "ndjson", + "ext": "ndjson", + "mime": "application/x-ndjson" + } + ], + "flags": [ + { + "name": "catalog", + "type": "string", + "description": "", + "default": "", + "required": false + }, + { + "name": "data-dir", + "type": "string", + "description": "Directory holding telltale's trace data (default: $XDG_CACHE_HOME/telltale or the OS cache dir)", + "default": "", + "required": false + }, + { + "name": "duration", + "type": "duration", + "description": "Stop the run after this long, e.g. 30s (0 runs until the command exits or Ctrl+C)", + "default": "0s", + "required": false + }, + { + "name": "environment-keys", + "type": "stringSlice", + "description": "Environment variables to read onto each process", + "default": "[]", + "required": false + }, + { + "name": "environment-prefixes", + "type": "stringSlice", + "description": "Prefixes of the environment variables to read onto each process", + "default": "[]", + "required": false + }, + { + "name": "facets", + "type": "stringSlice", + "description": "", + "default": "[]", + "required": false + }, + { + "name": "follow", + "type": "bool", + "description": "Print every armed capture's records while recording", + "default": "false", + "required": false + }, + { + "name": "interval", + "type": "string", + "description": "How often the tree is sampled such as 1s (at least 250ms)", + "default": "", + "required": false + }, + { + "name": "net", + "type": "bool", + "description": "", + "default": "false", + "required": false + }, + { + "name": "pid", + "type": "int", + "description": "Process to observe", + "default": "0", + "required": false + }, + { + "name": "seam", + "type": "string", + "description": "How the capture is armed (supported seams: [process host]) (one of process, host)", + "default": "process", + "required": false + }, + { + "name": "sort", + "type": "string", + "description": "Record order: seq or timestamp", + "default": "seq", + "required": false + }, + { + "name": "tui", + "type": "bool", + "description": "Follow records in an interactive terminal", + "default": "false", + "required": false + }, + { + "name": "ui", + "type": "bool", + "description": "Open this run's traces in a local web viewer; keep serving until Ctrl+C", + "default": "false", + "required": false + } + ] + }, + { + "name": "prometheus", + "title": "Prometheus scrapes", + "capabilities": { + "live": true + }, + "params": { + "type": "object", + "properties": { + "endpoints": { + "type": "array", + "title": "Endpoints", + "description": "Metrics URLs using http:// or https:// or Kubernetes portforward:// (also port-forward://) or proxy://", + "items": { + "type": "string" + } + }, + "interval": { + "type": "string", + "title": "Interval", + "description": "How often each endpoint is scraped such as 15s (at least 1s)", + "default": "15s" + }, + "labels": { + "type": "object", + "title": "Labels", + "description": "Extra labels every scraped sample is stored with such as env=dev", + "additionalProperties": { + "type": "string" + } + }, + "target": { + "properties": { + "agent": { + "type": "string", + "title": "Agent", + "description": "Agent ID to observe" + }, + "allowMutation": { + "type": "boolean", + "title": "Allow mutation", + "description": "Allow probes to change runtime settings" + }, + "pid": { + "type": "integer", + "title": "PID", + "format": "int32", + "description": "Process to observe" + }, + "seam": { + "type": "string", + "title": "Seam", + "description": "How the capture is armed (supported seams: [host])", + "enum": [ + "host" + ], + "default": "host" + }, + "selector": { + "type": "object", + "title": "Selector", + "description": "Agent resource attributes to match", + "additionalProperties": { + "type": "string" + } + }, + "wait": { + "type": "boolean", + "title": "Wait for agents", + "description": "Wait for a matching agent to connect" + } + }, + "type": "object", + "x-telltale-seams": [ + "host" + ] + }, + "timeout": { + "type": "string", + "title": "Timeout", + "description": "How long one scrape may take such as 5s (at most the interval; default the shorter of 10s and the interval)" + } + } + }, + "columns": [ + { + "name": "takenAt", + "label": "Scraped", + "type": "datetime", + "kind": "timestamp" + }, + { + "name": "endpoint", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "health", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "durationMs", + "label": "Duration", + "type": "duration", + "unit": "ms" + }, + { + "name": "samples", + "type": "number", + "format": "integer" + }, + { + "name": "error", + "type": "string" + } + ], + "exportFormats": [ + { + "name": "ndjson", + "ext": "ndjson", + "mime": "application/x-ndjson" + } + ], + "flags": [ + { + "name": "data-dir", + "type": "string", + "description": "Directory holding telltale's trace data (default: $XDG_CACHE_HOME/telltale or the OS cache dir)", + "default": "", + "required": false + }, + { + "name": "duration", + "type": "duration", + "description": "Stop the run after this long, e.g. 30s (0 runs until the command exits or Ctrl+C)", + "default": "0s", + "required": false + }, + { + "name": "endpoints", + "type": "stringSlice", + "description": "Metrics URLs using http:// or https:// or Kubernetes portforward:// (also port-forward://) or proxy://", + "default": "[]", + "required": false + }, + { + "name": "follow", + "type": "bool", + "description": "Print every armed capture's records while recording", + "default": "false", + "required": false + }, + { + "name": "interval", + "type": "string", + "description": "How often each endpoint is scraped such as 15s (at least 1s)", + "default": "15s", + "required": false + }, + { + "name": "labels", + "type": "stringSlice", + "description": "Extra labels every scraped sample is stored with such as env=dev (key=value pairs)", + "default": "[]", + "required": false + }, + { + "name": "pid", + "type": "int", + "description": "Process to observe", + "default": "0", + "required": false + }, + { + "name": "seam", + "type": "string", + "description": "How the capture is armed (supported seams: [host]) (one of host)", + "default": "host", + "required": false + }, + { + "name": "sort", + "type": "string", + "description": "Record order: seq or timestamp", + "default": "seq", + "required": false + }, + { + "name": "timeout", + "type": "string", + "description": "How long one scrape may take such as 5s (at most the interval; default the shorter of 10s and the interval)", + "default": "", + "required": false + }, + { + "name": "tui", + "type": "bool", + "description": "Follow records in an interactive terminal", + "default": "false", + "required": false + }, + { + "name": "ui", + "type": "bool", + "description": "Open this run's traces in a local web viewer; keep serving until Ctrl+C", + "default": "false", + "required": false + } + ] + }, + { + "name": "query", + "title": "Query profile reads", + "capabilities": { + "historical": true, + "follow": true + }, + "params": { + "type": "object", + "properties": { + "every": { + "description": "Re-read the rows written since the last read at this interval; e.g. 15s", + "title": "Every", + "type": "string", + "x-clicky-order": 5 + }, + "from": { + "description": "Start of the window; a time or date math such as now-1h", + "title": "From", + "type": "string", + "x-clicky-order": 3 + }, + "limit": { + "description": "Most rows one read keeps; 0 keeps all", + "format": "int32", + "title": "Limit", + "type": "integer", + "x-clicky-order": 7 + }, + "params": { + "additionalProperties": { + "type": "object", + "description": "Dynamic response value" + }, + "description": "The profile's params by name", + "title": "Params", + "type": "object", + "x-clicky-order": 2 + }, + "profile": { + "description": "Saved query profile to read", + "title": "Profile", + "type": "string", + "x-clicky-order": 0 + }, + "provider": { + "type": "object", + "title": "Provider", + "format": "json", + "description": "A provider read in place: its type and connection or query and that provider's typed options", + "properties": { + "bigquery": { + "type": "object", + "properties": { + "mapping": { + "type": "object", + "properties": { + "host": { + "type": "array", + "items": { + "type": "string" + } + }, + "id": { + "type": "array", + "items": { + "type": "string" + } + }, + "ignore": { + "type": "array", + "items": { + "type": "string" + } + }, + "message": { + "type": "array", + "items": { + "type": "string" + } + }, + "severity": { + "type": "array", + "items": { + "type": "string" + } + }, + "source": { + "type": "array", + "items": { + "type": "string" + } + }, + "timestamp": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "nullable": true + }, + "project": { + "type": "string" + } + }, + "nullable": true + }, + "cloudwatchmetrics": { + "type": "object", + "properties": { + "concurrency": { + "type": "integer", + "format": "int32" + }, + "dimensions": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "endpoint": { + "type": "string" + }, + "language": { + "type": "string" + }, + "metrics": { + "type": "array", + "items": { + "type": "object", + "properties": { + "dimensions": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + }, + "statistic": { + "type": "string" + }, + "unit": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ] + } + }, + "namespace": { + "type": "string" + }, + "range": { + "type": "object", + "properties": { + "end": { + "type": "string" + }, + "start": { + "type": "string" + }, + "step": { + "type": "string" + } + }, + "required": [ + "start", + "end", + "step" + ], + "nullable": true + }, + "region": { + "type": "string" + }, + "statistic": { + "type": "string" + }, + "unit": { + "type": "string" + } + }, + "nullable": true + }, + "connection": { + "type": "string", + "title": "Connection" + }, + "http": { + "type": "object", + "properties": { + "body": { + "type": "string" + }, + "jsonpath": { + "type": "string" + }, + "method": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "nullable": true + }, + "jaeger": { + "type": "object", + "properties": { + "end": { + "type": "string" + }, + "limit": { + "type": "string" + }, + "lookback": { + "type": "string" + }, + "maxDuration": { + "type": "string" + }, + "minDuration": { + "type": "string" + }, + "operation": { + "type": "string" + }, + "service": { + "type": "string" + }, + "start": { + "type": "string" + }, + "tags": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "nullable": true + }, + "opensearch": { + "type": "object", + "properties": { + "address": { + "type": "string" + }, + "index": { + "type": "string" + }, + "limit": { + "type": "string" + }, + "pagingMode": { + "type": "string" + }, + "pagingThreshold": { + "type": "integer", + "format": "int32", + "nullable": true + }, + "search": { + "type": "object", + "properties": { + "aggregations": { + "type": "object", + "additionalProperties": {} + }, + "fields": { + "type": "array", + "items": { + "type": "string" + } + }, + "from": { + "type": "integer", + "format": "int32", + "nullable": true + }, + "query": { + "type": "object", + "properties": { + "analyzer": { + "type": "string" + }, + "boost": { + "type": "number", + "format": "double", + "nullable": true + }, + "caseInsensitive": { + "type": "boolean", + "nullable": true + }, + "conditions": { + "type": "array", + "items": { + "type": "object", + "description": "Recursive reference to esdsl.Condition" + } + }, + "escape": { + "type": "boolean", + "nullable": true + }, + "field": { + "type": "string" + }, + "fields": { + "type": "array", + "items": { + "type": "string" + } + }, + "format": { + "type": "string" + }, + "fuzziness": { + "type": "string" + }, + "gt": { + "type": "object", + "properties": { + "Literal": { + "type": "object", + "description": "Dynamic response value" + }, + "Param": { + "type": "string" + } + }, + "required": [ + "Param" + ], + "nullable": true + }, + "gte": { + "type": "object", + "properties": { + "Literal": { + "type": "object", + "description": "Dynamic response value" + }, + "Param": { + "type": "string" + } + }, + "required": [ + "Param" + ], + "nullable": true + }, + "lt": { + "type": "object", + "properties": { + "Literal": { + "type": "object", + "description": "Dynamic response value" + }, + "Param": { + "type": "string" + } + }, + "required": [ + "Param" + ], + "nullable": true + }, + "lte": { + "type": "object", + "properties": { + "Literal": { + "type": "object", + "description": "Dynamic response value" + }, + "Param": { + "type": "string" + } + }, + "required": [ + "Param" + ], + "nullable": true + }, + "matchOperator": { + "type": "string" + }, + "minimumShouldMatch": { + "type": "string" + }, + "multiMatchType": { + "type": "string" + }, + "occur": { + "type": "string" + }, + "op": { + "type": "string" + }, + "optional": { + "type": "boolean" + }, + "path": { + "type": "string" + }, + "scoreMode": { + "type": "string" + }, + "slop": { + "type": "integer", + "format": "int32", + "nullable": true + }, + "timeZone": { + "type": "string" + }, + "value": { + "type": "object", + "properties": { + "Literal": { + "type": "object", + "description": "Dynamic response value" + }, + "Param": { + "type": "string" + } + }, + "required": [ + "Param" + ], + "nullable": true + }, + "values": { + "type": "array", + "items": { + "type": "object", + "properties": { + "Literal": { + "type": "object", + "description": "Dynamic response value" + }, + "Param": { + "type": "string" + } + }, + "required": [ + "Param" + ] + } + }, + "when": { + "type": "string" + } + }, + "required": [ + "op" + ], + "nullable": true + }, + "size": { + "type": "integer", + "format": "int32", + "nullable": true + }, + "sort": { + "type": "array", + "items": { + "type": "object", + "properties": { + "field": { + "type": "string" + }, + "missing": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "order": { + "type": "string" + }, + "unmappedType": { + "type": "string" + } + }, + "required": [ + "field" + ] + } + }, + "source": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "nullable": true + }, + "excludes": { + "type": "array", + "items": { + "type": "string" + } + }, + "includes": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "nullable": true + }, + "storedFields": { + "type": "array", + "items": { + "type": "string" + } + }, + "timeField": { + "type": "string" + }, + "timeFieldFormat": { + "type": "string" + }, + "trackTotalHits": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "nullable": true + }, + "threshold": { + "type": "integer", + "format": "int32", + "nullable": true + } + }, + "nullable": true + } + }, + "nullable": true + }, + "tailLag": { + "type": "string" + }, + "tailPoll": { + "type": "string" + } + }, + "nullable": true + }, + "opentelemetry": { + "type": "object", + "properties": { + "dateField": { + "type": "string" + }, + "format": { + "type": "string" + }, + "index": { + "type": "string" + }, + "limit": { + "type": "integer", + "format": "int32" + }, + "pagingMode": { + "type": "string" + }, + "pagingThreshold": { + "type": "integer", + "format": "int32", + "nullable": true + }, + "search": { + "type": "object", + "properties": { + "aggregations": { + "type": "object", + "additionalProperties": {} + }, + "fields": { + "type": "array", + "items": { + "type": "string" + } + }, + "from": { + "type": "integer", + "format": "int32", + "nullable": true + }, + "query": { + "type": "object", + "properties": { + "analyzer": { + "type": "string" + }, + "boost": { + "type": "number", + "format": "double", + "nullable": true + }, + "caseInsensitive": { + "type": "boolean", + "nullable": true + }, + "conditions": { + "type": "array", + "items": { + "type": "object", + "description": "Recursive reference to esdsl.Condition" + } + }, + "escape": { + "type": "boolean", + "nullable": true + }, + "field": { + "type": "string" + }, + "fields": { + "type": "array", + "items": { + "type": "string" + } + }, + "format": { + "type": "string" + }, + "fuzziness": { + "type": "string" + }, + "gt": { + "type": "object", + "properties": { + "Literal": { + "type": "object", + "description": "Dynamic response value" + }, + "Param": { + "type": "string" + } + }, + "required": [ + "Param" + ], + "nullable": true + }, + "gte": { + "type": "object", + "properties": { + "Literal": { + "type": "object", + "description": "Dynamic response value" + }, + "Param": { + "type": "string" + } + }, + "required": [ + "Param" + ], + "nullable": true + }, + "lt": { + "type": "object", + "properties": { + "Literal": { + "type": "object", + "description": "Dynamic response value" + }, + "Param": { + "type": "string" + } + }, + "required": [ + "Param" + ], + "nullable": true + }, + "lte": { + "type": "object", + "properties": { + "Literal": { + "type": "object", + "description": "Dynamic response value" + }, + "Param": { + "type": "string" + } + }, + "required": [ + "Param" + ], + "nullable": true + }, + "matchOperator": { + "type": "string" + }, + "minimumShouldMatch": { + "type": "string" + }, + "multiMatchType": { + "type": "string" + }, + "occur": { + "type": "string" + }, + "op": { + "type": "string" + }, + "optional": { + "type": "boolean" + }, + "path": { + "type": "string" + }, + "scoreMode": { + "type": "string" + }, + "slop": { + "type": "integer", + "format": "int32", + "nullable": true + }, + "timeZone": { + "type": "string" + }, + "value": { + "type": "object", + "properties": { + "Literal": { + "type": "object", + "description": "Dynamic response value" + }, + "Param": { + "type": "string" + } + }, + "required": [ + "Param" + ], + "nullable": true + }, + "values": { + "type": "array", + "items": { + "type": "object", + "properties": { + "Literal": { + "type": "object", + "description": "Dynamic response value" + }, + "Param": { + "type": "string" + } + }, + "required": [ + "Param" + ] + } + }, + "when": { + "type": "string" + } + }, + "required": [ + "op" + ], + "nullable": true + }, + "size": { + "type": "integer", + "format": "int32", + "nullable": true + }, + "sort": { + "type": "array", + "items": { + "type": "object", + "properties": { + "field": { + "type": "string" + }, + "missing": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "order": { + "type": "string" + }, + "unmappedType": { + "type": "string" + } + }, + "required": [ + "field" + ] + } + }, + "source": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "nullable": true + }, + "excludes": { + "type": "array", + "items": { + "type": "string" + } + }, + "includes": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "nullable": true + }, + "storedFields": { + "type": "array", + "items": { + "type": "string" + } + }, + "timeField": { + "type": "string" + }, + "timeFieldFormat": { + "type": "string" + }, + "trackTotalHits": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "nullable": true + }, + "threshold": { + "type": "integer", + "format": "int32", + "nullable": true + } + }, + "nullable": true + } + }, + "nullable": true + }, + "selectFields": { + "type": "array", + "items": { + "type": "string" + } + }, + "sourceExcludes": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "nullable": true + }, + "postgrest": { + "type": "object", + "properties": { + "jsonpath": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "nullable": true + }, + "prometheus": { + "type": "object", + "properties": { + "language": { + "type": "string" + }, + "range": { + "type": "object", + "properties": { + "end": { + "type": "string" + }, + "start": { + "type": "string" + }, + "step": { + "type": "string" + } + }, + "required": [ + "start", + "end", + "step" + ], + "nullable": true + }, + "selectLabels": { + "type": "array", + "items": { + "type": "string" + } + }, + "url": { + "type": "string" + } + }, + "nullable": true + }, + "query": { + "type": "string", + "title": "Query" + }, + "sql": { + "type": "object", + "properties": { + "database": { + "type": "string" + }, + "partitions": { + "type": "object", + "properties": { + "concurrency": { + "type": "integer", + "format": "int32" + }, + "files": { + "type": "array", + "items": { + "type": "string" + } + }, + "glob": { + "type": "string" + }, + "immutable": { + "type": "boolean" + }, + "labels": { + "type": "string" + } + }, + "nullable": true + }, + "type": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "nullable": true + }, + "type": { + "type": "string", + "title": "Type" + } + }, + "required": [ + "type" + ], + "nullable": true + }, + "target": { + "properties": { + "agent": { + "type": "string", + "title": "Agent", + "description": "Agent ID to observe" + }, + "allowMutation": { + "type": "boolean", + "title": "Allow mutation", + "description": "Allow probes to change runtime settings" + }, + "pid": { + "type": "integer", + "title": "PID", + "format": "int32", + "description": "Process to observe" + }, + "seam": { + "type": "string", + "title": "Seam", + "description": "How the capture is armed (supported seams: [host])", + "enum": [ + "host" + ], + "default": "host" + }, + "selector": { + "type": "object", + "title": "Selector", + "description": "Agent resource attributes to match", + "additionalProperties": { + "type": "string" + } + }, + "wait": { + "type": "boolean", + "title": "Wait for agents", + "description": "Wait for a matching agent to connect" + } + }, + "type": "object", + "x-telltale-seams": [ + "host" + ] + }, + "to": { + "description": "End of the window; empty is now", + "title": "To", + "type": "string", + "x-clicky-order": 4 + }, + "untilStopped": { + "description": "Read once when the capture stops; up to that moment", + "title": "Until stopped", + "type": "boolean", + "x-clicky-order": 6 + } + } + }, + "columns": [ + { + "name": "time", + "label": "Time", + "type": "datetime", + "kind": "timestamp" + }, + { + "name": "profile", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "provider", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "row", + "type": "json", + "hidden": true + } + ], + "exportFormats": [ + { + "name": "ndjson", + "ext": "ndjson", + "mime": "application/x-ndjson" + } + ], + "flags": [ + { + "name": "data-dir", + "type": "string", + "description": "Directory holding telltale's trace data (default: $XDG_CACHE_HOME/telltale or the OS cache dir)", + "default": "", + "required": false + }, + { + "name": "duration", + "type": "duration", + "description": "Stop the run after this long, e.g. 30s (0 runs until the command exits or Ctrl+C)", + "default": "0s", + "required": false + }, + { + "name": "every", + "type": "string", + "description": "Re-read the rows written since the last read at this interval; e.g. 15s", + "default": "", + "required": false + }, + { + "name": "follow", + "type": "bool", + "description": "Print every armed capture's records while recording", + "default": "false", + "required": false + }, + { + "name": "from", + "type": "string", + "description": "Start of the window; a time or date math such as now-1h", + "default": "", + "required": false + }, + { + "name": "limit", + "type": "int", + "description": "Most rows one read keeps; 0 keeps all", + "default": "0", + "required": false + }, + { + "name": "params", + "type": "string", + "description": "The profile's params by name (JSON object)", + "default": "", + "required": false + }, + { + "name": "pid", + "type": "int", + "description": "Process to observe", + "default": "0", + "required": false + }, + { + "name": "profile", + "type": "string", + "description": "Saved query profile to read", + "default": "", + "required": false + }, + { + "name": "provider", + "type": "string", + "description": "A provider read in place: its type and connection or query and that provider's typed options (JSON object)", + "default": "", + "required": false + }, + { + "name": "seam", + "type": "string", + "description": "How the capture is armed (supported seams: [host]) (one of host)", + "default": "host", + "required": false + }, + { + "name": "sort", + "type": "string", + "description": "Record order: seq or timestamp", + "default": "seq", + "required": false + }, + { + "name": "to", + "type": "string", + "description": "End of the window; empty is now", + "default": "", + "required": false + }, + { + "name": "tui", + "type": "bool", + "description": "Follow records in an interactive terminal", + "default": "false", + "required": false + }, + { + "name": "ui", + "type": "bool", + "description": "Open this run's traces in a local web viewer; keep serving until Ctrl+C", + "default": "false", + "required": false + }, + { + "name": "until-stopped", + "type": "bool", + "description": "Read once when the capture stops; up to that moment", + "default": "false", + "required": false + } + ] + }, + { + "name": "spans", + "title": "Spans", + "capabilities": { + "live": true, + "historical": true, + "follow": true + }, + "params": { + "type": "object", + "properties": { + "opensearch": { + "type": "object", + "title": "OpenSearch", + "description": "Index to read spans from on the host seam", + "properties": { + "connection": { + "type": "string", + "title": "Connection" + }, + "dateField": { + "type": "string", + "title": "Date field" + }, + "followIndex": { + "type": "boolean", + "title": "Follow index" + }, + "format": { + "type": "string", + "title": "Format" + }, + "from": { + "type": "string", + "title": "From" + }, + "index": { + "type": "string", + "title": "Index" + }, + "lag": { + "type": "string", + "title": "Lag" + }, + "poll": { + "type": "string", + "title": "Poll" + }, + "search": { + "type": "object", + "title": "Structured search", + "format": "json", + "properties": { + "aggregations": { + "type": "object", + "additionalProperties": {} + }, + "fields": { + "type": "array", + "items": { + "type": "string" + } + }, + "from": { + "type": "integer", + "format": "int32", + "nullable": true + }, + "query": { + "type": "object", + "properties": { + "analyzer": { + "type": "string" + }, + "boost": { + "type": "number", + "format": "double", + "nullable": true + }, + "caseInsensitive": { + "type": "boolean", + "nullable": true + }, + "conditions": { + "type": "array", + "items": { + "type": "object", + "description": "Recursive reference to esdsl.Condition" + } + }, + "escape": { + "type": "boolean", + "nullable": true + }, + "field": { + "type": "string" + }, + "fields": { + "type": "array", + "items": { + "type": "string" + } + }, + "format": { + "type": "string" + }, + "fuzziness": { + "type": "string" + }, + "gt": { + "type": "object", + "properties": { + "Literal": { + "type": "object", + "description": "Dynamic response value" + }, + "Param": { + "type": "string" + } + }, + "required": [ + "Param" + ], + "nullable": true + }, + "gte": { + "type": "object", + "properties": { + "Literal": { + "type": "object", + "description": "Dynamic response value" + }, + "Param": { + "type": "string" + } + }, + "required": [ + "Param" + ], + "nullable": true + }, + "lt": { + "type": "object", + "properties": { + "Literal": { + "type": "object", + "description": "Dynamic response value" + }, + "Param": { + "type": "string" + } + }, + "required": [ + "Param" + ], + "nullable": true + }, + "lte": { + "type": "object", + "properties": { + "Literal": { + "type": "object", + "description": "Dynamic response value" + }, + "Param": { + "type": "string" + } + }, + "required": [ + "Param" + ], + "nullable": true + }, + "matchOperator": { + "type": "string" + }, + "minimumShouldMatch": { + "type": "string" + }, + "multiMatchType": { + "type": "string" + }, + "occur": { + "type": "string" + }, + "op": { + "type": "string" + }, + "optional": { + "type": "boolean" + }, + "path": { + "type": "string" + }, + "scoreMode": { + "type": "string" + }, + "slop": { + "type": "integer", + "format": "int32", + "nullable": true + }, + "timeZone": { + "type": "string" + }, + "value": { + "type": "object", + "properties": { + "Literal": { + "type": "object", + "description": "Dynamic response value" + }, + "Param": { + "type": "string" + } + }, + "required": [ + "Param" + ], + "nullable": true + }, + "values": { + "type": "array", + "items": { + "type": "object", + "properties": { + "Literal": { + "type": "object", + "description": "Dynamic response value" + }, + "Param": { + "type": "string" + } + }, + "required": [ + "Param" + ] + } + }, + "when": { + "type": "string" + } + }, + "required": [ + "op" + ], + "nullable": true + }, + "size": { + "type": "integer", + "format": "int32", + "nullable": true + }, + "sort": { + "type": "array", + "items": { + "type": "object", + "properties": { + "field": { + "type": "string" + }, + "missing": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "order": { + "type": "string" + }, + "unmappedType": { + "type": "string" + } + }, + "required": [ + "field" + ] + } + }, + "source": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "nullable": true + }, + "excludes": { + "type": "array", + "items": { + "type": "string" + } + }, + "includes": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "nullable": true + }, + "storedFields": { + "type": "array", + "items": { + "type": "string" + } + }, + "timeField": { + "type": "string" + }, + "timeFieldFormat": { + "type": "string" + }, + "trackTotalHits": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "nullable": true + }, + "threshold": { + "type": "integer", + "format": "int32", + "nullable": true + } + }, + "nullable": true + } + }, + "nullable": true + }, + "to": { + "type": "string", + "title": "To" + } + }, + "required": [ + "connection" + ], + "nullable": true + }, + "target": { + "properties": { + "agent": { + "type": "string", + "title": "Agent", + "description": "Agent ID to observe" + }, + "allowMutation": { + "type": "boolean", + "title": "Allow mutation", + "description": "Allow probes to change runtime settings" + }, + "pid": { + "type": "integer", + "title": "PID", + "format": "int32", + "description": "Process to observe" + }, + "seam": { + "type": "string", + "title": "Seam", + "description": "How the capture is armed (supported seams: [self agent host])", + "enum": [ + "self", + "agent", + "host" + ], + "default": "self" + }, + "selector": { + "type": "object", + "title": "Selector", + "description": "Agent resource attributes to match", + "additionalProperties": { + "type": "string" + } + }, + "wait": { + "type": "boolean", + "title": "Wait for agents", + "description": "Wait for a matching agent to connect" + } + }, + "type": "object", + "x-telltale-seams": [ + "self", + "agent", + "host" + ] + } + } + }, + "columns": [ + { + "name": "key", + "type": "string", + "filter": { + "kind": "exact" + } + }, + { + "name": "traceId", + "type": "string", + "filter": { + "kind": "exact" + } + }, + { + "name": "spanId", + "type": "string", + "filter": { + "kind": "exact" + } + }, + { + "name": "parentId", + "type": "string", + "filter": { + "kind": "exact" + } + }, + { + "name": "service", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "operation", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "kind", + "type": "string" + }, + { + "name": "startedAt", + "type": "datetime", + "kind": "timestamp" + }, + { + "name": "durationMs", + "type": "number" + }, + { + "name": "status", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "attributes", + "type": "json", + "hidden": true + }, + { + "name": "resource", + "type": "json", + "hidden": true + }, + { + "name": "source", + "type": "json", + "hidden": true + }, + { + "name": "native", + "type": "string", + "filter": { + "disabled": true + }, + "hidden": true + }, + { + "name": "digest", + "type": "string", + "filter": { + "disabled": true + }, + "hidden": true + } + ], + "exportFormats": [ + { + "name": "ndjson", + "ext": "ndjson", + "mime": "application/x-ndjson" + } + ], + "flags": [ + { + "name": "agent", + "type": "string", + "description": "Agent ID to observe", + "default": "", + "required": false + }, + { + "name": "allow-mutation", + "type": "bool", + "description": "Allow probes to change runtime settings", + "default": "false", + "required": false + }, + { + "name": "data-dir", + "type": "string", + "description": "Directory holding telltale's trace data (default: $XDG_CACHE_HOME/telltale or the OS cache dir)", + "default": "", + "required": false + }, + { + "name": "duration", + "type": "duration", + "description": "Stop the run after this long, e.g. 30s (0 runs until the command exits or Ctrl+C)", + "default": "0s", + "required": false + }, + { + "name": "follow", + "type": "bool", + "description": "Print every armed capture's records while recording", + "default": "false", + "required": false + }, + { + "name": "opensearch-connection", + "type": "string", + "description": "Connection", + "default": "", + "required": false + }, + { + "name": "opensearch-date-field", + "type": "string", + "description": "Date field", + "default": "", + "required": false + }, + { + "name": "opensearch-follow-index", + "type": "bool", + "description": "Follow index", + "default": "false", + "required": false + }, + { + "name": "opensearch-format", + "type": "string", + "description": "Format", + "default": "", + "required": false + }, + { + "name": "opensearch-from", + "type": "string", + "description": "From", + "default": "", + "required": false + }, + { + "name": "opensearch-index", + "type": "string", + "description": "Index", + "default": "", + "required": false + }, + { + "name": "opensearch-lag", + "type": "string", + "description": "Lag", + "default": "", + "required": false + }, + { + "name": "opensearch-poll", + "type": "string", + "description": "Poll", + "default": "", + "required": false + }, + { + "name": "opensearch-search", + "type": "string", + "description": "Structured search (JSON object)", + "default": "", + "required": false + }, + { + "name": "opensearch-to", + "type": "string", + "description": "To", + "default": "", + "required": false + }, + { + "name": "pid", + "type": "int", + "description": "Process to observe", + "default": "0", + "required": false + }, + { + "name": "seam", + "type": "string", + "description": "How the capture is armed (supported seams: [self agent host]) (one of self, agent, host)", + "default": "self", + "required": false + }, + { + "name": "selector", + "type": "string", + "description": "Agent resource attributes to match", + "default": "", + "required": false + }, + { + "name": "sort", + "type": "string", + "description": "Record order: seq or timestamp", + "default": "seq", + "required": false + }, + { + "name": "tui", + "type": "bool", + "description": "Follow records in an interactive terminal", + "default": "false", + "required": false + }, + { + "name": "ui", + "type": "bool", + "description": "Open this run's traces in a local web viewer; keep serving until Ctrl+C", + "default": "false", + "required": false + }, + { + "name": "wait", + "type": "bool", + "description": "Wait for a matching agent to connect", + "default": "false", + "required": false + } + ] + }, + { + "name": "sql", + "title": "SQL statements", + "capabilities": { + "live": true + }, + "params": { + "type": "object", + "properties": { + "connection": { + "type": "string", + "title": "Connection label", + "description": "Label the statements are recorded under", + "default": "proxy" + }, + "connections": { + "type": "array", + "title": "Connections", + "description": "Connections to observe", + "default": [ + "self" + ], + "items": { + "type": "string" + } + }, + "dsn": { + "type": "string", + "title": "Upstream DSN", + "description": "The postgres:// URL to proxy with $VAR references expanded; empty reads GAVEL_DB_DSN then DATABASE_URL" + }, + "env": { + "type": "string", + "title": "DSN variable", + "description": "Variable the command reads its DSN from; the upstream is read from it and the proxied DSN exported under it" + }, + "export": { + "type": "array", + "title": "Export", + "description": "The only variables the proxied DSN is exported under instead of GAVEL_DB_DSN DATABASE_URL and the PG* variables", + "items": { + "type": "string" + } + }, + "listen": { + "type": "string", + "title": "Listen", + "description": "host:port the proxy listens on such as 127.0.0.1:15432; empty takes a free loopback port" + }, + "maxStatements": { + "type": "integer", + "title": "Maximum statements", + "format": "int32", + "description": "Most statements one capture stores before it is marked truncated", + "default": 20000 + }, + "minDuration": { + "type": "string", + "title": "Minimum duration", + "description": "Record statements that ran at least this long" + }, + "params": { + "type": "boolean", + "title": "Capture bind values", + "description": "Keep bind parameter values which carry row data and may hold secrets" + }, + "target": { + "properties": { + "agent": { + "type": "string", + "title": "Agent", + "description": "Agent ID to observe" + }, + "allowMutation": { + "type": "boolean", + "title": "Allow mutation", + "description": "Allow probes to change runtime settings" + }, + "pid": { + "type": "integer", + "title": "PID", + "format": "int32", + "description": "Process to observe" + }, + "seam": { + "type": "string", + "title": "Seam", + "description": "How the capture is armed (supported seams: [self agent proxy])", + "enum": [ + "self", + "agent", + "proxy" + ], + "default": "self" + }, + "selector": { + "type": "object", + "title": "Selector", + "description": "Agent resource attributes to match", + "additionalProperties": { + "type": "string" + } + }, + "wait": { + "type": "boolean", + "title": "Wait for agents", + "description": "Wait for a matching agent to connect" + } + }, + "type": "object", + "x-telltale-seams": [ + "self", + "agent", + "proxy" + ] + } + } + }, + "columns": [ + { + "name": "truncated", + "label": "Truncated", + "type": "json", + "filter": { + "kind": "terms", + "array": true + } + }, + { + "name": "startedAt", + "label": "Started", + "type": "datetime", + "kind": "timestamp" + }, + { + "name": "connection", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "driver", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "op", + "label": "Op", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "tables", + "type": "json", + "filter": { + "kind": "terms", + "array": true + } + }, + { + "name": "sql", + "label": "SQL", + "type": "string", + "filter": { + "kind": "text" + } + }, + { + "name": "repeatKey", + "type": "string", + "filter": { + "kind": "exact" + }, + "hidden": true + }, + { + "name": "args", + "type": "json", + "hidden": true + }, + { + "name": "durationMs", + "label": "Duration", + "type": "duration", + "unit": "ms" + }, + { + "name": "rows", + "type": "number", + "format": "integer" + }, + { + "name": "error", + "type": "string", + "filter": { + "kind": "text" + } + }, + { + "name": "resource", + "type": "json" + } + ], + "exportFormats": [ + { + "name": "ndjson", + "ext": "ndjson", + "mime": "application/x-ndjson" + } + ], + "flags": [ + { + "name": "agent", + "type": "string", + "description": "Agent ID to observe", + "default": "", + "required": false + }, + { + "name": "allow-mutation", + "type": "bool", + "description": "Allow probes to change runtime settings", + "default": "false", + "required": false + }, + { + "name": "connection", + "type": "string", + "description": "Label the statements are recorded under", + "default": "proxy", + "required": false + }, + { + "name": "connections", + "type": "stringSlice", + "description": "Connections to observe", + "default": "[self]", + "required": false + }, + { + "name": "data-dir", + "type": "string", + "description": "Directory holding telltale's trace data (default: $XDG_CACHE_HOME/telltale or the OS cache dir)", + "default": "", + "required": false + }, + { + "name": "dsn", + "type": "string", + "description": "The postgres:// URL to proxy with $VAR references expanded; empty reads GAVEL_DB_DSN then DATABASE_URL", + "default": "", + "required": false + }, + { + "name": "duration", + "type": "duration", + "description": "Stop the run after this long, e.g. 30s (0 runs until the command exits or Ctrl+C)", + "default": "0s", + "required": false + }, + { + "name": "env", + "type": "string", + "description": "Variable the command reads its DSN from; the upstream is read from it and the proxied DSN exported under it", + "default": "", + "required": false + }, + { + "name": "export", + "type": "stringSlice", + "description": "The only variables the proxied DSN is exported under instead of GAVEL_DB_DSN DATABASE_URL and the PG* variables", + "default": "[]", + "required": false + }, + { + "name": "follow", + "type": "bool", + "description": "Print every armed capture's records while recording", + "default": "false", + "required": false + }, + { + "name": "listen", + "type": "string", + "description": "host:port the proxy listens on such as 127.0.0.1:15432; empty takes a free loopback port", + "default": "", + "required": false + }, + { + "name": "max-statements", + "type": "int", + "description": "Most statements one capture stores before it is marked truncated", + "default": "20000", + "required": false + }, + { + "name": "min-duration", + "type": "string", + "description": "Record statements that ran at least this long", + "default": "", + "required": false + }, + { + "name": "params", + "type": "bool", + "description": "Keep bind parameter values which carry row data and may hold secrets", + "default": "false", + "required": false + }, + { + "name": "pid", + "type": "int", + "description": "Process to observe", + "default": "0", + "required": false + }, + { + "name": "seam", + "type": "string", + "description": "How the capture is armed (supported seams: [self agent proxy]) (one of self, agent, proxy)", + "default": "self", + "required": false + }, + { + "name": "selector", + "type": "string", + "description": "Agent resource attributes to match", + "default": "", + "required": false + }, + { + "name": "sort", + "type": "string", + "description": "Record order: seq or timestamp", + "default": "seq", + "required": false + }, + { + "name": "tui", + "type": "bool", + "description": "Follow records in an interactive terminal", + "default": "false", + "required": false + }, + { + "name": "ui", + "type": "bool", + "description": "Open this run's traces in a local web viewer; keep serving until Ctrl+C", + "default": "false", + "required": false + }, + { + "name": "wait", + "type": "bool", + "description": "Wait for a matching agent to connect", + "default": "false", + "required": false + } + ] + }, + { + "name": "sql_deadlock", + "title": "SQL Server deadlocks", + "capabilities": { + "historical": true, + "follow": true + }, + "params": { + "type": "object", + "properties": { + "connection": { + "type": "string", + "title": "Connection" + }, + "databases": { + "type": "array", + "title": "Databases", + "description": "Participant database patterns; empty selects the connected database", + "items": { + "type": "string" + } + }, + "from": { + "type": "string", + "title": "From", + "description": "Historical window start in RFC3339; empty follows from now" + }, + "poll": { + "type": "string", + "title": "Poll", + "description": "Follow poll interval", + "default": "30s" + }, + "target": { + "properties": { + "agent": { + "type": "string", + "title": "Agent", + "description": "Agent ID to observe" + }, + "allowMutation": { + "type": "boolean", + "title": "Allow mutation", + "description": "Allow probes to change runtime settings" + }, + "pid": { + "type": "integer", + "title": "PID", + "format": "int32", + "description": "Process to observe" + }, + "seam": { + "type": "string", + "title": "Seam", + "description": "How the capture is armed (supported seams: [connection])", + "enum": [ + "connection" + ], + "default": "connection" + }, + "selector": { + "type": "object", + "title": "Selector", + "description": "Agent resource attributes to match", + "additionalProperties": { + "type": "string" + } + }, + "wait": { + "type": "boolean", + "title": "Wait for agents", + "description": "Wait for a matching agent to connect" + } + }, + "type": "object", + "x-telltale-seams": [ + "connection" + ] + }, + "to": { + "type": "string", + "title": "To", + "description": "Historical window end in RFC3339; empty uses server time" + } + }, + "required": [ + "connection" + ] + }, + "columns": [ + { + "name": "id", + "type": "string", + "filter": { + "kind": "exact" + } + }, + { + "name": "timestamp", + "type": "datetime", + "kind": "timestamp" + }, + { + "name": "database", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "databases", + "type": "json", + "filter": { + "kind": "terms", + "array": true + } + }, + { + "name": "shape", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "signature", + "type": "string", + "filter": { + "kind": "text" + } + }, + { + "name": "objects", + "type": "json", + "filter": { + "kind": "terms", + "array": true + } + }, + { + "name": "indexes", + "type": "json", + "filter": { + "kind": "terms", + "array": true + } + }, + { + "name": "lockTypes", + "type": "json", + "filter": { + "kind": "terms", + "array": true + } + }, + { + "name": "hosts", + "type": "json", + "filter": { + "kind": "terms", + "array": true + } + }, + { + "name": "apps", + "type": "json", + "filter": { + "kind": "terms", + "array": true + } + }, + { + "name": "victims", + "type": "number", + "format": "integer", + "filter": { + "kind": "range" + } + }, + { + "name": "participants", + "type": "number", + "format": "integer", + "filter": { + "kind": "range" + } + }, + { + "name": "victimHost", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "victimApp", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "victimStatement", + "type": "string", + "filter": { + "kind": "text" + } + }, + { + "name": "statementTruncated", + "type": "boolean", + "filter": { + "kind": "boolean" + } + }, + { + "name": "statements", + "type": "string", + "filter": { + "kind": "text" + } + }, + { + "name": "processes", + "type": "json" + }, + { + "name": "resources", + "type": "json" + }, + { + "name": "xml", + "type": "string", + "filter": { + "disabled": true + } + }, + { + "name": "decodeError", + "type": "string", + "filter": { + "kind": "text" + } + } + ], + "exportFormats": [ + { + "name": "ndjson", + "ext": "ndjson", + "mime": "application/x-ndjson" + } + ], + "flags": [ + { + "name": "connection", + "type": "string", + "description": "Connection", + "default": "", + "required": true + }, + { + "name": "data-dir", + "type": "string", + "description": "Directory holding telltale's trace data (default: $XDG_CACHE_HOME/telltale or the OS cache dir)", + "default": "", + "required": false + }, + { + "name": "databases", + "type": "stringSlice", + "description": "Participant database patterns; empty selects the connected database", + "default": "[]", + "required": false + }, + { + "name": "duration", + "type": "duration", + "description": "Stop the run after this long, e.g. 30s (0 runs until the command exits or Ctrl+C)", + "default": "0s", + "required": false + }, + { + "name": "follow", + "type": "bool", + "description": "Print every armed capture's records while recording", + "default": "false", + "required": false + }, + { + "name": "from", + "type": "string", + "description": "Historical window start in RFC3339; empty follows from now", + "default": "", + "required": false + }, + { + "name": "poll", + "type": "string", + "description": "Follow poll interval", + "default": "30s", + "required": false + }, + { + "name": "seam", + "type": "string", + "description": "How the capture is armed (supported seams: [connection]) (one of connection)", + "default": "connection", + "required": false + }, + { + "name": "sort", + "type": "string", + "description": "Record order: seq or timestamp", + "default": "seq", + "required": false + }, + { + "name": "to", + "type": "string", + "description": "Historical window end in RFC3339; empty uses server time", + "default": "", + "required": false + }, + { + "name": "tui", + "type": "bool", + "description": "Follow records in an interactive terminal", + "default": "false", + "required": false + }, + { + "name": "ui", + "type": "bool", + "description": "Open this run's traces in a local web viewer; keep serving until Ctrl+C", + "default": "false", + "required": false + } + ] + }, + { + "name": "sql_plancache", + "title": "SQL Server plan cache samples", + "capabilities": { + "follow": true + }, + "params": { + "type": "object", + "properties": { + "connection": { + "type": "string", + "title": "Connection", + "description": "SQL Server connection reference" + }, + "databases": { + "type": "array", + "title": "Databases", + "description": "Database patterns with * wildcards and ! exclusions; empty selects the connected database", + "items": { + "type": "string" + } + }, + "minExecutions": { + "type": "integer", + "title": "Minimum executions", + "format": "int32", + "description": "Minimum interval executions to record" + }, + "poll": { + "type": "string", + "title": "Poll", + "description": "Poll interval such as 30s", + "default": "30s" + }, + "target": { + "properties": { + "agent": { + "type": "string", + "title": "Agent", + "description": "Agent ID to observe" + }, + "allowMutation": { + "type": "boolean", + "title": "Allow mutation", + "description": "Allow probes to change runtime settings" + }, + "pid": { + "type": "integer", + "title": "PID", + "format": "int32", + "description": "Process to observe" + }, + "seam": { + "type": "string", + "title": "Seam", + "description": "How the capture is armed (supported seams: [connection])", + "enum": [ + "connection" + ], + "default": "connection" + }, + "selector": { + "type": "object", + "title": "Selector", + "description": "Agent resource attributes to match", + "additionalProperties": { + "type": "string" + } + }, + "wait": { + "type": "boolean", + "title": "Wait for agents", + "description": "Wait for a matching agent to connect" + } + }, + "type": "object", + "x-telltale-seams": [ + "connection" + ] + }, + "text": { + "type": "array", + "title": "Statement text", + "description": "SQL text patterns with * wildcards and ! exclusions; bare values match substrings", + "items": { + "type": "string" + } + }, + "top": { + "type": "integer", + "title": "Top", + "format": "int32", + "description": "Maximum cost samples per poll ranked by interval CPU; defaults to 200", + "default": 200 + } + }, + "required": [ + "connection" + ] + }, + "columns": [ + { + "name": "sampledAt", + "label": "Sampled", + "type": "datetime", + "kind": "timestamp" + }, + { + "name": "intervalMs", + "label": "Interval", + "type": "duration", + "unit": "ms" + }, + { + "name": "planHandle", + "label": "Plan handle", + "type": "string", + "filter": { + "kind": "exact" + } + }, + { + "name": "queryHash", + "type": "string", + "filter": { + "kind": "exact" + } + }, + { + "name": "queryPlanHash", + "type": "string", + "filter": { + "kind": "exact" + } + }, + { + "name": "database", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "object", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "objType", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "executions", + "type": "number", + "format": "integer" + }, + { + "name": "cpuMs", + "label": "CPU", + "type": "duration", + "unit": "ms" + }, + { + "name": "elapsedMs", + "label": "Elapsed", + "type": "duration", + "unit": "ms" + }, + { + "name": "logicalReads", + "type": "number", + "format": "integer" + }, + { + "name": "physicalReads", + "type": "number", + "format": "integer" + }, + { + "name": "writes", + "type": "number", + "format": "integer" + }, + { + "name": "rows", + "type": "number", + "format": "integer" + }, + { + "name": "avgElapsedMs", + "label": "Average elapsed", + "type": "duration", + "unit": "ms" + }, + { + "name": "avgCpuMs", + "label": "Average CPU", + "type": "duration", + "unit": "ms" + }, + { + "name": "totalExecutions", + "type": "number", + "format": "integer" + }, + { + "name": "totalCpuMs", + "label": "Total CPU", + "type": "duration", + "unit": "ms" + }, + { + "name": "totalElapsedMs", + "label": "Total elapsed", + "type": "duration", + "unit": "ms" + }, + { + "name": "useCounts", + "type": "number", + "format": "integer" + }, + { + "name": "sizeBytes", + "type": "bytes" + }, + { + "name": "creationTime", + "type": "datetime" + }, + { + "name": "recompiled", + "type": "boolean", + "filter": { + "kind": "terms" + } + }, + { + "name": "evicted", + "type": "boolean", + "filter": { + "kind": "terms" + } + }, + { + "name": "sql", + "label": "SQL", + "type": "string", + "filter": { + "kind": "text" + } + } + ], + "exportFormats": [ + { + "name": "ndjson", + "ext": "ndjson", + "mime": "application/x-ndjson" + } + ], + "flags": [ + { + "name": "connection", + "type": "string", + "description": "SQL Server connection reference", + "default": "", + "required": true + }, + { + "name": "data-dir", + "type": "string", + "description": "Directory holding telltale's trace data (default: $XDG_CACHE_HOME/telltale or the OS cache dir)", + "default": "", + "required": false + }, + { + "name": "databases", + "type": "stringSlice", + "description": "Database patterns with * wildcards and ! exclusions; empty selects the connected database", + "default": "[]", + "required": false + }, + { + "name": "duration", + "type": "duration", + "description": "Stop the run after this long, e.g. 30s (0 runs until the command exits or Ctrl+C)", + "default": "0s", + "required": false + }, + { + "name": "every", + "type": "duration", + "description": "Read the checkpointed captures this often, e.g. 10s", + "default": "0s", + "required": false + }, + { + "name": "follow", + "type": "bool", + "description": "Print every armed capture's records while recording", + "default": "false", + "required": false + }, + { + "name": "min-executions", + "type": "int", + "description": "Minimum interval executions to record", + "default": "0", + "required": false + }, + { + "name": "poll", + "type": "string", + "description": "Poll interval such as 30s", + "default": "30s", + "required": false + }, + { + "name": "seam", + "type": "string", + "description": "How the capture is armed (supported seams: [connection]) (one of connection)", + "default": "connection", + "required": false + }, + { + "name": "sort", + "type": "string", + "description": "Record order: seq or timestamp", + "default": "seq", + "required": false + }, + { + "name": "text", + "type": "stringSlice", + "description": "SQL text patterns with * wildcards and ! exclusions; bare values match substrings", + "default": "[]", + "required": false + }, + { + "name": "top", + "type": "int", + "description": "Maximum cost samples per poll ranked by interval CPU; defaults to 200", + "default": "200", + "required": false + }, + { + "name": "tui", + "type": "bool", + "description": "Follow records in an interactive terminal", + "default": "false", + "required": false + }, + { + "name": "ui", + "type": "bool", + "description": "Open this run's traces in a local web viewer; keep serving until Ctrl+C", + "default": "false", + "required": false + } + ] + }, + { + "name": "sql_top", + "title": "SQL running requests", + "capabilities": { + "follow": true + }, + "params": { + "type": "object", + "properties": { + "connection": { + "type": "string", + "title": "Connection", + "description": "Postgres or SQL Server connection reference" + }, + "databases": { + "type": "array", + "title": "Databases", + "description": "Database patterns with * wildcards and ! exclusions; empty keeps every database", + "items": { + "type": "string" + } + }, + "idle": { + "type": "boolean", + "title": "Idle sessions", + "description": "Also record idle sessions; sessions holding an open transaction are always recorded" + }, + "minElapsed": { + "type": "string", + "title": "Minimum elapsed", + "description": "Only record requests running at least this long such as 500ms" + }, + "poll": { + "type": "string", + "title": "Poll", + "description": "Poll interval such as 5s", + "default": "5s" + }, + "target": { + "properties": { + "agent": { + "type": "string", + "title": "Agent", + "description": "Agent ID to observe" + }, + "allowMutation": { + "type": "boolean", + "title": "Allow mutation", + "description": "Allow probes to change runtime settings" + }, + "pid": { + "type": "integer", + "title": "PID", + "format": "int32", + "description": "Process to observe" + }, + "seam": { + "type": "string", + "title": "Seam", + "description": "How the capture is armed (supported seams: [connection])", + "enum": [ + "connection" + ], + "default": "connection" + }, + "selector": { + "type": "object", + "title": "Selector", + "description": "Agent resource attributes to match", + "additionalProperties": { + "type": "string" + } + }, + "wait": { + "type": "boolean", + "title": "Wait for agents", + "description": "Wait for a matching agent to connect" + } + }, + "type": "object", + "x-telltale-seams": [ + "connection" + ] + }, + "text": { + "type": "array", + "title": "Statement text", + "description": "SQL text patterns with * wildcards and ! exclusions; bare values match substrings", + "items": { + "type": "string" + } + }, + "top": { + "type": "integer", + "title": "Top", + "format": "int32", + "description": "Maximum requests per poll ranked by elapsed time; defaults to 50", + "default": 50 + } + }, + "required": [ + "connection" + ] + }, + "columns": [ + { + "name": "sampledAt", + "label": "Sampled", + "type": "datetime", + "kind": "timestamp", + "filter": { + "kind": "time" + } + }, + { + "name": "engine", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "database", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "sessionId", + "label": "Session", + "type": "number", + "format": "integer", + "filter": { + "kind": "exact" + } + }, + { + "name": "requestId", + "label": "Request", + "type": "number", + "format": "integer", + "filter": { + "kind": "exact" + } + }, + { + "name": "blockedBy", + "label": "Blocked by", + "type": "number", + "format": "integer", + "filter": { + "kind": "exact" + } + }, + { + "name": "blocked", + "type": "boolean", + "filter": { + "kind": "boolean" + } + }, + { + "name": "user", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "application", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "clientHost", + "label": "Client host", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "state", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "command", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "waitType", + "label": "Wait", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "waitResource", + "label": "Wait resource", + "type": "string", + "filter": { + "kind": "exact" + } + }, + { + "name": "waitMs", + "label": "Waited", + "type": "duration", + "unit": "ms", + "filter": { + "kind": "duration" + } + }, + { + "name": "startedAt", + "label": "Started", + "type": "datetime", + "filter": { + "kind": "time" + } + }, + { + "name": "transactionStartedAt", + "label": "Transaction started", + "type": "datetime", + "filter": { + "kind": "time" + } + }, + { + "name": "elapsedMs", + "label": "Elapsed", + "type": "duration", + "unit": "ms", + "filter": { + "kind": "duration" + } + }, + { + "name": "transactionMs", + "label": "Transaction", + "type": "duration", + "unit": "ms", + "filter": { + "kind": "duration" + } + }, + { + "name": "cpuMs", + "label": "CPU", + "type": "duration", + "unit": "ms", + "filter": { + "kind": "duration" + } + }, + { + "name": "logicalReads", + "type": "number", + "format": "integer", + "filter": { + "kind": "range" + } + }, + { + "name": "reads", + "type": "number", + "format": "integer", + "filter": { + "kind": "range" + } + }, + { + "name": "writes", + "type": "number", + "format": "integer", + "filter": { + "kind": "range" + } + }, + { + "name": "memoryBytes", + "label": "Memory", + "type": "bytes", + "filter": { + "kind": "range" + } + }, + { + "name": "openTransactions", + "type": "number", + "format": "integer", + "filter": { + "kind": "range" + } + }, + { + "name": "queryId", + "label": "Query id", + "type": "string", + "filter": { + "kind": "exact" + } + }, + { + "name": "sql", + "label": "SQL", + "type": "string", + "filter": { + "kind": "text" + } + } + ], + "exportFormats": [ + { + "name": "ndjson", + "ext": "ndjson", + "mime": "application/x-ndjson" + } + ], + "flags": [ + { + "name": "connection", + "type": "string", + "description": "Postgres or SQL Server connection reference", + "default": "", + "required": true + }, + { + "name": "data-dir", + "type": "string", + "description": "Directory holding telltale's trace data (default: $XDG_CACHE_HOME/telltale or the OS cache dir)", + "default": "", + "required": false + }, + { + "name": "databases", + "type": "stringSlice", + "description": "Database patterns with * wildcards and ! exclusions; empty keeps every database", + "default": "[]", + "required": false + }, + { + "name": "duration", + "type": "duration", + "description": "Stop the run after this long, e.g. 30s (0 runs until the command exits or Ctrl+C)", + "default": "0s", + "required": false + }, + { + "name": "every", + "type": "duration", + "description": "Read the checkpointed captures this often, e.g. 10s", + "default": "0s", + "required": false + }, + { + "name": "follow", + "type": "bool", + "description": "Print every armed capture's records while recording", + "default": "false", + "required": false + }, + { + "name": "idle", + "type": "bool", + "description": "Also record idle sessions; sessions holding an open transaction are always recorded", + "default": "false", + "required": false + }, + { + "name": "min-elapsed", + "type": "string", + "description": "Only record requests running at least this long such as 500ms", + "default": "", + "required": false + }, + { + "name": "poll", + "type": "string", + "description": "Poll interval such as 5s", + "default": "5s", + "required": false + }, + { + "name": "seam", + "type": "string", + "description": "How the capture is armed (supported seams: [connection]) (one of connection)", + "default": "connection", + "required": false + }, + { + "name": "sort", + "type": "string", + "description": "Record order: seq or timestamp", + "default": "seq", + "required": false + }, + { + "name": "text", + "type": "stringSlice", + "description": "SQL text patterns with * wildcards and ! exclusions; bare values match substrings", + "default": "[]", + "required": false + }, + { + "name": "top", + "type": "int", + "description": "Maximum requests per poll ranked by elapsed time; defaults to 50", + "default": "50", + "required": false + }, + { + "name": "tui", + "type": "bool", + "description": "Follow records in an interactive terminal", + "default": "false", + "required": false + }, + { + "name": "ui", + "type": "bool", + "description": "Open this run's traces in a local web viewer; keep serving until Ctrl+C", + "default": "false", + "required": false + } + ] + }, + { + "name": "sql_xevent", + "title": "SQL Server events", + "capabilities": { + "live": true + }, + "params": { + "type": "object", + "properties": { + "apps": { + "type": "array", + "items": { + "type": "string" + } + }, + "connection": { + "type": "string", + "title": "Connection" + }, + "databases": { + "type": "array", + "items": { + "type": "string" + } + }, + "events": { + "type": "array", + "items": { + "type": "string" + } + }, + "hosts": { + "type": "array", + "items": { + "type": "string" + } + }, + "maxEvents": { + "type": "integer", + "format": "int32" + }, + "maxMemoryKb": { + "type": "integer", + "format": "int32" + }, + "minDuration": { + "type": "string" + }, + "poll": { + "type": "string" + }, + "session": { + "enum": [ + "system_health" + ], + "title": "Existing session", + "type": "string" + }, + "sessionName": { + "type": "string" + }, + "tables": { + "type": "array", + "items": { + "type": "string" + } + }, + "target": { + "properties": { + "agent": { + "type": "string", + "title": "Agent", + "description": "Agent ID to observe" + }, + "allowMutation": { + "type": "boolean", + "title": "Allow mutation", + "description": "Allow probes to change runtime settings" + }, + "pid": { + "type": "integer", + "title": "PID", + "format": "int32", + "description": "Process to observe" + }, + "seam": { + "type": "string", + "title": "Seam", + "description": "How the capture is armed (supported seams: [connection])", + "enum": [ + "connection" + ], + "default": "connection" + }, + "selector": { + "type": "object", + "title": "Selector", + "description": "Agent resource attributes to match", + "additionalProperties": { + "type": "string" + } + }, + "wait": { + "type": "boolean", + "title": "Wait for agents", + "description": "Wait for a matching agent to connect" + } + }, + "type": "object", + "x-telltale-seams": [ + "connection" + ] + }, + "types": { + "type": "array", + "items": { + "type": "string" + } + }, + "users": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "connection" + ] + }, + "columns": [ + { + "name": "name", + "label": "Event", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "statementType", + "label": "Type", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "timestamp", + "label": "Time", + "type": "datetime", + "kind": "timestamp" + }, + { + "name": "durationMs", + "label": "Duration", + "type": "duration", + "unit": "ms" + }, + { + "name": "cpuMs", + "label": "CPU", + "type": "duration", + "unit": "ms" + }, + { + "name": "logicalReads", + "label": "Logical reads", + "type": "number", + "format": "integer" + }, + { + "name": "physicalReads", + "label": "Physical reads", + "type": "number", + "format": "integer" + }, + { + "name": "writes", + "type": "number", + "format": "integer" + }, + { + "name": "rowCount", + "label": "Rows", + "type": "number", + "format": "integer" + }, + { + "name": "database", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "clientApp", + "label": "App", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "clientHost", + "label": "Host", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "username", + "label": "User", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "sessionId", + "label": "Session", + "type": "number", + "format": "integer", + "filter": { + "kind": "exact" + } + }, + { + "name": "sql", + "label": "SQL", + "type": "string", + "filter": { + "kind": "text" + } + }, + { + "name": "rawStatement", + "type": "string", + "hidden": true + }, + { + "name": "tables", + "type": "json", + "filter": { + "kind": "terms", + "array": true + } + }, + { + "name": "errorNumber", + "label": "Error", + "type": "number", + "format": "integer" + }, + { + "name": "errorMessage", + "label": "Error message", + "type": "string", + "filter": { + "kind": "text" + } + }, + { + "name": "objectName", + "label": "Object", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "objectType", + "label": "Object type", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "paramsUnavailable", + "label": "Params unavailable", + "type": "boolean" + }, + { + "name": "additionalFields", + "type": "json", + "hidden": true + }, + { + "name": "deadlock", + "type": "json", + "hidden": true + }, + { + "name": "activityId", + "type": "string", + "hidden": true + }, + { + "name": "activitySeq", + "type": "number", + "format": "integer", + "hidden": true + }, + { + "name": "children", + "type": "json", + "hidden": true + } + ], + "exportFormats": [ + { + "name": "ndjson", + "ext": "ndjson", + "mime": "application/x-ndjson" + } + ], + "flags": [ + { + "name": "apps", + "type": "stringSlice", + "description": "", + "default": "[]", + "required": false + }, + { + "name": "connection", + "type": "string", + "description": "Connection", + "default": "", + "required": true + }, + { + "name": "data-dir", + "type": "string", + "description": "Directory holding telltale's trace data (default: $XDG_CACHE_HOME/telltale or the OS cache dir)", + "default": "", + "required": false + }, + { + "name": "databases", + "type": "stringSlice", + "description": "", + "default": "[]", + "required": false + }, + { + "name": "duration", + "type": "duration", + "description": "Stop the run after this long, e.g. 30s (0 runs until the command exits or Ctrl+C)", + "default": "0s", + "required": false + }, + { + "name": "events", + "type": "stringSlice", + "description": "", + "default": "[]", + "required": false + }, + { + "name": "follow", + "type": "bool", + "description": "Print every armed capture's records while recording", + "default": "false", + "required": false + }, + { + "name": "hosts", + "type": "stringSlice", + "description": "", + "default": "[]", + "required": false + }, + { + "name": "max-events", + "type": "int", + "description": "", + "default": "0", + "required": false + }, + { + "name": "max-memory-kb", + "type": "int", + "description": "", + "default": "0", + "required": false + }, + { + "name": "min-duration", + "type": "string", + "description": "", + "default": "", + "required": false + }, + { + "name": "poll", + "type": "string", + "description": "", + "default": "", + "required": false + }, + { + "name": "seam", + "type": "string", + "description": "How the capture is armed (supported seams: [connection]) (one of connection)", + "default": "connection", + "required": false + }, + { + "name": "session", + "type": "string", + "description": "Existing session (one of system_health)", + "default": "", + "required": false + }, + { + "name": "session-name", + "type": "string", + "description": "", + "default": "", + "required": false + }, + { + "name": "sort", + "type": "string", + "description": "Record order: seq or timestamp", + "default": "seq", + "required": false + }, + { + "name": "tables", + "type": "stringSlice", + "description": "", + "default": "[]", + "required": false + }, + { + "name": "tui", + "type": "bool", + "description": "Follow records in an interactive terminal", + "default": "false", + "required": false + }, + { + "name": "types", + "type": "stringSlice", + "description": "", + "default": "[]", + "required": false + }, + { + "name": "ui", + "type": "bool", + "description": "Open this run's traces in a local web viewer; keep serving until Ctrl+C", + "default": "false", + "required": false + }, + { + "name": "users", + "type": "stringSlice", + "description": "", + "default": "[]", + "required": false + } + ] + }, + { + "name": "stacks", + "title": "Stack dumps", + "capabilities": { + "live": true + }, + "params": { + "type": "object", + "properties": { + "target": { + "properties": { + "agent": { + "type": "string", + "title": "Agent", + "description": "Agent ID to observe" + }, + "allowMutation": { + "type": "boolean", + "title": "Allow mutation", + "description": "Allow probes to change runtime settings" + }, + "pid": { + "type": "integer", + "title": "PID", + "format": "int32", + "description": "Process to observe" + }, + "seam": { + "type": "string", + "title": "Seam", + "description": "How the capture is armed (supported seams: [process agent self])", + "enum": [ + "process", + "agent", + "self" + ], + "default": "process" + }, + "selector": { + "type": "object", + "title": "Selector", + "description": "Agent resource attributes to match", + "additionalProperties": { + "type": "string" + } + }, + "wait": { + "type": "boolean", + "title": "Wait for agents", + "description": "Wait for a matching agent to connect" + } + }, + "type": "object", + "x-telltale-seams": [ + "process", + "agent", + "self" + ] + } + } + }, + "columns": [ + { + "name": "truncated", + "label": "Truncated", + "type": "json", + "filter": { + "kind": "terms", + "array": true + } + }, + { + "name": "checkpointId", + "type": "string", + "filter": { + "kind": "exact" + }, + "hidden": true + }, + { + "name": "takenAt", + "label": "Taken", + "type": "datetime", + "kind": "timestamp" + }, + { + "name": "reason", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "transport", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "key", + "type": "string", + "filter": { + "kind": "exact" + }, + "hidden": true + }, + { + "name": "pid", + "label": "PID", + "type": "number", + "format": "integer", + "filter": { + "kind": "terms" + } + }, + { + "name": "format", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "goroutineId", + "label": "Goroutine", + "type": "number", + "format": "integer" + }, + { + "name": "state", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "waitMinutes", + "label": "Waiting (min)", + "type": "number", + "format": "integer" + }, + { + "name": "lockedToThread", + "type": "boolean" + }, + { + "name": "frames", + "type": "json", + "hidden": true + }, + { + "name": "createdBy", + "type": "json", + "hidden": true + }, + { + "name": "raw", + "type": "string", + "filter": { + "kind": "none" + }, + "hidden": true + }, + { + "name": "error", + "type": "string" + } + ], + "exportFormats": [ + { + "name": "ndjson", + "ext": "ndjson", + "mime": "application/x-ndjson" + }, + { + "name": "text", + "ext": "txt", + "mime": "text/plain" + } + ], + "flags": [ + { + "name": "agent", + "type": "string", + "description": "Agent ID to observe", + "default": "", + "required": false + }, + { + "name": "allow-mutation", + "type": "bool", + "description": "Allow probes to change runtime settings", + "default": "false", + "required": false + }, + { + "name": "data-dir", + "type": "string", + "description": "Directory holding telltale's trace data (default: $XDG_CACHE_HOME/telltale or the OS cache dir)", + "default": "", + "required": false + }, + { + "name": "duration", + "type": "duration", + "description": "Stop the run after this long, e.g. 30s (0 runs until the command exits or Ctrl+C)", + "default": "0s", + "required": false + }, + { + "name": "every", + "type": "duration", + "description": "Read the checkpointed captures this often, e.g. 10s", + "default": "0s", + "required": false + }, + { + "name": "follow", + "type": "bool", + "description": "Print every armed capture's records while recording", + "default": "false", + "required": false + }, + { + "name": "pid", + "type": "int", + "description": "Process to observe", + "default": "0", + "required": false + }, + { + "name": "seam", + "type": "string", + "description": "How the capture is armed (supported seams: [process agent self]) (one of process, agent, self)", + "default": "process", + "required": false + }, + { + "name": "selector", + "type": "string", + "description": "Agent resource attributes to match", + "default": "", + "required": false + }, + { + "name": "sort", + "type": "string", + "description": "Record order: seq or timestamp", + "default": "seq", + "required": false + }, + { + "name": "tui", + "type": "bool", + "description": "Follow records in an interactive terminal", + "default": "false", + "required": false + }, + { + "name": "ui", + "type": "bool", + "description": "Open this run's traces in a local web viewer; keep serving until Ctrl+C", + "default": "false", + "required": false + }, + { + "name": "wait", + "type": "bool", + "description": "Wait for a matching agent to connect", + "default": "false", + "required": false + } + ] + }, + { + "name": "task", + "title": "Tasks", + "capabilities": { + "live": true + }, + "params": { + "type": "object", + "properties": { + "group": { + "type": "string", + "description": "Parent group name glob" + }, + "name": { + "type": "string", + "description": "Task name glob" + }, + "progressIntervalMs": { + "type": "integer", + "format": "int32", + "description": "Minimum milliseconds between progress records of a task; 0 takes the agent's default" + }, + "target": { + "properties": { + "agent": { + "type": "string", + "title": "Agent", + "description": "Agent ID to observe" + }, + "allowMutation": { + "type": "boolean", + "title": "Allow mutation", + "description": "Allow probes to change runtime settings" + }, + "pid": { + "type": "integer", + "title": "PID", + "format": "int32", + "description": "Process to observe" + }, + "seam": { + "type": "string", + "title": "Seam", + "description": "How the capture is armed (supported seams: [agent])", + "enum": [ + "agent" + ], + "default": "agent" + }, + "selector": { + "type": "object", + "title": "Selector", + "description": "Agent resource attributes to match", + "additionalProperties": { + "type": "string" + } + }, + "wait": { + "type": "boolean", + "title": "Wait for agents", + "description": "Wait for a matching agent to connect" + } + }, + "type": "object", + "x-telltale-seams": [ + "agent" + ] + } + } + }, + "columns": [ + { + "name": "truncated", + "label": "Truncated", + "type": "json", + "filter": { + "kind": "terms", + "array": true + } + }, + { + "name": "phase", + "type": "string" + }, + { + "name": "id", + "type": "string" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "type", + "type": "string" + }, + { + "name": "group", + "type": "string" + }, + { + "name": "status", + "type": "string" + }, + { + "name": "error", + "type": "string" + }, + { + "name": "retry", + "type": "json" + }, + { + "name": "message", + "type": "string" + }, + { + "name": "logs", + "type": "json" + }, + { + "name": "logsClearedThrough", + "type": "number", + "format": "integer" + }, + { + "name": "total", + "type": "number", + "format": "integer" + }, + { + "name": "completed", + "type": "number", + "format": "integer" + }, + { + "name": "failed", + "type": "number", + "format": "integer" + }, + { + "name": "running", + "type": "number", + "format": "integer" + }, + { + "name": "canceled", + "type": "number", + "format": "integer" + }, + { + "name": "work", + "type": "json" + }, + { + "name": "description", + "type": "string" + }, + { + "name": "progress", + "type": "number", + "format": "integer" + }, + { + "name": "maxValue", + "type": "number", + "format": "integer" + }, + { + "name": "progressMetrics", + "type": "json" + }, + { + "name": "groupId", + "type": "string" + }, + { + "name": "parentGroupId", + "type": "string" + }, + { + "name": "dependencyIds", + "type": "json", + "filter": { + "kind": "terms", + "array": true + } + }, + { + "name": "kind", + "type": "string" + }, + { + "name": "labels", + "type": "json" + }, + { + "name": "owner", + "type": "string" + }, + { + "name": "startedAt", + "type": "string" + }, + { + "name": "finishedAt", + "type": "string" + }, + { + "name": "href", + "type": "string" + }, + { + "name": "session", + "type": "json" + }, + { + "name": "controls", + "type": "json", + "filter": { + "kind": "terms", + "array": true + } + }, + { + "name": "stdout", + "type": "string" + }, + { + "name": "stderr", + "type": "string" + }, + { + "name": "stdoutOffset", + "type": "number", + "format": "integer" + }, + { + "name": "stderrOffset", + "type": "number", + "format": "integer" + }, + { + "name": "stdoutTruncated", + "type": "boolean" + }, + { + "name": "stderrTruncated", + "type": "boolean" + }, + { + "name": "details", + "type": "json" + }, + { + "name": "prompts", + "type": "json" + }, + { + "name": "resource", + "type": "json" + } + ], + "exportFormats": [ + { + "name": "ndjson", + "ext": "ndjson", + "mime": "application/x-ndjson" + } + ], + "flags": [ + { + "name": "agent", + "type": "string", + "description": "Agent ID to observe", + "default": "", + "required": false + }, + { + "name": "allow-mutation", + "type": "bool", + "description": "Allow probes to change runtime settings", + "default": "false", + "required": false + }, + { + "name": "data-dir", + "type": "string", + "description": "Directory holding telltale's trace data (default: $XDG_CACHE_HOME/telltale or the OS cache dir)", + "default": "", + "required": false + }, + { + "name": "duration", + "type": "duration", + "description": "Stop the run after this long, e.g. 30s (0 runs until the command exits or Ctrl+C)", + "default": "0s", + "required": false + }, + { + "name": "follow", + "type": "bool", + "description": "Print every armed capture's records while recording", + "default": "false", + "required": false + }, + { + "name": "group", + "type": "string", + "description": "Parent group name glob", + "default": "", + "required": false + }, + { + "name": "name", + "type": "string", + "description": "Task name glob", + "default": "", + "required": false + }, + { + "name": "pid", + "type": "int", + "description": "Process to observe", + "default": "0", + "required": false + }, + { + "name": "progress-interval-ms", + "type": "int", + "description": "Minimum milliseconds between progress records of a task; 0 takes the agent's default", + "default": "0", + "required": false + }, + { + "name": "seam", + "type": "string", + "description": "How the capture is armed (supported seams: [agent]) (one of agent)", + "default": "agent", + "required": false + }, + { + "name": "selector", + "type": "string", + "description": "Agent resource attributes to match", + "default": "", + "required": false + }, + { + "name": "sort", + "type": "string", + "description": "Record order: seq or timestamp", + "default": "seq", + "required": false + }, + { + "name": "tui", + "type": "bool", + "description": "Follow records in an interactive terminal", + "default": "false", + "required": false + }, + { + "name": "ui", + "type": "bool", + "description": "Open this run's traces in a local web viewer; keep serving until Ctrl+C", + "default": "false", + "required": false + }, + { + "name": "wait", + "type": "bool", + "description": "Wait for a matching agent to connect", + "default": "false", + "required": false + } + ] + }, + { + "name": "terminal", + "title": "Terminal", + "capabilities": { + "live": true + }, + "params": { + "type": "object", + "properties": { + "command": { + "type": "array", + "title": "Command", + "description": "Command line named in the cast's header", + "items": { + "type": "string" + } + }, + "height": { + "type": "integer", + "title": "Height", + "format": "int32", + "description": "Terminal height in cells", + "default": 40 + }, + "maxBytes": { + "type": "string", + "title": "Max bytes", + "description": "Most output recorded as events such as 4MiB", + "default": "4MiB" + }, + "snapshotInterval": { + "type": "string", + "title": "Snapshot interval", + "description": "How often the settled screen is recorded such as 100ms", + "default": "100ms" + }, + "target": { + "properties": { + "agent": { + "type": "string", + "title": "Agent", + "description": "Agent ID to observe" + }, + "allowMutation": { + "type": "boolean", + "title": "Allow mutation", + "description": "Allow probes to change runtime settings" + }, + "pid": { + "type": "integer", + "title": "PID", + "format": "int32", + "description": "Process to observe" + }, + "seam": { + "type": "string", + "title": "Seam", + "description": "How the capture is armed (supported seams: [self])", + "enum": [ + "self" + ], + "default": "self" + }, + "selector": { + "type": "object", + "title": "Selector", + "description": "Agent resource attributes to match", + "additionalProperties": { + "type": "string" + } + }, + "wait": { + "type": "boolean", + "title": "Wait for agents", + "description": "Wait for a matching agent to connect" + } + }, + "type": "object", + "x-telltale-seams": [ + "self" + ] + }, + "width": { + "type": "integer", + "title": "Width", + "format": "int32", + "description": "Terminal width in cells", + "default": 120 + } + } + }, + "columns": [ + { + "name": "at", + "label": "At", + "type": "datetime", + "kind": "timestamp" + }, + { + "name": "offset", + "label": "Offset", + "type": "number" + }, + { + "name": "kind", + "type": "string", + "filter": { + "kind": "terms" + } + }, + { + "name": "data", + "type": "string", + "filter": { + "kind": "none" + }, + "hidden": true + }, + { + "name": "width", + "type": "number", + "format": "integer" + }, + { + "name": "height", + "type": "number", + "format": "integer" + }, + { + "name": "exitCode", + "type": "number", + "format": "integer" + }, + { + "name": "truncated", + "type": "boolean" + }, + { + "name": "command", + "type": "string" + } + ], + "exportFormats": [ + { + "name": "ndjson", + "ext": "ndjson", + "mime": "application/x-ndjson" + }, + { + "name": "cast", + "ext": "cast", + "mime": "application/x-asciicast" + } + ], + "flags": [ + { + "name": "allow-mutation", + "type": "bool", + "description": "Allow probes to change runtime settings", + "default": "false", + "required": false + }, + { + "name": "command", + "type": "stringSlice", + "description": "Command line named in the cast's header", + "default": "[]", + "required": false + }, + { + "name": "data-dir", + "type": "string", + "description": "Directory holding telltale's trace data (default: $XDG_CACHE_HOME/telltale or the OS cache dir)", + "default": "", + "required": false + }, + { + "name": "duration", + "type": "duration", + "description": "Stop the run after this long, e.g. 30s (0 runs until the command exits or Ctrl+C)", + "default": "0s", + "required": false + }, + { + "name": "follow", + "type": "bool", + "description": "Print every armed capture's records while recording", + "default": "false", + "required": false + }, + { + "name": "height", + "type": "int", + "description": "Terminal height in cells", + "default": "40", + "required": false + }, + { + "name": "max-bytes", + "type": "string", + "description": "Most output recorded as events such as 4MiB", + "default": "4MiB", + "required": false + }, + { + "name": "pid", + "type": "int", + "description": "Process to observe", + "default": "0", + "required": false + }, + { + "name": "seam", + "type": "string", + "description": "How the capture is armed (supported seams: [self]) (one of self)", + "default": "self", + "required": false + }, + { + "name": "snapshot-interval", + "type": "string", + "description": "How often the settled screen is recorded such as 100ms", + "default": "100ms", + "required": false + }, + { + "name": "sort", + "type": "string", + "description": "Record order: seq or timestamp", + "default": "seq", + "required": false + }, + { + "name": "tui", + "type": "bool", + "description": "Follow records in an interactive terminal", + "default": "false", + "required": false + }, + { + "name": "ui", + "type": "bool", + "description": "Open this run's traces in a local web viewer; keep serving until Ctrl+C", + "default": "false", + "required": false + }, + { + "name": "width", + "type": "int", + "description": "Terminal width in cells", + "default": "120", + "required": false + } + ] + } +] diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..1cb9332 --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,50 @@ +# Quick start + +Telltale combines a CLI and an HTTP server with a browser workspace. This guide builds it from a source checkout and records a command. + +## Build and start the server + +Install Go 1.27.1, Node 24, pnpm 11.22.0, a JDK (17 or 21 recommended), [Task](https://taskfile.dev/installation/), Gavel, and actionlint. Then run from the repository root: + +```sh +make setup +make build +.bin/telltale serve +``` + +Open `http://localhost:8080`. The server also exposes `/health` and `/api/openapi.json`. Use `--host` and `--port` to change the listener. + +## Record a command + +In another terminal, run: + +```sh +.bin/telltale trace run -- curl -sS https://example.com +.bin/telltale trace list +``` + +The default capture uses the HTTP proxy seam, and Telltale also records the process tree. Open the web workspace to inspect the run. Add `--follow --format ndjson` to print captured records as they arrive, or run `.bin/telltale trace run --help` for the available capture kinds and parameters. + +Run `.bin/telltale explain` to list trace kinds, or `.bin/telltale explain http` to inspect a kind's parameters and record schema. Add `--json` for structured output. + +Runs are stored under `traces/` in the data directory. Set `--data-dir` to choose it; by default Telltale uses `$XDG_CACHE_HOME/telltale` or the operating system cache directory. + +## Next steps + +- Learn how to [navigate the web workspace](/web-ui). +- [Receive OpenTelemetry traces](/otlp-receiver) from an instrumented application. +- [Preview resource monitoring rules](/rules) before enabling hooks. +- Consult the [README](https://github.com/flanksource/telltale#readme) for trace definitions, connections, process grouping, and development commands. + +## Work on this documentation + +From the repository root, install the documentation site's locked dependencies and start its development server: + +```sh +npm --prefix docs ci +npm --prefix docs run dev +``` + +Run `npm --prefix docs run build` to generate the static site in `docs/doc_build`. Preview the build with `npm --prefix docs run preview`. + +The trace-kind argument and record schemas are generated from Telltale's registered catalog. After changing a kind, run `npm --prefix docs run generate:catalog` and commit the updated snapshot. The documentation build checks that the snapshot is current. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..74920da --- /dev/null +++ b/docs/index.md @@ -0,0 +1,15 @@ +# Telltale + +Telltale records traces and process activity for commands, running processes, and hosts. Explore captures from the terminal or the web workspace, receive OpenTelemetry traces, and monitor resource usage across process groups. + +[Get started](/getting-started) + +## Explore the guides + +- [Web workspace](/web-ui): start captures, inspect records, manage connections, and explore traces. +- [OpenTelemetry receiver](/otlp-receiver): receive traces over OTLP and inspect them in Telltale. +- [Resource monitoring rules](/rules): preview CEL-based rules and configure hooks for process groups. +- [SQL Server and JVM captures](/sqlserver-jvm): collect SQL Server events and JVM profiles. +- [Trace kinds](/traces/): choose a capture and see its target, options, and examples. + +For the full command and configuration reference, see the [repository README](https://github.com/flanksource/telltale#readme). The CLI also provides `telltale --help` and help for each subcommand. diff --git a/docs/jms.md b/docs/jms.md new file mode 100644 index 0000000..01ea049 --- /dev/null +++ b/docs/jms.md @@ -0,0 +1,7 @@ +# JMS messages and receive handlers + +`telltale run jms --agent api` records actual sent and received messages. Empty receive polls and timeouts produce no message events. Failed sends with an observed Message remain error events. Bodies are disabled by default; `--body --max-body-bytes 65536` enables bounded text capture. + +`telltale run jms_top --agent api --poll 1s --top 50` samples destination-to-handler mappings. Discovered handlers remain visible while idle. The active count and elapsed time describe handler processing, not a consumer waiting for a message. `--destinations orders` selects a destination. Handler mappings are discovered through observed execution; handlers that have never been observed cannot be enumerated from the JVM heap. + +Use `--tui` for the fullscreen Top table. Without `--tui`, captures print human-readable records to stderr automatically. Structured results and explicit NDJSON following remain on stdout. `--no-color` disables styling. The web viewer and TUI show Top snapshots in their Top views, separate from All events. diff --git a/docs/jvm-diagnostics-port.md b/docs/jvm-diagnostics-port.md new file mode 100644 index 0000000..d72d623 --- /dev/null +++ b/docs/jvm-diagnostics-port.md @@ -0,0 +1,291 @@ +# JVM diagnostics + +Telltale includes the generic diagnostic services ported from `oipa-cli` revision `c9e0ce0099d335543d9ef28bde5b9b111fe3842c`, under `com.flanksource.telltale.diag`. JMX, threads, metrics, logging, class browsing/decompilation, method instrumentation, compilation, redefinition and restoration use those implementations. Application settings, Spring/OIPA operations and rule-location interpretation belong in extensions. + +## Build and install + +`make build` creates the embedded agent at `pkg/agentinstall/java/assets/telltale-agent.jar` and three artifacts in `sdk/java/build/libs`: `telltale-diag.war`, `telltale-extension-api.jar` and `telltale-example-extension.jar`. Runtime classes target Java 8. CI tests Java 8, 11, 17, 21 and 25. The WAR uses `javax.servlet` and requires Servlet 3.1+ (for example, Tomcat 9). + +Substitute a disposable application's PID and absolute target-host directories: + +```bash +telltale agents install java --pid 1234 --directory /home/example/telltale/agent --service orders --allow-mutation --extensions-dir /home/example/telltale/extensions +telltale jvm catalog --agent 1234 +telltale jvm jmx search --agent 1234 --pattern 'java.lang:*' +telltale jvm loaders --agent 1234 +``` + +The installer also supports Kubernetes and SSH; `--hub-url` and `--token-file` connect the remote agent to Telltale. Local agents use PID identities; hub IDs come from `/api/agents`. Read-only inspection works by default. Writes, invocations, scripts, logging/bytecode changes and extension changes require both installation-time and per-call `--allow-mutation`. Operations affect the whole JVM, including other applications sharing a container. + +## Operations and captures + +`GET /api/agents/{agent}/catalog` returns live operation names, schemas and mutation flags. `POST /api/agents/{agent}/call` accepts `operation`, `params` and `allowMutation`. Agent identity comes from the route. Existing API authentication and cross-origin protection apply. The Agents page links to `/jvm/{agent}` for schema forms, results, scripts, extension uploads and compile previews. Class, loader, method and search selections persist in the URL; executable source does not. + +Dedicated pages at `/jvm/{agent}/instrument`, `/jvm/{agent}/traces` and `/jvm/{agent}/notifications` use Clicky UI workspace panes, tables and live event feeds. Instrument accepts up to eight class/method targets, concrete loader identities, watch/trace options, event limits and a capture duration. Method selectors accept a name, wildcard or a name with its JVM descriptor, such as `compute(J)D`. Class and MBean links seed the corresponding form. Capture IDs and catalog filters persist in the URL; reloading the page reconnects to the selected server-owned recording. Stop ends the recording and restores instrumented bytecode; leaving a page does not stop it. + +Traces keeps an agent-specific catalog beside runtime probe inspection. Notifications keeps the recording form, durable history, event stream and subscriber queue/drop counts together. The Records tab reuses the existing query browser, including the method call tree for JVM traces. Run responses and saved manifests carry each capture's target. Selector-based captures and older recordings without a target remain available through All runs instead of being attributed to a JVM by guesswork. + +In Records / call tree, run a query and choose a call to load its parent invocation and nested calls independently of the query's filters and pagination. The viewer reads a fixed committed sequence for the selected thread and probe, with a 10,000-event limit per thread snapshot. Arguments, return values, exceptions, thread identity and measured counters remain available in the shared Clicky UI execution tree and object browser. Missing entries or closing events are marked incomplete; truncated and opaque values retain their explanations. Refresh invocation takes a new snapshot of a running capture. + +Groovy and replacement Java source use Clicky UI's Monaco editor. Compile preview shows compiler messages with line/column markers; editing the source invalidates approval to apply the previous compilation. Direct patches offer explicit 2-, 5- or 15-minute restore leases. Runtime-disabled mutations remain unavailable while drafts can still be edited. Instrument offers optional overload lookup alongside manual JVM selectors and retains recent methods; OGNL retains recent successful expressions in this browser, scoped to the agent. + +Operation families include `jvm.jmx.*`, `jvm.metrics`, `jvm.settings`, `jvm.resources`, `jvm.flags`, `jvm.threads*`, `jvm.logging.*`, `jvm.classes*`, `jvm.loaders`, `jvm.probes.*`, `jvm.ognl.eval`, `jvm.groovy.eval`, `jvm.heap.histogram` and `jvm.extensions.*`. `jvm.heap.histogram` (`telltale jvm heap histogram`) returns the heap's classes by retained bytes from `GC.class_histogram`: `top` rows (default 100), `classPattern` as a glob over class names, and `live` to count only reachable objects, which forces a full GC first. `jvm snapshot` collects several of these into one redacted support bundle; see [JVM support snapshot](jvm-snapshot.md). + +Every built-in operation is a typed command, grouped by task: + +| Command | Serves | +| --- | --- | +| `jvm decompile`, `jvm decompile bytecode` | `jvm.classes.source`, `jvm.classes.bytecode` | +| `jvm patch compile` / `apply` / `lease` / `restore` / `list` | `jvm.classes.compile` / `patch` / `lease` / `restore` / `patches` | +| `jvm watch` | a `jvm_trace` capture of arguments, return values and exceptions, without nested calls | +| `jvm trace` | a `jvm_trace` capture with nested calls and timings | +| `jvm trace instrument` / `probes` / `read` / `freeze` | `jvm.classes.instrument`, `jvm.probes.list` / `read` / `freeze` | +| `jvm profiler` | a `jvm_jfr` capture on the agent seam | +| `jvm classes`, `jvm loaders`, `jvm jmx …`, `jvm threads …`, `jvm heap histogram`, `jvm metrics`, `jvm settings`, `jvm resources`, `jvm flags`, `jvm logging …`, `jvm ognl eval`, `jvm groovy eval`, `jvm extensions …`, `jvm notifications list`, `jvm status`, `jvm catalog` | the operation of the same name | + +```bash +telltale jvm decompile --agent 1234 --class example.Worker +telltale jvm patch compile --agent 1234 --class example.Worker --source-file Worker.java +telltale jvm patch apply --agent 1234 --allow-mutation --class example.Worker --source-file Worker.java --lease-seconds 300 +telltale jvm patch list --agent 1234 +telltale jvm patch restore --agent 1234 --allow-mutation --class example.Worker +telltale jvm watch --agent 1234 --allow-mutation --duration 10s --targets '[{"class":"example.Worker","method":"compute"}]' +telltale jvm trace --agent 1234 --allow-mutation --duration 10s --targets '[{"class":"example.Worker","method":"compute"}]' +telltale jvm profiler --agent 1234 --engine async-profiler --duration 30s +telltale jvm heap histogram --agent 1234 --top 20 --class-pattern 'com.example.*' +``` + +Every operation's result has a typed model in `pkg/tracing/jvm` and its subpackages (`jvmruntime`, `classes`, `jmx`, `agentops`). In a terminal, a status or an acknowledgement prints on one line, such as `mutations allowed · instrumentation on · 0 patches` or `✓ logger global → FINE`. Lists print as a table under a one-line heading. Decompiled source is syntax-highlighted, and thread dumps and stacks print one frame per line. `--format json` (or yaml, csv and the others) prints the same model as structured data. An operation without a model, such as an extension's through `jvm call`, prints its JSON indented. A result that reports a failure, such as a compile with errors, a patch that was not applied or an instrument that armed no method, prints its diagnostics and the command exits non-zero. A test fails when a built-in operation in the catalog has no model. + +`watch`, `trace` and `profiler` are the `jvm_trace` and `jvm_jfr` kinds with preset defaults. They record under the base kind, so the existing views read them. They take every flag of `telltale run jvm_trace|jvm_jfr`, which remain the generic forms, and their flags override the presets (`jvm trace --calls=false`). OpenAPI documents their typed inputs. + +With `--follow`, the method events are folded into calls and printed one line per call when it exits: the time, the thread, the call with its arguments, the return value or exception, and the duration. `jvm trace` prints the calls nested inside each call as a tree under its line. When the run ends, a line per stream totals the calls, how many threw, the slowest, and any events the agent dropped. The run summary follows, with one line per capture and the `telltale records` command that reads it back. `telltale trace replay ` prints a recorded run the same way, and `--format ndjson` keeps the raw events. + +```text +02:16:16.761 main Idle.work(103328) → 464967 1.4ms +02:16:16.761 main ├─ Idle.step 1.2ms +02:16:16.762 main └─ Idle.step 138µs +jvm_trace · 5 calls · slowest Idle.work 1.4ms +Run host20261116T001616Z · 1.1s + jvm_trace · agent 88158 · 35 records · telltale records jvm_trace --run host20261116T001616Z +``` + +Over HTTP, start the same capture with `POST /api/traces`, which records into the server's own data dir. Each capture goes under `traces` by kind, with `"enabled": true` (or an array of such objects to arm a kind more than once, each element its own capture); on the agent seam no `pid` or command is needed. For example: `{"duration":"10s","traces":{"jvm_trace":{"enabled":true,"target":{"seam":"agent","agent":"1234","allowMutation":true},"targets":[{"class":"example.Worker","method":"compute"}],"calls":true}}}`. + +Each parameter is a flag (`objectName` is `--object-name`). Open-ended values such as `jvm.jmx.write`'s `--value` or `jvm.jmx.invoke`'s `--args` take JSON. An empty `--level` for `jvm.logging.set` sends `null`, which inherits the parent's level. Source parameters also accept `--source-file`, and `jvm.extensions.install`/`replace` take `--jar-file`. These file flags read the local filesystem, so a request that arrives over HTTP must send the content inline. Mutating operations add `--allow-mutation`. The same commands appear in `/api/openapi.json` under `/api/v1/jvm/…` with typed inputs. `jvm call --operation --params '{…}'` remains for operations only the live catalog knows, such as extension operations. + +The commands come from the definitions. Each operation declares its command in `DiagnosticSchemas`, and each kind preset is a `Commands` entry on the kind's `traces.Registration`. Go reads operations from `internal/jvm/catalog.json`, a snapshot of the Java registry's `Diagnostics.catalog()`. After changing an operation, its command or its schema in `Diagnostics`/`DiagnosticSchemas`, regenerate the snapshot with `task generate`, or with `./sdk/java/gradlew -p sdk/java jvmCatalog`. `CatalogSnapshotTest` fails while the snapshot is stale. `jvm.classes.source` returns source, descriptors and line mappings. `jvm.classes.bytecode` returns base64 class bytes. `jvm.classes.compile` previews compilation without applying bytecode. + +Instrumented classes are armed by retransformation through a retransform-capable transformer, so a later retransformation of the class by another agent (an APM agent, Arthas) or by a bytecode read re-applies every probe still armed on it instead of dropping them. + +Captures share the existing run API, definitions, checkpoints and record stores: + +| Kind | Purpose | Parameters | +| --- | --- | --- | +| `jvm_jfr` | JFR or async-profiler | Existing engine/event/settings options | +| `jvm_trace` | Watch values or trace nested calls | One to eight `targets` with `class`, `method`, optional `loader`; `args`, `return`, `exception`, `calls`, `platform`, `events`, `expand`, `arg-chars`, `gc`, `alloc` | +| `jvm_redefine` | Replacement source for a capture | `class`, optional `method`/`loader`, `source`, optional `classpath` | +| `jvm_notifications` | JMX notifications and decoded GC | `mbean` name/pattern, optional `types` prefixes and `capacity` (default 500, maximum 50,000) | +| `jvm_ognl` | Periodic expression samples | `expression`, optional `loader`, `intervalMs` (default 1,000, minimum 100) | + +`jvm_redefine` requires a verifiable local `CodeSource` class artifact matching the selected runtime logical method after Telltale hooks are excluded, or an exact previously coordinated Telltale source body. Unverifiable origins and uncoordinated foreign bytecode changes to that method are rejected before mutation, with the class and descriptor in the error; opaque advice stays intact. Stopping a source capture preserves generic method traces, and stopping a trace preserves source edits. Manual class restoration and lease expiry still clear all coordinated edits and method probes for that class. + +```bash +telltale jvm trace --agent 1234 --allow-mutation --duration 10s --targets '[{"class":"example.Worker","method":"compute"}]' +telltale run jvm_notifications --agent 1234 --duration 10s --mbean 'java.lang:type=GarbageCollector,*' +telltale run jvm_ognl --agent 1234 --allow-mutation --duration 5s --expression '@java.lang.Runtime@getRuntime().freeMemory()' +``` + +Method captures stream events and restore bytecode on stop, deadline, disconnect or event-budget completion. They renew a 120-second JVM-owned lease every 30 seconds; the JVM reaper restores abandoned leased patches. Direct patches expose lease controls through `jvm.classes.lease` and restore through `jvm.classes.restore`; inspect them with `jvm.classes.patches`. Ambiguous write targets require a real loader hash; grouped loaders are browsing filters. Dropped events remain visible in stored records. + +## One-shot Groovy + +```bash +telltale jvm groovy eval --agent 1234 --allow-mutation --source 'println args.name; return 3 + 4' --args '{"name":"example"}' +telltale jvm groovy eval --agent 1234 --allow-mutation --source-file sdk/java/examples/inspect.groovy --args '{"name":"orders"}' --timeout-seconds 30 +``` + +`--loader` selects an application loader hash from `jvm.loaders`; omitting it selects the system loader. Bindings are `args` (JSON object), `application` (selected loader), `jmx`, `extensions`, `out` and `err`. Every execution isolates and closes its Groovy loader. Scalars return as `value`; other results use bounded object graphs. Standard output/error each have a 1 MiB limit. Errors propagate to the caller. Deadlines and cancellation are cooperative and cannot forcibly stop blocking application methods. Scripts and extensions are trusted operator code, not a sandbox. + +## Hot extensions + +```bash +telltale jvm extensions install --jar-file sdk/java/build/libs/telltale-example-extension.jar --agent 1234 --allow-mutation +telltale jvm call --agent 1234 --operation extension.example.greet --params '{"name":"operator"}' +telltale jvm extensions replace --jar-file sdk/java/build/libs/telltale-example-extension.jar --agent 1234 --allow-mutation +telltale jvm extensions list --agent 1234 +telltale jvm extensions unload --id example --agent 1234 --allow-mutation +``` + +Each JAR has exactly one `DiagnosticExtension` provider listed in `META-INF/services/com.flanksource.telltale.DiagnosticExtension`. Compile against `telltale-extension-api.jar`; runnable source and a service descriptor live in `sdk/java/examples`. API version 1 defines ID, version, startup and cleanup. `OperationSpec` supplies operation schemas, title, description, mutation flag and handler. Names must start with `extension..`. Handlers return JSON-compatible JDK values. Child-first loaders isolate bundled dependencies while retaining one shared SPI identity. + +`ExtensionContext.bindings` supplies application-loader-specific Groovy values under `extensions.`. `mbeans()` exposes the MBean server. Register owned listeners/executors/resources using `own(AutoCloseable)`; unload closes them in reverse order. Failed cleanup remains visible as a failed extension and disables its operations. Cleanup must support retry after partial failure. + +Installation verifies SHA-256 and stores immutable digest-named JARs under the absolute `extensionsDir`. An atomic `active.properties` manifest restores activation after JVM restart; cached JARs remain after unload. Uploads are limited to 16 MiB. Replace/unload refuse during active operation calls or Groovy executions. Catalog refresh exposes changes without restarting the agent, WAR or Telltale server. + +## Standalone WAR + +A `Deployer` (below) hot-deploys the WAR with its configuration embedded as `WEB-INF/telltale-diag.properties`. To deploy it by hand instead, configure an absolute properties file using the servlet/context `configFile` parameter or `-Dtelltale.diag.config=/etc/telltale/diagnostics.properties`; an embedded and an external configuration together are an error. Start from `sdk/java/examples/diagnostics.properties`: + +- `issuer` and `audience`: the `iss` and `aud` every token must carry. The audience is the target id. +- `jwks` (inline JWKS JSON) or `jwksFile` (an absolute path), and/or `jwksUrl`: the issuer's public keys. On an unknown `kid` the WAR re-reads `jwksUrl`, at most once every 30 seconds, so a key rotation needs no redeploy. +- `runtimeDir` stores the extracted runtime, `extensionsDir` enables extensions and `allowMutation=true` allows writes. +- `configDigest` is echoed by `/health` so a deployer can tell whether the WAR it finds has its provision. + +There is no shared token: the WAR holds only public keys, and `tokenFile` fails initialization. Unknown keys fail initialization too. + +Every execution request carries `X-Telltale-Authorization: Bearer `. The Kubernetes apiserver deletes `Authorization` from every request it proxies (`pods/proxy`, `services/proxy`), so the token never relies on it, and it is never a query parameter. A direct connection may send `Authorization` instead; both present with different values is a 400. The token is an RS256 JWT with `kid`, `iss`, `aud`, `sub`, `exp` (60 seconds leeway) and a space-separated `scope`. RS256 is used because the agent runs on Java 8, which has no Ed25519. The WAR verifies each call and fails closed: a missing, malformed, wrongly signed, wrong-audience or expired token is 401, and a token without the route's scope is 403. Scopes are one table in the agent. Every catalog operation and `/hello` probe names its scope, and minters ask for exactly that: + +| Scope | Grants | +| --- | --- | +| `jvm:read` | `/hello`, `/catalog` and every read-only operation | +| `jvm:trace` | `jvm_trace`, `jvm_jfr` and `jvm_notifications` captures, `jvm.probes.freeze` | +| `jvm:instrument` | `jvm_redefine`, `jvm.classes.patch/instrument/restore/lease`, extension install/replace/unload | +| `jvm:eval` | `jvm.groovy.eval`, `jvm.ognl.eval`, `jvm_ognl` captures | +| `jvm:write` | `jvm.jmx.write/invoke`, `jvm.logging.set`, mutating extension operations | + +Routes: + +- `GET /health` needs no token. It answers `{status, apiVersion, configDigest}`. +- `GET /hello` gives the agent API, probes and catalog. `GET /catalog` gives the catalog alone. +- `POST /call` takes `{operation, params, allowMutation}`. +- `POST /captures` takes `{kind, params, allowMutation}` and answers `{id, streaming, contentType, fileName}`. +- `POST /captures/{id}` reads the capture so far and sets `X-Telltale-Complete`. `POST /captures/{id}/renew` extends its lease (204). `DELETE /captures/{id}` finishes it and returns the last reading. + +Capture IDs belong to the WAR deployment; unloading it closes its captures, while agent captures stay independent. + +The agent API version is `1`. A WAR that finds an older runtime already loaded in the JVM stays deployed, but answers `/health` with 503 `outdated` and refuses every other route. Restart the JVM to replace the runtime. + +The first adapter creates the JVM-wide `com.flanksource.telltale:type=Diagnostics` MBean in a loader outside the servlet application. Other adapters share it through JDK types. Mutation/extension-directory configuration must agree. Original bytes, probe buffers, leases and extensions survive servlet redeployment. Restart the JVM to replace this shared runtime version. Instrumentation attaches lazily when required: use a full JDK in `java.home`/`JAVA_HOME`, or `-Dtelltale.diag.jdk-home` to select a matching JDK. Attach restrictions are reported as errors. + +## Deployers + +A `Deployer` (`crds/telltale.flanksource.com_deployers.yaml`, Go package `pkg/deploy`) puts the agent into the JVMs a selector picks and owns their auth. Executions never install anything. They reach a deployed target with a short-lived token the deployer's issuer minted. `examples/deployers/orders.yaml` shows both kinds. + +- **`spec.selector`** has `kind`, `namespace`, `name` or `labels`, and `container`. + - `kind` is `Deployment`, `StatefulSet` or `Pod`. `labels` match the workloads of that kind, and the targets are the running pods the workloads' own pod selectors match. + - `name` selects the one workload (or pod) of that kind instead, which is what the web wizard's workload picker sends. A Helm-managed Deployment often carries only release labels, so naming it is the way to reach exactly its pods. A selector with both `name` and `labels` is refused. + - `container` is the JVM container. A selector that matches no running pods, or names a container a pod lacks, is an error. + - Each target is named `deployer///`. That name is the audience of its tokens. +- **`spec.javaContainer`** is keyed by `kind`, and each kind is a registry entry (`deploy.RegisterJavaContainer`) that validates its own fields. + - **`tomcat`** embeds commons-db `HTTPConnection` as the Tomcat manager login: `connection`, `username`/`password`, `bearer`, `oauth` and `tls`. + - `url` is a Go template rendered per target to the Tomcat base URL. It binds `.Pod.Name`, `.Pod.Namespace`, `.Pod.Labels`, `.Pod.IP`, `.Container` and `.Port ""`. A binding the target lacks is an error, never ``. + - The rendered URL goes through commons-db `ResolveConnectionURL` (`proxy://`, `portforward://`, `svc://`, `http(s)://`). + - `url` is what the deploy uses: the manager upload and the `/health` probe. The deployer uploads a per-target `telltale-diag.war` to `/text/deploy?path=&update=true`. `managerPath` defaults to `/manager` and `contextPath` to `/telltale-diag`; `runtimeDir` is required. + - The manager login needs its `Authorization` header, which the apiserver deletes from proxied requests. So `url` is a `portforward://.:?kind=pod` or a direct `http(s)://` URL. A `proxy://` URL, or a literal apiserver `.../pods|services//proxy` URL, is refused when the deployer is decoded. + - `executionUrl` (optional, default `url`) is the base executions use, with `contextPath` appended ahead of any query. It may be `proxy://.:?kind=pod`, because the token travels in `X-Telltale-Authorization`. Executions use the connection's TLS and no manager login. The commons-db connection transport adds the apiserver's own credentials to proxy URLs. + - **`attach`** copies the agent jar with `kubectl exec` and attaches it to `pid`. It starts the agent's HTTP listener on `bind`:`port`, which serves the same routes and verification as the WAR. `url` is the listener's URL template. + - **`javaagent`** is registered but not implemented yet. A deployer naming it fails with that message. +- **`spec.auth.issuer`** names the connection whose `signingKey` named secret (an RSA PEM key) signs tokens. Its reference is the tokens' `iss`. + - Agents are provisioned with the public JWKS only. + - `spec.auth.jwksUrl` names where the issuer runtime serves its JWKS (`auth.Issuer` is an `http.Handler`), so a rotated key is picked up on its first unknown `kid`. + +Deploying is idempotent. The deployer reads each target's `/health`, and a target already running agent API `1` with the same provision digest is left alone. A different provision is redeployed with `tomcat`. An attached runtime is replaced only when `--upgrade` is given. + +### Upgrade an attached runtime + +After building and installing the new CLI, stop the target's captures and restore its bytecode patches, then deploy with `--upgrade`: + +```bash +telltale jvm deploy --upgrade --file "$HOME/.config/telltale/deployers/api.yaml" +telltale agents install java --upgrade --pid 1234 --directory /home/example/telltale/agent --service orders --allow-mutation --extensions-dir /home/example/telltale/extensions +``` + +The launcher loads the replacement artifact in an isolated classloader, validates it, and waits up to ten seconds for diagnostic calls to finish. Active captures or patches refuse the upgrade before the running runtime is stopped. An idle runtime releases its transports, observer and patch transformers, and extension resources before the replacement starts. Clients reconnect to the new runtime; the application PID and JVM source identity stay the same. If activation fails, the launcher attempts to restore the previous artifact and configuration and reports whether recovery succeeded. + +The attached listener's `/health` reports `runtimeVersion`, `upgradeApiVersion` and the configured `artifactSHA256`; `/hello` also reports the runtime's `sdkVersion`. The same `upgrade` boolean is available through both deployment RPC routes. `--upgrade --dry-run` previews the selected targets without replacing anything. This flag applies to `attach` deployers and single-PID attachments. + +An agent installed before the upgrade-capable launcher requires one JVM restart to adopt it. Later runtime upgrades can change class structure without restarting the application. The small system-loader probe bridge remains versioned separately: a replacement requiring an incompatible bridge API is rejected before shutdown and requires a JVM restart. + +Each target reports `{target, namespace, pod, container, state, agentAPIVersion, endpoint, error}`, where `state` is `deployed`, `outdated` or `failed`. `outdated` means the JVM still runs another agent API after the deploy; restart the JVM to replace it. + +`deploy.Connect` dials each deployed target over direct HTTP (`agentclient.DialHTTP`) into an agent registry. The agent ID is the target id, and its resource carries `telltale.deployer`, `telltale.target.id`, `k8s.namespace.name`, `k8s.pod.name` and `k8s.container.name`. Captures then name the target (`target.agent: `) or select it (`target.selector: {telltale.deployer: orders}`), never a URL. + +Arm, checkpoint, renew, disarm and calls run through the same `agentclient.Agent` and `seam.AgentSeam` as hub agents: + +- Each request carries a token for the scope the agent's `/hello` names. +- Streaming captures are read every 250 ms and renewed every 30 seconds. +- A capture ends complete when the agent sets `X-Telltale-Complete`. + +For Kubernetes attachment, omit `pid` to scan `/proc//exe`. Exactly one executable ending in `/bin/java` must be visible; multiple JVMs require an explicit PID. The installer reads the selected process's start time from `/proc//stat` and derives its Java home from `/proc//exe`, including Java 8's `jre/bin/java` layout. Discovery needs `sh` and `readlink`, with no `ps`, `jps`, or configured `JAVA_HOME`. SSH and local attachment require an explicit PID. + +On Linux, the installer inspects the runtime's `release` metadata and Attach API before staging the agent. If the runtime is a JRE, it uses `deps` to download and checksum-verify a matching major-version JDK for the target architecture, caches it on the Telltale host, and copies it through the exec transport into `/jdk--`. The transfer preserves symlinks and executable permissions, validates the copied JDK, and publishes it only after a successful transfer. Later attaches reuse the copied JDK. Pod recreation removes these temporary files; a later deployment provisions them again from the host cache. The target container needs no downloader. `--java-home` selects an explicit full JDK and is rejected if it lacks the Attach API. + +### From the CLI + +On Linux, a JRE without the Attach API triggers automatic JDK provisioning. Telltale reads the Java major from the runtime's `release` file, uses `deps` to download and verify a JDK for the target OS and architecture, then copies it into a version-family directory under `--directory` (for example, `/tmp/telltale/jdk-8-amd64`). The copy preserves executable permissions and symlinks, validates the Attach API, and publishes the directory only after transfer succeeds. Later installations reuse that JDK without downloading or copying it again. A new pod provisions its own copy. The helper runs from that JDK with Java option environment variables removed; the application JVM keeps running. An explicit `--java-home` is honored and must contain the Attach API. + +`telltale jvm deploy` and `telltale agents install java` are one command with the same flags: either name deploys a Deployer or attaches to a single PID. Which inputs you give picks the mode, and a flag the chosen mode would ignore is refused rather than dropped: + +| Inputs | Mode | +|---|---| +| `-f ` or `--deployer ''` (at most one) | Deploy that Deployer. | +| a `[kind/]name` target, or `--issuer connection:///` | Deploy a Deployer built from the flags (below). | +| neither | Attach to one `--pid` at a `local`, `ssh` or `kubernetes` `--destination`, verified by local discovery or by a hub (`--hub-url`, `--token-file`). See [sqlserver-jvm.md](sqlserver-jvm.md). | + +A Deployer is deployed with your kubeconfig and the connections telltale reads, which must define the `spec.auth.issuer` connection and its `signingKey`. `--kubeconfig ` and `--context ` choose the cluster; the API client, the `kubectl exec` that attaches the agent and every port-forward use it. The command prints each target's state and keeps the deployer with that status in the global config dir (`deployers/.yaml`), even when some targets fail, so the deployed ones stay reachable. The kept deployer records the kubeconfig and context as `telltale.flanksource.com/kubeconfig` and `telltale.flanksource.com/kubecontext` annotations, so a later `--agent deployer/…` reconnects to the same cluster whatever the current context is. Run it again after pods are replaced; it is idempotent. `telltale jvm deployers` lists the kept targets (`--format json` prints them as JSON records). `--dry-run` resolves the selector and prints each pod it picks with state `selected`, without an issuer, without installing anything and without keeping the deployer. In single-attach mode `--dry-run` previews the artifact and its path instead. + +A Deployer built from flags is one `attach` container per selected pod. Everything but the target defaults: + +| Part | Flags | Default | +|---|---|---| +| Pods | a target: `sts/`, `deploy/`, `pod/` (or the long and plural forms), or a bare pod name. Without one, `--pod `, or `--kind Deployment\|StatefulSet\|Pod` with `--workload ` or `--labels k=v,…` | — | +| Namespace | `--namespace` | the kubeconfig context's namespace | +| Container | `--container` | the pod's only container; a pod running several is refused with their names | +| JVM | `--pid`, `--directory` | discover the only Java process from `/proc`, `/tmp/telltale` | +| Identity | `--service`, `--name` | the target's name; `--name` defaults to `--service` and names the deployer in its target ids | +| Listener | `--bind`, `--port`, `--url` | `127.0.0.1` (reachable only through a port-forward), `7070`, `portforward://{{.Pod.Name}}.{{.Pod.Namespace}}:?kind=pod` | +| Auth | `--issuer`, `--jwks-url`, `--jwks-mode read\|publish` with `--jwks-kind ConfigMap\|Secret`, `--jwks-name`, `--jwks-key` | `connection://telltale/issuer`; agents get the issuer's own JWKS inline | + +`--java-home`, `--extensions-dir` and `--allow-mutation` apply as for a single attach. The `tomcat` kind is deployed from a Deployer document. + +On first use the default issuer is created as `/flanksource/connections/issuer.yaml`, the connections every Flanksource tool shares: namespace `telltale`, name `issuer`, a new 3072-bit RSA key as its `signingKey` secret, readable only by you. The command says so on stderr. An existing issuer, in any connection layer, keeps its key, so agents deployed with it stay valid. A dry run never creates it. `go run ./hack/signing-connection ` writes another issuer the same way. + +```bash +telltale agents install java --namespace shop sts/orders # every running pod of StatefulSet orders +telltale jvm deploy orders-0 --dry-run # the pods it would select, nothing installed +telltale jvm snapshot --agent deployer/orders/shop/orders-0 +``` + +Over the API, `telltale serve` exposes the same command as `POST /api/v1/jvm/deploy` and `POST /api/v1/agents/install/java`, with the same inputs, executed on the server with its kubeconfig and connections. Send the document as `deployer`; `file` is refused over HTTP because it would read the server's filesystem. For example, `{"deployer":"apiVersion: telltale.flanksource.com/v1alpha1\nkind: Deployer\n...","dry-run":true}` answers the selected targets as `[{"target":"deployer/orders/shop/orders-0","namespace":"shop","pod":"orders-0","container":"app","state":"selected"}]`. Without `dry-run` it deploys and keeps the deployer as the CLI does, and `/api/v1/jvm/deployers` lists the kept targets as the same records. + +Every `jvm` command and capture then names a target by its id, or by a short name. `--agent` takes the agent's id, its service, the last part of its id, its pid, a deployed target's deployer or pod, or a unique prefix of any of these. An exact name wins over prefixes. Leave `--agent` out when only one agent can serve the command: the one connected Java agent, the one connected agent offering the capture's probe, or the one deployed target. A name that matches several agents, or an omitted one with several to choose from, is refused with the candidates listed. The web wizard likewise arms a new agent capture on the only agent that can run it. + +```bash +telltale jvm metrics --agent checkout # the deployer's one target +telltale jvm metrics --agent checkout-7d # a pod-name prefix +telltale jvm metrics # the only Java agent there is +``` + +A deploy confirms each target's health for up to 30 seconds and resolves the target's URL again on every attempt. A port-forward opened before the attached listener came up is torn down by its first refused connection, and resolving opens a new one, so a first deploy reports `deployed` once the agent answers. + +By id: + +```bash +telltale jvm deploy -f examples/deployers/orders.yaml +telltale jvm deployers +telltale jvm metrics --agent deployer/orders-batch/orders/orders-batch-7d9f-abcde +telltale jvm watch --agent deployer/orders-batch/orders/orders-batch-7d9f-abcde --allow-mutation --duration 10s --targets '[{"class":"example.Worker","method":"compute"}]' +``` + +The first use of a target id in a process loads its deployer, builds the issuer from the connection, resolves the target's endpoint (opening its port-forward) and connects. Each request then carries a token minted for that target, the scope of the operation or capture, and the subject `@`. Only `deployer/…` ids reach a cluster, so local agent lookups never depend on it. A server started with `telltale serve` resolves the same ids for `/api/v1/jvm/…` and `POST /api/traces`. + +## Verification + +For an interactive local demo, first run `make build`, then: + +```bash +gavel fixtures examples/jvm-diagnostics.fixture.yaml --ui --addr 127.0.0.1 +``` + +The fixture builds a disposable Spring application with embedded Tomcat, H2 and Artemis using the pinned dependencies in Gradle's `jvmDemo` task. It starts a local Telltale OTLP receiver, warms the application's clients and existing OpenTelemetry exporter, then attaches Telltale without restarting the JVM. A full JDK 17+ (`java`, `javac`, `jcmd`), Go, curl and the built `.bin/telltale` must be available. Use a Gavel build that includes `kind: telltale` and form steps. All listeners bind to loopback on allocated ports; no external application or database is required. + +Native capture steps retain the Groovy, method tracing, compile preview, managed redefinition, OGNL and original-byte restoration checks. Protocol checks exercise incoming Spring MVC requests, outgoing RestTemplate/WebClient calls, JDBC/R2DBC statements and Spring JMS sending/listening. Deterministic held operations verify successive HTTP/SQL Top snapshots retain their operation IDs while elapsed time increases. Releasing the operations verifies the schema-v2 Profiles control sample clears the Top rows. Mirrored spans are compared with normal OTLP exporter delivery. + +After verification, periodic traffic keeps the captures active. The final Exit form shows the diagnostics, capture and workload URLs and waits while you explore. Submit Exit or cancel to stop the owned captures, JVM and server. Captures, native snapshot evidence and agent artifacts remain in the unique `.tmp/jvm-demo/run.*` directory. Without `--ui`, run from a terminal to answer the same final form there. To run the same checks and clean up immediately, use: + +```bash +gavel fixtures examples/jvm-diagnostics.fixture.yaml --param interactive=false +``` + +Java tests cover the copied services, real extension JAR lifecycle, isolated Groovy and embedded Tomcat WAR reload. The disposable-JVM installer integration checks packaged calls, trace budget completion, deadlines, method replacement/restoration and JFR decoding. API specs check live catalog changes, script integer precision and extension install/replace/unload over HTTP. UI tests check mutation consent and result conversion. Work is tracked by Gavel TODO `821b9a83-f816-4e6b-8b3d-c96ce5751f57`. diff --git a/docs/jvm-snapshot.md b/docs/jvm-snapshot.md new file mode 100644 index 0000000..81e1a1d --- /dev/null +++ b/docs/jvm-snapshot.md @@ -0,0 +1,35 @@ +# JVM support snapshots + +`telltale jvm snapshot` creates a ZIP of diagnostic evidence from a Java process. Select a connected agent or a local PID. PID selection attaches the bundled agent when necessary and reuses an existing agent without changing its configuration. Attachment requires a compatible JDK and permission to attach. + +```bash +telltale jvm snapshot --pid 1234 --output support.zip +telltale jvm snapshot --agent 1234 --jfr 30s --heap-histogram --output profiled-support.zip +telltale jvm snapshot --server http://localhost:8080 --agent hub/orders --output orders-support.zip +``` + +`--agent` selects a connected Java agent; `--pid` selects a JVM on the machine running the command. They are mutually exclusive. `--java-home` chooses a target-compatible JDK for attachment. `--server` sends collection to a Telltale hub; set `TELLTALE_API_TOKEN` when it requires API authentication. `--output` names a ZIP that must not already exist; omitted, a timestamped filename is generated. Files are created with owner-only permissions. + +The bundle contains OS/JVM properties and environment, runtime and operating-system MBeans, filesystem capacity, Linux process and cgroup settings, effective HotSpot VM flags, memory/GC metrics, thread stacks and deadlocks, class loaders, logger levels, agent identity and diagnostic status. `summary.md` describes collection outcomes. `manifest.json` records schema version, collection times, completion status and SHA-256 hashes of evidence files. Unsupported or denied collectors produce explicit errors rather than invented values. Linux-specific resources are marked not applicable on other operating systems. + +Known secret properties, environment values, argument assignments and embedded credentials are masked before archive serialization. JFR uses a restricted built-in CPU/allocation/lock/GC event configuration, excluding arbitrary application events and raw environment/property/command-line events. Review diagnostic evidence before distributing it: paths, class names, thread names and application identifiers remain useful support information. + +`--jfr` enables profiling for a positive duration up to five minutes, such as `30s`. The Telltale host requires a JDK 19+ `jfr` tool on PATH for export scrubbing, independently of the target JVM version. It rebuilds the recording to exclude events enabled by other recordings and removes unused constant-pool data. Raw intermediate recordings are held in a private directory under `.tmp/jvm-snapshot` and removed after scrubbing or failure. It owns and closes only its recording. `--heap-histogram` counts all objects, including unreachable objects, without requesting a full collection. Heap inspection can pause the JVM; cancelling the caller cannot interrupt a native heap walk already in progress. These collectors are disabled by default. Full heap dumps and application logs are not collected. + +## Native activity telemetry + +The ZIP includes `telemetry.json` and `telemetry.html`. The HTML report is self-contained: it embeds the sanitized JSON, uses no external assets, and opens directly from the extracted archive without Telltale or internet access. It shows each application's telemetry state and counters, histogram labels, observation counts, total duration, and expandable cumulative buckets. + +The read-only `jvm.telemetry` operation finds loaded `com.oipa.telemetry.Telemetry` implementations through JVM instrumentation and invokes their native `snapshot()` API with the application's classloader. It collects every matching loader separately, so multiple webapps in one JVM keep their own identity and measurements. It does not arm telemetry, register extensions, scrape an HTTP listener, or query activity MBeans. Each application entry contains `loader`, `label`, `status`, and `histograms`; the JSON preserves 64-bit counts exactly and the HTML displays them without converting them to JavaScript numbers. + +A JVM with no loaded activity telemetry produces an empty applications list and an explanatory report. An older implementation without the native snapshot API, a denied read, or a provider failure is recorded as an explicit collector failure and preserves the remaining evidence in a partial archive. Update the telemetry extension and restart that JVM when the native API is missing. + +## Workload panel + +`--workload` (default `1m`, from `5s` to `5m`; pass `--workload ""` to skip) samples `jvm.metrics`, `jvm.resources` and `jvm.threads` every 5 seconds after the other collectors finish. `workload.json` holds the samples. `workload.html` opens straight from the extracted ZIP and needs no Telltale server: it holds the summarised figures inline and renders them with the published `@flanksource/clicky-ui` `WorkloadPanel`. It loads React and clicky-ui from esm.sh and unpkg, so the viewer needs internet access. + +The panel shows CPU in cores and container memory, both read from cgroup v2 with the cgroup limits; a JVM outside a cgroup v2 container fails this collector. It also shows heap regions, metaspace, code cache and direct buffers, GC, and threads. Values are averages with P90 and P95 over the samples. GC pause percentiles are the average pause within each 5-second interval, because the JVM only exposes cumulative collection counts and times. Peak threads is the highest live count sampled. Kubernetes requests are not visible from inside the container, so the panel shows no request marker. + +Collection failures preserve a partial ZIP and cause a nonzero CLI exit; inspect `summary.md`. Invalid requests and unavailable targets fail before collecting evidence. Existing destination files are never overwritten. + +The JVM Diagnostics Snapshot tab provides the same options, cancellation and a download link. `POST /api/v1/jvm/snapshot` accepts `{ "agent": "hub/orders", "jfr": "30s", "heapHistogram": true, "workload": "1m" }` and returns `application/zip`. The optional fields may be omitted; an omitted `workload` skips the workload panel. `X-Telltale-Snapshot-Status` is `complete` or `partial`. A valid partial archive uses HTTP 200 so callers can retain the evidence; invalid input uses HTTP 400. Local filesystem output paths are not accepted through HTTP. The endpoint inherits API authentication and same-origin protection. diff --git a/docs/jvm-top.md b/docs/jvm-top.md new file mode 100644 index 0000000..a5ac3f7 --- /dev/null +++ b/docs/jvm-top.md @@ -0,0 +1,20 @@ +# JVM Top + +Monitor one Java runtime from a terminal or the web. JVM Top reads the agent's existing metrics and thread operations; install or deploy an agent first. With one discovered Java agent, no selector is needed. Otherwise, `--agent` accepts its ID, service, pod, PID, or another unique short name. + +```sh +telltale jvm top +telltale jvm top --agent orders --every 2s +telltale jvm top --agent orders --server http://localhost:8080 +telltale jvm top --agent orders --once --format json +``` + +`--every` sets the refresh delay after each completed sample (default 2s, minimum 1s). `--once` prints one snapshot and exits. Redirected output and structured formats also read one snapshot. `--server` samples through a running Telltale hub and uses `TELLTALE_API_TOKEN` for authenticated hubs. + +The terminal uses the same compact meters, styles and Clicky table as `telltale top`. Use arrows or j/k to select a thread, s to cycle sorting, S to reverse it, / to filter, P or space to pause/resume, d or Enter to inspect the selected thread's stack, ? for help, and q to quit. Enter keeps a filter; Esc clears it. In stack inspection, arrows scroll the wrapped stack, Esc or d closes it while remaining paused, and P or space resumes live sampling. Threads that exit before inspection produce an explicit error. + +Open `/jvm/{agent}/top`, or choose JVM diagnostics from the Agents page. Top is the default JVM view. Thread selection, filters and ordering persist in the URL. Polling pauses during inspection and while the page is hidden. A failed refresh leaves the last snapshot visibly marked stale. + +Process CPU is a percentage of the JVM's available processor capacity; thread CPU is a percentage of one core, sampled over 200ms. Garbage collector counts and times are lifetime totals. Unavailable values remain unavailable; unbounded memory pools have no artificial maximum. This view does not enable CPU timing, force garbage collection, instrument methods or record history. + +`POST /api/v1/jvm/top` accepts `agent`, `once` and `format: "json"` and always returns one snapshot through the usual Clicky command response. Its `output` string contains a frame with `agentId`, `sampledAt`, `metrics` and `threads`; the generated OpenAPI includes the `JVMTopFrame` schema. Sampling and stack reads share the existing API authentication rules. diff --git a/docs/otlp-receiver.md b/docs/otlp-receiver.md index d1d7604..0f323b9 100644 --- a/docs/otlp-receiver.md +++ b/docs/otlp-receiver.md @@ -50,7 +50,7 @@ From this source checkout, `go run ./examples/otel` exports a two-span demonstra ## Forward from an OpenTelemetry Collector -The runnable [Collector configuration](../examples/otel-collector.yaml) listens on ports 14317 and 14318 and forwards traces to Telltale over HTTP/protobuf. With Telltale already listening on 4318: +The runnable [Collector configuration](https://github.com/flanksource/telltale/blob/main/examples/otel-collector.yaml) listens on ports 14317 and 14318 and forwards traces to Telltale over HTTP/protobuf. With Telltale already listening on 4318: ```bash otelcol-contrib --config examples/otel-collector.yaml diff --git a/docs/package-lock.json b/docs/package-lock.json new file mode 100644 index 0000000..bfd0f3f --- /dev/null +++ b/docs/package-lock.json @@ -0,0 +1,3622 @@ +{ + "name": "@flanksource/telltale-docs", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@flanksource/telltale-docs", + "devDependencies": { + "@rspress/core": "2.1.0" + } + }, + "node_modules/@emnapi/core": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.3.tgz", + "integrity": "sha512-zLpS5asjEb7lq8jYLq37N6XKaE41DIexlY1rF/z4/tIl3wo13Sqm28fRyfIsKZD+NZ8mM5RoKkpW/rBcuoSZSg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.3", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", + "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.3.tgz", + "integrity": "sha512-ELEBe8PsLvvJ6QMr0zLt8ffvOHW/dc1m3CEzNMg7aJUv3bMaoDtw2TXyDAwkYBuroxxuHEwhRTLJSe5sya547g==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@mdx-js/mdx": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@mdx-js/mdx/-/mdx-3.1.1.tgz", + "integrity": "sha512-f6ZO2ifpwAQIpzGWaBQT2TXxPv6z3RBzQKpVftEWN78Vl/YweF1uwussDx8ECAXVtr3Rs89fKyG9YlzUs9DyGQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdx": "^2.0.0", + "acorn": "^8.0.0", + "collapse-white-space": "^2.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "estree-util-scope": "^1.0.0", + "estree-walker": "^3.0.0", + "hast-util-to-jsx-runtime": "^2.0.0", + "markdown-extensions": "^2.0.0", + "recma-build-jsx": "^1.0.0", + "recma-jsx": "^1.0.0", + "recma-stringify": "^1.0.0", + "rehype-recma": "^1.0.0", + "remark-mdx": "^3.0.0", + "remark-parse": "^11.0.0", + "remark-rehype": "^11.0.0", + "source-map": "^0.7.0", + "unified": "^11.0.0", + "unist-util-position-from-estree": "^2.0.0", + "unist-util-stringify-position": "^4.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/@mdx-js/react": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@mdx-js/react/-/react-3.1.1.tgz", + "integrity": "sha512-f++rKLQgUVYDAtECQ6fn/is15GkEH9+nZPM3MS0RcxVqoTfawHvDlSCH7JbMhAM6uJ32v3eXLvLmLvjGu7PTQw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mdx": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + }, + "peerDependencies": { + "@types/react": ">=16", + "react": ">=16" + } + }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz", + "integrity": "sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.3" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } + }, + "node_modules/@rsbuild/core": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@rsbuild/core/-/core-2.3.1.tgz", + "integrity": "sha512-dv5tL7djb8ex5z4sqnDmUbd5PnGNr/CXuFfiMD4RkUsF1JEZZkGH8lc+QW4fqYQIHM7RTvcMUakgvwVU/O9WMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rspack/core": "~2.3.1", + "@swc/helpers": "^0.5.23" + }, + "bin": { + "rsbuild": "bin/rsbuild.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "peerDependencies": { + "core-js": ">= 3.0.0" + }, + "peerDependenciesMeta": { + "core-js": { + "optional": true + } + } + }, + "node_modules/@rsbuild/plugin-react": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@rsbuild/plugin-react/-/plugin-react-2.1.1.tgz", + "integrity": "sha512-tLtUedsRtTLS6M4VtwzI6aakFLOFdgl9Sh1ZNBWvxb0Te3EnHrppaE7UiYEmDUXQDJ50gfr6yyirtxY5N6moyg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rspack/plugin-react-refresh": "^2.0.2", + "react-refresh": "^0.19.0" + }, + "peerDependencies": { + "@rsbuild/core": "^2.0.0" + }, + "peerDependenciesMeta": { + "@rsbuild/core": { + "optional": true + } + } + }, + "node_modules/@rspack/binding": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@rspack/binding/-/binding-2.3.1.tgz", + "integrity": "sha512-pfCJGBFNK2tnNnoDP6ZoTbOjxbKr1U2eGFDJ8bIuZQai4FBNhWQlKFmNup+X5K935XYRulZBomIBAL6GL6W2YQ==", + "dev": true, + "license": "MIT", + "optionalDependencies": { + "@rspack/binding-darwin-arm64": "2.3.1", + "@rspack/binding-darwin-x64": "2.3.1", + "@rspack/binding-linux-arm64-gnu": "2.3.1", + "@rspack/binding-linux-arm64-musl": "2.3.1", + "@rspack/binding-linux-ppc64-gnu": "2.3.1", + "@rspack/binding-linux-riscv64-gnu": "2.3.1", + "@rspack/binding-linux-riscv64-musl": "2.3.1", + "@rspack/binding-linux-s390x-gnu": "2.3.1", + "@rspack/binding-linux-x64-gnu": "2.3.1", + "@rspack/binding-linux-x64-musl": "2.3.1", + "@rspack/binding-wasm32-wasi": "2.3.1", + "@rspack/binding-win32-arm64-msvc": "2.3.1", + "@rspack/binding-win32-ia32-msvc": "2.3.1", + "@rspack/binding-win32-x64-msvc": "2.3.1" + } + }, + "node_modules/@rspack/binding-darwin-arm64": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@rspack/binding-darwin-arm64/-/binding-darwin-arm64-2.3.1.tgz", + "integrity": "sha512-koLyL2r8PUcLD0StjuZ/n5XM6GJ0C8dzJN0IuIV9d7nQcibJxwjjSAWEPGVz6bm8WuzypvuVKNoa1+GGAHeMQA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rspack/binding-darwin-x64": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@rspack/binding-darwin-x64/-/binding-darwin-x64-2.3.1.tgz", + "integrity": "sha512-YK39JLiu1YC6+3BMxkP7e848pHD7e2T/0OSNogDi7whfiZNg95wa9Ncd7I5ppAf0nhmV+FNew/sCBIFp5h67Jw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rspack/binding-linux-arm64-gnu": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@rspack/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-2.3.1.tgz", + "integrity": "sha512-Q+RCA/KM2tn6m6R5U4lktXDs5cCRm3SyleQk86iqqUFX7wKzVRUvr5BTsreg6K9of3gpYmQ6+g1PGT5cwo0rLA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rspack/binding-linux-arm64-musl": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@rspack/binding-linux-arm64-musl/-/binding-linux-arm64-musl-2.3.1.tgz", + "integrity": "sha512-He+muVDON1fohGtdYamh3zHVwf6/E6fZu/+iEAkxjclE0P94bsCfYVcZW/a4/l1tJ2zvLZtB4jD7Qr6r4vG/WA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rspack/binding-linux-ppc64-gnu": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@rspack/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-2.3.1.tgz", + "integrity": "sha512-blIZUgyPt4FcSHkZL6sJ2xbzc7ACgDeU49An4mXLSebwdgnlhTzxeSFnh2VeJDlGza8CUaZYB/zS/6oZ2kCDcw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rspack/binding-linux-riscv64-gnu": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@rspack/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-2.3.1.tgz", + "integrity": "sha512-3FtW2DUFmXxhBORUA3zSc+ot4d6R/1/U6z/48TDN4jeIYDUG9VqYBF6LUohX4vBY6IP0DCyXb2N9WayD/9QXoA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rspack/binding-linux-riscv64-musl": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@rspack/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-2.3.1.tgz", + "integrity": "sha512-+5qNgdNdXzp/K8f8BjgJ3m7tcbBKPENN7WPDfBcSHvWb5GF4+o/pbrX26c0+LEugo0bYUOAoitBNd94AwcNvpA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rspack/binding-linux-s390x-gnu": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@rspack/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-2.3.1.tgz", + "integrity": "sha512-yNcg/y1+MV0wZ984CVU4d1HAYYwMcDPXoQO0xynU+K5YuBxPWALHOxuRRCKCcykBdL1GIEshAIewEFrsW/cmJg==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rspack/binding-linux-x64-gnu": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@rspack/binding-linux-x64-gnu/-/binding-linux-x64-gnu-2.3.1.tgz", + "integrity": "sha512-ffclKBuH8wwimOBd6xGUxrBfVKKY5Ow4E7OFdHZeg4xffm2O8auGszmOgvFu1eoDoOYHaVJn+cEV6UD0KwZk8A==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rspack/binding-linux-x64-musl": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@rspack/binding-linux-x64-musl/-/binding-linux-x64-musl-2.3.1.tgz", + "integrity": "sha512-qcDwwuyxR7TMtrW+r5Y9RIW25boXd1Ip/9jz04jhdq/OiU/Am7rCP2X90CH8sWdzovIF9UNmsGjY4KklR0F2Mg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rspack/binding-wasm32-wasi": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@rspack/binding-wasm32-wasi/-/binding-wasm32-wasi-2.3.1.tgz", + "integrity": "sha512-3dr6zjFaCK3A920ftCriSJkiRhEfhHtat7LtBdj6g0mcV5Rt+iealmli0vjeAvD0UdpE1u8EKWlfw82JLgFLSQ==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "1.11.3", + "@emnapi/runtime": "1.11.3", + "@napi-rs/wasm-runtime": "1.1.6" + } + }, + "node_modules/@rspack/binding-win32-arm64-msvc": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@rspack/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-2.3.1.tgz", + "integrity": "sha512-AZbuLPylt1xvWssN93AsirbYs3WgfcrA8tMsqHmVJIHahLu9E+JPKZ3Gmsw6IYEZk3EfnV3ujQzeQ1/VfF0Vsw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rspack/binding-win32-ia32-msvc": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@rspack/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-2.3.1.tgz", + "integrity": "sha512-R5NVU68TePZcgRdYUzrQX9YFNxmcyXZEPwaPv40AAgsqNyacG3iRaYVmV79jt3rWYfa0QJaqvRLUchDe2SZ6zQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rspack/binding-win32-x64-msvc": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@rspack/binding-win32-x64-msvc/-/binding-win32-x64-msvc-2.3.1.tgz", + "integrity": "sha512-5T9DYjNPQpLbrNXjCssGd2B6xMQPlc3LxVOKHGUA1O0GV+88vBkTXAcryJ+Fo+yVEMPCzp4VvwBMGAVDpCjcSA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rspack/core": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@rspack/core/-/core-2.3.1.tgz", + "integrity": "sha512-9BWfaiEJLgLK9U53Q4WVruBZ19Mu/XWHXHEu0UFla4lUtH8Z+/ET2m044tiqbtyIgqQyG58fMrxHeVIAiXMXHg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rspack/binding": "2.3.1" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "peerDependencies": { + "@module-federation/runtime-tools": "^0.24.1 || ^2.0.0", + "@swc/helpers": "^0.5.23" + }, + "peerDependenciesMeta": { + "@module-federation/runtime-tools": { + "optional": true + }, + "@swc/helpers": { + "optional": true + } + } + }, + "node_modules/@rspack/plugin-react-refresh": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@rspack/plugin-react-refresh/-/plugin-react-refresh-2.0.2.tgz", + "integrity": "sha512-dGNZiCxQxgAUI9sah7gd8u+O7OJZRCmqtEJNDOd8xW5RqcieC86F7p5qcShyw6onH5pKf57evpr2VjGbaFGkZg==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@rspack/core": "^2.0.0", + "react-refresh": ">=0.10.0 <1.0.0" + }, + "peerDependenciesMeta": { + "@rspack/core": { + "optional": true + } + } + }, + "node_modules/@rspress/core": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@rspress/core/-/core-2.1.0.tgz", + "integrity": "sha512-xw5iGhzLn4O8vGUj8+y7AiVo51Pd1l1GkIBF9q5btfXTYR0xkGY984rxMFOYI43etvlRRkMM2du+IwiyqUnPGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@mdx-js/mdx": "^3.1.1", + "@mdx-js/react": "^3.1.1", + "@rsbuild/core": "^2.2.14", + "@rsbuild/plugin-react": "~2.1.1", + "@rspress/shared": "2.1.0", + "@shikijs/rehype": "^4.2.0", + "@types/mdast": "^4.0.4", + "@types/react": "^19.3.0", + "@unhead/react": "^2.1.17", + "body-scroll-lock": "4.0.0-beta.0", + "clsx": "2.1.1", + "copy-to-clipboard": "^3.3.3", + "flexsearch": "0.8.212", + "hast-util-heading-rank": "^3.0.0", + "hast-util-to-jsx-runtime": "^2.3.7", + "mdast-util-mdx": "^3.0.0", + "mdast-util-mdxjs-esm": "^2.0.1", + "medium-zoom": "1.1.2", + "nprogress": "^0.2.0", + "react": "^19.3.1", + "react-dom": "^19.3.1", + "react-lazy-with-preload": "^2.2.1", + "react-render-to-markdown": "19.1.0", + "react-router-dom": "^7.18.5", + "rehype-external-links": "^3.0.0", + "rehype-raw": "^7.0.0", + "remark-cjk-friendly": "^2.3.3", + "remark-cjk-friendly-gfm-strikethrough": "^2.3.2", + "remark-gfm": "^4.0.1", + "remark-mdx": "^3.1.1", + "remark-parse": "^11.0.0", + "remark-stringify": "^11.0.0", + "scroll-into-view-if-needed": "^3.1.0", + "shiki": "^4.2.0", + "unified": "^11.0.5", + "unist-util-visit": "^5.1.0", + "unist-util-visit-children": "^3.0.0", + "vfile": "^6.0.3" + }, + "bin": { + "rspress": "bin/rspress.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rspress/shared": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@rspress/shared/-/shared-2.1.0.tgz", + "integrity": "sha512-D2wnLO4aV1eFxNbUqQaLY4QfZZktegM+lCIDbQ9H2wxNQHKde39U1hpRdZKIA4aCuPYf/RSb6EdmJiZOkoZaWw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rsbuild/core": "^2.2.14", + "@shikijs/rehype": "^4.2.0", + "@types/react": "^19.3.0", + "mdast-util-mdx-jsx": "^3.2.0", + "unified": "^11.0.5" + } + }, + "node_modules/@shikijs/core": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@shikijs/core/-/core-4.6.0.tgz", + "integrity": "sha512-bfd4OKgvoyXYiYmMRuleA656P4V7M/Du9lcTl38Ngm6KszMj7bq4J1IGZwSYR4c95Tv5UQALtYNWUwaI9UT3xw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/primitive": "4.6.0", + "@shikijs/types": "4.6.0", + "@shikijs/vscode-textmate": "^10.1.0", + "@types/hast": "^3.0.5", + "hast-util-to-html": "^9.0.6" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/engine-javascript": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@shikijs/engine-javascript/-/engine-javascript-4.6.0.tgz", + "integrity": "sha512-K0oqxFoL+kEMdz89mHLFO15Tsbv3kFZ6WR1cXZKMvLSc+oqaM00A6pyga1qSeXB22dP0tTiSApRcGBaKWsEccg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/types": "4.6.0", + "@shikijs/vscode-textmate": "^10.1.0", + "oniguruma-to-es": "^4.3.6" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/engine-oniguruma": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@shikijs/engine-oniguruma/-/engine-oniguruma-4.6.0.tgz", + "integrity": "sha512-aggd6Q7SuJOXNnsUbfG2XesxBJKBMAw/PDTUo9PyRjMaaJJ2dNrI+q0C+25ARXAK9yLHAlwxc9oRT2u68YKZ7w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/types": "4.6.0", + "@shikijs/vscode-textmate": "^10.1.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/langs": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@shikijs/langs/-/langs-4.6.0.tgz", + "integrity": "sha512-1yMEX3oCVV54LUq/ovfpRkDQoYDkcvjSWFFa/YLUDMiSoRzMi2uxLJYbv41ahT8Lu9lFMaa7hecd3Q39trqUyg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/types": "4.6.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/primitive": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@shikijs/primitive/-/primitive-4.6.0.tgz", + "integrity": "sha512-c54+TQEo9ToqgfL/AU8FpjnV09+GA9Oa7M5ROkkXteZfp5XzpAJGXjjvZQFQioJOwm/w/6Z3lIzQstAkl8coag==", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/types": "4.6.0", + "@shikijs/vscode-textmate": "^10.1.0", + "@types/hast": "^3.0.5" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/rehype": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@shikijs/rehype/-/rehype-4.6.0.tgz", + "integrity": "sha512-qv3mGfYJuy+TW6UUyC2QrYSEmTxJNGXZ/NmCEsyH0NHfJIOYGvpWm0kNu6P2Ec14jmGwwTfVLMta/FZtY+MrJQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/types": "4.6.0", + "@types/hast": "^3.0.5", + "hast-util-to-string": "^3.0.1", + "shiki": "4.6.0", + "unified": "^11.0.5", + "unist-util-visit": "^5.2.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/themes": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@shikijs/themes/-/themes-4.6.0.tgz", + "integrity": "sha512-qHAVAs80RiGD+IRuH9G+JPDOEc9A/kHsng4EzCUCwB9bLzW6Z0y9WETh2YyCcADJBmfrIaJ1Jp7KBNP6FHkgtQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/types": "4.6.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/types": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@shikijs/types/-/types-4.6.0.tgz", + "integrity": "sha512-+ENG1adsKt/X7cRYw1Nl7BEpBmPOjmQ7bR/eGmHuMlT8gtOFf3KEi5l31DJYyAWhW6Y6bjP1plU7FmaxWGENFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/vscode-textmate": "^10.1.0", + "@types/hast": "^3.0.5" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/vscode-textmate": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/@shikijs/vscode-textmate/-/vscode-textmate-10.1.0.tgz", + "integrity": "sha512-cVBuhDQjBZaNMTwBquYfPydrUP+E3hE6oYGasx8h6ZrODIM7/r+gxLDXdd66oszbrggZHEntdE6j+5gYIVyMWQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@swc/helpers": { + "version": "0.5.24", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.24.tgz", + "integrity": "sha512-6tcXo4kA8HgGO7AVmawB57RYi/piBPYVHVcO6dXjX+SZw8GoHmi262sp4v2Ha2kNP9Cx84qei27yT+fjS1lxgg==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.4", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.4.tgz", + "integrity": "sha512-W3c4gRigFS0T/Ma4qIYF3GDAc5AQdHb1yL5znJT1Zv1YaD9Kitx656wBjvr19qbiosmZT8lWDM5BEMynUqX65A==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@types/debug": { + "version": "4.1.13", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", + "integrity": "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, + "node_modules/@types/estree": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.10.tgz", + "integrity": "sha512-lTyOMnOcmRBzdCv6/4zUZCrEhV0xrqQiUZBZtVcbmLHBXKLnN8q89v/wKRF/Q0HbwVsqbIG3o5HDoFl9sduM+g==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree-jsx": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@types/estree-jsx/-/estree-jsx-1.0.5.tgz", + "integrity": "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "*" + } + }, + "node_modules/@types/hast": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.5.tgz", + "integrity": "sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/mdast": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", + "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/mdx": { + "version": "2.0.14", + "resolved": "https://registry.npmjs.org/@types/mdx/-/mdx-2.0.14.tgz", + "integrity": "sha512-T48PeuJtvLosNTPVhfnIp3i/n3a4g4Bad7YCq5k64D4u7NwDrAotikQ+5+sjtUvBmxCMlbo3dVL+C2dP0rWHzg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/react": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.3.0.tgz", + "integrity": "sha512-N0rFCuH9YoxG9/m61l9MfpJKfmLOVU0em7ipIz6TRgSSkvReLB9vL85GB+yr8Bs5leqpvg96JSwF4ZS1s4viQg==", + "dev": true, + "license": "MIT", + "dependencies": { + "csstype": "^3.2.2" + } + }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/@ungap/structured-clone": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.4.0.tgz", + "integrity": "sha512-1mEZtMKPM09vDmQt5y7YvmN2+DFTP7Tg0EWXdic8/C6VRnpb33e4ghisCIE3WZjsE2N8mf+QV1Zqh7ZFYLWInQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/@unhead/react": { + "version": "2.1.17", + "resolved": "https://registry.npmjs.org/@unhead/react/-/react-2.1.17.tgz", + "integrity": "sha512-KmcYksDjlLozL0fxUjIDwH/0k6+Lg2HWHLUPlkRW8Gl08hdB1AbLB5iU9QxFvkaT8p0Q4dFEqZWnz3g6RMeiRA==", + "dev": true, + "license": "MIT", + "dependencies": { + "unhead": "2.1.17" + }, + "funding": { + "url": "https://github.com/sponsors/harlan-zw" + }, + "peerDependencies": { + "react": ">=18.3.1" + } + }, + "node_modules/acorn": { + "version": "8.19.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.19.0.tgz", + "integrity": "sha512-oJlA3XiRm7Cyk6qFD2Jn8ak9B7jSy0qy00ADO3+8dpT0LSjFihQYv4C02LFCSYJV3Q37xYwwRy5m+IpIiUzqWw==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/astring": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/astring/-/astring-1.9.0.tgz", + "integrity": "sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg==", + "dev": true, + "license": "MIT", + "bin": { + "astring": "bin/astring" + } + }, + "node_modules/bail": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz", + "integrity": "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/body-scroll-lock": { + "version": "4.0.0-beta.0", + "resolved": "https://registry.npmjs.org/body-scroll-lock/-/body-scroll-lock-4.0.0-beta.0.tgz", + "integrity": "sha512-a7tP5+0Mw3YlUJcGAKUqIBkYYGlYxk2fnCasq/FUph1hadxlTRjF+gAcZksxANnaMnALjxEddmSi/H3OR8ugcQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/ccount": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", + "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-html4": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz", + "integrity": "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-reference-invalid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", + "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/collapse-white-space": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/collapse-white-space/-/collapse-white-space-2.1.0.tgz", + "integrity": "sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/comma-separated-tokens": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", + "integrity": "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/compute-scroll-into-view": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/compute-scroll-into-view/-/compute-scroll-into-view-3.1.1.tgz", + "integrity": "sha512-VRhuHOLoKYOy4UbilLbUzbYg93XLjv2PncJC50EuTWPA3gaja1UjBsUP/D/9/juV3vQFr6XBEzn9KCAHdUvOHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/cookie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/copy-to-clipboard": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/copy-to-clipboard/-/copy-to-clipboard-3.3.3.tgz", + "integrity": "sha512-2KV8NhB5JqC3ky0r9PMCAZKbUHSwtEo4CwCs0KXgruG43gX5PMqDEBbVU4OUzw2MuAWUfsuFmWvEKG5QRfSnJA==", + "dev": true, + "license": "MIT", + "dependencies": { + "toggle-selection": "^1.0.6" + } + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decode-named-character-reference": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", + "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "dev": true, + "license": "MIT", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/entities": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-9.0.0.tgz", + "integrity": "sha512-oJFNDJg3kTPwAD1cNn2+WmUq4wFoo09npc3Ydf1D7ED4smRiWrsKnaLxykrTWgyGEcusc87P5dxlAleol9kpkg==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/esast-util-from-estree": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/esast-util-from-estree/-/esast-util-from-estree-2.0.0.tgz", + "integrity": "sha512-4CyanoAudUSBAn5K13H4JhsMH6L9ZP7XbLVe/dKybkxMO7eDyLsT8UHl9TRNrU2Gr9nz+FovfSIjuXWJ81uVwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "devlop": "^1.0.0", + "estree-util-visit": "^2.0.0", + "unist-util-position-from-estree": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/esast-util-from-js": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/esast-util-from-js/-/esast-util-from-js-2.0.1.tgz", + "integrity": "sha512-8Ja+rNJ0Lt56Pcf3TAmpBZjmx8ZcK5Ts4cAzIOjsjevg9oSXJnl6SUQ2EevU8tv3h6ZLWmoKL5H4fgWvdvfETw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "acorn": "^8.0.0", + "esast-util-from-estree": "^2.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/escape-string-regexp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", + "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/estree-util-attach-comments": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/estree-util-attach-comments/-/estree-util-attach-comments-3.0.0.tgz", + "integrity": "sha512-cKUwm/HUcTDsYh/9FgnuFqpfquUbwIqwKM26BVCGDPVgvaCl/nDCCjUfiLlx6lsEZ3Z4RFxNbOQ60pkaEwFxGw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-build-jsx": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/estree-util-build-jsx/-/estree-util-build-jsx-3.0.1.tgz", + "integrity": "sha512-8U5eiL6BTrPxp/CHbs2yMgP8ftMhR5ww1eIKoWRMlqvltHF8fZn5LRDvTKuxD3DUn+shRbLGqXemcP51oFCsGQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "estree-walker": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-is-identifier-name": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/estree-util-is-identifier-name/-/estree-util-is-identifier-name-3.0.0.tgz", + "integrity": "sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==", + "dev": true, + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-scope": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/estree-util-scope/-/estree-util-scope-1.0.1.tgz", + "integrity": "sha512-B0np3dcdxqILX5e9nEi5/Fr4K7gL4oYFVPV1zRa2e9wRCbQoZZNWOZFYyoInvXUPJXBXjss+QXlWLJChDEHDkA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-to-js": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/estree-util-to-js/-/estree-util-to-js-2.0.0.tgz", + "integrity": "sha512-WDF+xj5rRWmD5tj6bIqRi6CkLIXbbNQUcxQHzGysQzvHmdYG2G7p/Tf0J0gpxGgkeMZNTIjT/AoSvC9Xehcgdg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "astring": "^1.8.0", + "source-map": "^0.7.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-visit": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/estree-util-visit/-/estree-util-visit-2.0.0.tgz", + "integrity": "sha512-m5KgiH85xAhhW8Wta0vShLcUvOsh3LLPI2YVwcbio1l7E09NTLL1EyMZFM1OyWowoH0skScNbhOPl4kcBgzTww==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "dev": true, + "license": "MIT" + }, + "node_modules/flexsearch": { + "version": "0.8.212", + "resolved": "https://registry.npmjs.org/flexsearch/-/flexsearch-0.8.212.tgz", + "integrity": "sha512-wSyJr1GUWoOOIISRu+X2IXiOcVfg9qqBRyCPRUdLMIGJqPzMo+jMRlvE83t14v1j0dRMEaBbER/adQjp6Du2pw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/ts-thomas" + }, + { + "type": "paypal", + "url": "https://www.paypal.com/donate/?hosted_button_id=GEVR88FC9BWRW" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/flexsearch" + }, + { + "type": "patreon", + "url": "https://patreon.com/user?u=96245532" + }, + { + "type": "liberapay", + "url": "https://liberapay.com/ts-thomas" + } + ], + "license": "Apache-2.0" + }, + "node_modules/get-east-asian-width": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.7.0.tgz", + "integrity": "sha512-XjH1AECxf0giL2V1aU8vKyRR2ppRUb5c0EvT7zuJTokQ74bNo52zOtghqdWIqrhUD79fo3x0WfKZdOqxF6LG1Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/hast-util-from-parse5": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/hast-util-from-parse5/-/hast-util-from-parse5-8.0.4.tgz", + "integrity": "sha512-Yq7XlXor0Cpb71sI2E0zpv8QxQ5a6rfhDEETk4bcgrFX2SvJAY2PHocWwFtL6U9qyfiyByW3wZ6sZEfP2kW81g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "devlop": "^1.0.0", + "hastscript": "^9.0.0", + "property-information": "^7.0.0", + "vfile": "^6.0.0", + "vfile-location": "^5.0.0", + "web-namespaces": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-heading-rank": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-heading-rank/-/hast-util-heading-rank-3.0.0.tgz", + "integrity": "sha512-EJKb8oMUXVHcWZTDepnr+WNbfnXKFNf9duMesmr4S8SXTJBJ9M4Yok08pu9vxdJwdlGRhVumk9mEhkEvKGifwA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-is-element": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-is-element/-/hast-util-is-element-3.0.0.tgz", + "integrity": "sha512-Val9mnv2IWpLbNPqc/pUem+a7Ipj2aHacCwgNfTiK0vJKl0LF+4Ba4+v1oPHFpf3bLYmreq0/l3Gud9S5OH42g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-parse-selector": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-4.0.0.tgz", + "integrity": "sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-raw": { + "version": "9.2.0", + "resolved": "https://registry.npmjs.org/hast-util-raw/-/hast-util-raw-9.2.0.tgz", + "integrity": "sha512-1zVq3M9AlZiBZ4mlMPnLtkK2rdd+I8VhmFLNLb//XJm6xA2AVH2tz19EHy25RgRWut6z6FpTARAs7P8rnmRJUQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "@ungap/structured-clone": "^1.0.0", + "hast-util-from-parse5": "^8.0.0", + "hast-util-to-parse5": "^8.0.0", + "html-void-elements": "^3.0.0", + "mdast-util-to-hast": "^13.0.0", + "parse5": "^8.0.0", + "unist-util-position": "^5.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0", + "web-namespaces": "^2.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-estree": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/hast-util-to-estree/-/hast-util-to-estree-3.1.3.tgz", + "integrity": "sha512-48+B/rJWAp0jamNbAAf9M7Uf//UVqAoMmgXhBdxTDJLGKY+LRnZ99qcG+Qjl5HfMpYNzS5v4EAwVEF34LeAj7w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "devlop": "^1.0.0", + "estree-util-attach-comments": "^3.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "hast-util-whitespace": "^3.0.0", + "mdast-util-mdx-expression": "^2.0.0", + "mdast-util-mdx-jsx": "^3.0.0", + "mdast-util-mdxjs-esm": "^2.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "style-to-js": "^1.0.0", + "unist-util-position": "^5.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-html": { + "version": "9.0.6", + "resolved": "https://registry.npmjs.org/hast-util-to-html/-/hast-util-to-html-9.0.6.tgz", + "integrity": "sha512-1SZUxCN3TtJAE7e2BQN2MBHsoZ1gnmi9GoCP5Di59CSWptIezSAW09yDZyVbanb78V9eCwrKqdQsjtfVTxYHWA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "ccount": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-whitespace": "^3.0.0", + "html-void-elements": "^3.0.0", + "mdast-util-to-hast": "^13.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "stringify-entities": "^4.0.0", + "zwitch": "^2.0.4" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-jsx-runtime": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.3.7.tgz", + "integrity": "sha512-F7DNV2NMcTxA4GqPVHnXzX+mpoPHAIY9rJAVH3CZ1FdBZOcBUEjUuvMkMpHVs/sicEtznh9tMGj5WUR9S4E2rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "hast-util-whitespace": "^3.0.0", + "mdast-util-mdx-expression": "^2.0.0", + "mdast-util-mdx-jsx": "^3.0.0", + "mdast-util-mdxjs-esm": "^2.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "style-to-object": "^2.0.0", + "unist-util-position": "^5.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-parse5": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/hast-util-to-parse5/-/hast-util-to-parse5-8.0.1.tgz", + "integrity": "sha512-MlWT6Pjt4CG9lFCjiz4BH7l9wmrMkfkJYCxFwKQic8+RTZgWPuWxwAfjJElsXkex7DJjfSJsQIt931ilUgmwdA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "devlop": "^1.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "web-namespaces": "^2.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-string": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/hast-util-to-string/-/hast-util-to-string-3.0.1.tgz", + "integrity": "sha512-XelQVTDWvqcl3axRfI0xSeoVKzyIFPwsAGSLIsKdJKQMXDYJS4WYrBNF/8J7RdhIcFI2BOHgAifggsvsxp/3+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-whitespace": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz", + "integrity": "sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hastscript": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-9.0.3.tgz", + "integrity": "sha512-1ALZXSmCGmW3h8AMz4R4+QKSKFi7sNgXLVwmwB8FYVtl1CwQ3WDkSAEY/d5tp1g/vbQgq2s1WlHgfO3jbw1TsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-parse-selector": "^4.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hookable": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/hookable/-/hookable-6.1.2.tgz", + "integrity": "sha512-+abwxtiEA52GCVIsQqut3S/uKTbUwYIp4Pe/vv+6py5XiXBCqMZHg6pA6Y5qhgLSEys0/cYuPbO0z1QFj5ZCmg==", + "dev": true, + "license": "MIT" + }, + "node_modules/html-void-elements": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/html-void-elements/-/html-void-elements-3.0.0.tgz", + "integrity": "sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/inline-style-parser": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.2.9.tgz", + "integrity": "sha512-IttN8BqKLxzUrnTqHI+/hwAJQItBAUjdoHERCDv+oOuX48VP6bzW8+QjOXSfoevzq3YrYAwX1uLHp8TqO9yIeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/is-absolute-url": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-absolute-url/-/is-absolute-url-4.0.1.tgz", + "integrity": "sha512-/51/TKE88Lmm7Gc4/8btclNXWS+g50wXhYJq8HWIBAGUBnoAdRu1aXeh364t/O7wXDAcTJDP8PNuNKWUDWie+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-alphabetical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", + "integrity": "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-alphanumerical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz", + "integrity": "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-alphabetical": "^2.0.0", + "is-decimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-decimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", + "integrity": "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-hexadecimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz", + "integrity": "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-plain-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", + "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/longest-streak": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", + "integrity": "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/markdown-extensions": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/markdown-extensions/-/markdown-extensions-2.0.0.tgz", + "integrity": "sha512-o5vL7aDWatOTX8LzaS1WMoaoxIiLRQJuIKKe2wAw6IeULDHaqbiqiggmx+pKvZDb1Sj+pE46Sn1T7lCqfFtg1Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/markdown-table": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.1.0.tgz", + "integrity": "sha512-xpeSCdM+B+6P3ncfgaB7FoDbtJ30xAe6Z18TGT7AgXJsHqCElCICZovQqdRMm7kHgpTyYv36EupqewoFFSB8vw==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/mdast-util-find-and-replace": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.4.tgz", + "integrity": "sha512-z4B5c6jrnJ1ltOQemhWYPBSAo6y42Z8Gt2yEgfgNTMcLSU59B8bfj8Z8AnAlJrBGLnqkdAUZ7jdDEqQw3KuipQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "escape-string-regexp": "^5.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-from-markdown": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.1.0.tgz", + "integrity": "sha512-KRLgmPaP2H1ul2bpoJkOS+9pAcDAAj9Nu1mqLnh8NLeJG5tlvj42TH80PIGBpROWlRXLk0lDrWNuZCFy/tsB4A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark": "^4.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-3.1.0.tgz", + "integrity": "sha512-0ulfdQOM3ysHhCJ1p06l0b0VKlhU0wuQs3thxZQagjcjPrlFRqY215uZGHHJan9GEAXd9MbfPjFJz+qMkVR6zQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-gfm-autolink-literal": "^2.0.0", + "mdast-util-gfm-footnote": "^2.0.0", + "mdast-util-gfm-strikethrough": "^2.0.0", + "mdast-util-gfm-table": "^2.0.0", + "mdast-util-gfm-task-list-item": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-autolink-literal": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-2.1.0.tgz", + "integrity": "sha512-Jx6QSK8wDo558Bcf7q+1pe0v81g2PVs09Hlvu9xJHmCFRF3sAPW7vXFVIAwBncs3sde92WbsVqNxwdqViA2XnQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "ccount": "^2.0.0", + "devlop": "^1.0.0", + "mdast-util-find-and-replace": "^3.0.0", + "micromark-util-character": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-footnote/-/mdast-util-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-sqpDWlsHn7Ac9GNZQMeUzPQSMzR6Wv0WKRNvQRg0KqHh02fpTz69Qc1QSseNX29bhz1ROIyNyxExfawVKTm1GQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.1.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-strikethrough": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-2.0.1.tgz", + "integrity": "sha512-OuJHqvr455pwu2OaOrir7dbzqs4jlWKOtlu9L6GjcIjjQwNyw02VntQsr/Ek6/A13vgnXfUhBekWRUP1OkNivw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-table": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-2.0.1.tgz", + "integrity": "sha512-fiIjPrAy4kz68/eFbPHhsiWDGHhqUSVoU1XsfLM5DsEPXw7MxgiVVyulUggFRckVWeT1KXQXkg5vm3kv4oMSOA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "markdown-table": "^3.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "mdast-util-to-string": "^4.0.0", + "unist-util-visit": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-task-list-item": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-2.0.1.tgz", + "integrity": "sha512-jXVO1hvB2hs82g1esi7I9nIJDqiijbQLIyH2jDUb5s7jIAZMr/79h0VHSq7XHKTqpLARRm+DDNjduB7S9+6XZA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-mdx/-/mdast-util-mdx-3.0.0.tgz", + "integrity": "sha512-JfbYLAW7XnYTTbUsmpu0kdBUVe+yKVJZBItEjwyYJiDJuZ9w4eeaqks4HQO+R7objWgS2ymV60GYpI14Ug554w==", + "dev": true, + "license": "MIT", + "dependencies": { + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-mdx-expression": "^2.0.0", + "mdast-util-mdx-jsx": "^3.0.0", + "mdast-util-mdxjs-esm": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-expression": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-expression/-/mdast-util-mdx-expression-2.0.1.tgz", + "integrity": "sha512-J6f+9hUp+ldTZqKRSg7Vw5V6MqjATc+3E4gf3CFNcuZNWD8XdyI6zQ8GqH7f8169MM6P7hMBRDVGnn7oHB9kXQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-jsx": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-jsx/-/mdast-util-mdx-jsx-3.2.1.tgz", + "integrity": "sha512-oXp1GIa+E9IwIs3a106+tr3Bz2MlTTBvlXjOZMoIat11kk7vjqgmBnksB9WHmWv+gZh7k3IUgpAHlwHB7DH4GQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "ccount": "^2.0.0", + "devlop": "^1.1.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "parse-entities": "^4.0.0", + "stringify-entities": "^4.0.0", + "unist-util-stringify-position": "^4.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdxjs-esm": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdxjs-esm/-/mdast-util-mdxjs-esm-2.0.1.tgz", + "integrity": "sha512-EcmOpxsZ96CvlP03NghtH1EsLtr0n9Tm4lPUJUBccV9RwUOneqSycg19n5HGzCf+10LozMRSObtVr3ee1WoHtg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-phrasing": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-4.1.0.tgz", + "integrity": "sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-hast": { + "version": "13.2.1", + "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-13.2.1.tgz", + "integrity": "sha512-cctsq2wp5vTsLIcaymblUriiTcZd0CwWtCbLvrOzYCDZoWyMNV8sZ7krj09FSnsiJi3WVsHLM4k6Dq/yaPyCXA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@ungap/structured-clone": "^1.0.0", + "devlop": "^1.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "trim-lines": "^3.0.0", + "unist-util-position": "^5.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-markdown": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-2.2.3.tgz", + "integrity": "sha512-dImdB6VZRMRac1SwAW6o1c0ESk/7pFFAMCMuKM+9v7LpTSthMM5nzXpqKw4Mjo66aXVKhNkyrP4bMB9MurPyWg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "longest-streak": "^3.0.0", + "mdast-util-phrasing": "^4.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "unist-util-visit": "^5.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-markdown-cjk-friendly": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-to-markdown-cjk-friendly/-/mdast-util-to-markdown-cjk-friendly-1.0.2.tgz", + "integrity": "sha512-1HCuZwTxQX3WfaNFk8PDb/VW8Ulvzn+nHD+W79Q9k5NXUSkGXH9Zv3eMJ4VI7GjZIz4Ff7WbruEFgMUaPmRJ8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "mdast-util-to-markdown": "^2.1.3", + "micromark-extension-cjk-friendly-util": "3.0.1", + "micromark-util-character": "^2.1.1", + "micromark-util-classify-character": "^2.0.1", + "micromark-util-html-tag-name": "^2.0.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/mdast": "*" + }, + "peerDependenciesMeta": { + "@types/mdast": { + "optional": true + } + } + }, + "node_modules/mdast-util-to-markdown-cjk-friendly-gfm-strikethrough": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-to-markdown-cjk-friendly-gfm-strikethrough/-/mdast-util-to-markdown-cjk-friendly-gfm-strikethrough-1.0.2.tgz", + "integrity": "sha512-aidUweNq42Mok1p6KTJG7m0Ds0+njX3Sp6shhh3/VhwIkyWdn2SqEzkeiwJsK+Ksz0pL6vZUHrVFgFtyX9NMYg==", + "dev": true, + "license": "MIT", + "dependencies": { + "mdast-util-gfm-strikethrough": "^2.0.1", + "mdast-util-to-markdown": "^2.1.3", + "mdast-util-to-markdown-cjk-friendly": "1.0.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/mdast": "*" + }, + "peerDependenciesMeta": { + "@types/mdast": { + "optional": true + } + } + }, + "node_modules/mdast-util-to-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-4.0.0.tgz", + "integrity": "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/medium-zoom": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/medium-zoom/-/medium-zoom-1.1.2.tgz", + "integrity": "sha512-0wTqeCQ4NGzjPghi7i2lMI6RyGj9N60fGvzY9HiNcxDTcvOw7S+ri7K3xVkrIZ+2uzUg4w/KSVMDdZXYMtHkdw==", + "dev": true, + "license": "MIT", + "workspaces": [ + "website", + "examples/*" + ] + }, + "node_modules/micromark": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.3.tgz", + "integrity": "sha512-oGYfQzHSG5dOMovQcJ3fyTmZlWAWpi0XA0sJwJs+i6OT88o1+Jtw/8z0CmdowSLxGhhFK89rQ/oXph/wN02PNw==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-edit-map": "^1.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.4.tgz", + "integrity": "sha512-wxEeE8v8XVvOrxn1TZj74qYhAtTQsSqufHVS3uNVyT1MupXxhyaHk8/9xDNPh9BmL77QNDgWTkZ3RsYHB2ry7w==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-destination": "^2.0.0", + "micromark-factory-label": "^2.0.0", + "micromark-factory-space": "^2.1.0", + "micromark-factory-title": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-edit-map": "^1.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.3" + } + }, + "node_modules/micromark-extension-cjk-friendly": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-extension-cjk-friendly/-/micromark-extension-cjk-friendly-2.0.2.tgz", + "integrity": "sha512-/BqZNCkDWg3Hj9zEEcRitZ5uMF3X78qmBFuP17oAjnVDZ0+H1GHiDKMJZDpJsXasCHEn/kzl6QHozUiWgCHtPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "devlop": "^1.1.0", + "micromark-extension-cjk-friendly-util": "3.0.1", + "micromark-util-chunked": "^2.0.1", + "micromark-util-resolve-all": "^2.0.1", + "micromark-util-symbol": "^2.0.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "micromark": "^4.0.0", + "micromark-util-types": "^2.0.0" + }, + "peerDependenciesMeta": { + "micromark-util-types": { + "optional": true + } + } + }, + "node_modules/micromark-extension-cjk-friendly-gfm-strikethrough": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-extension-cjk-friendly-gfm-strikethrough/-/micromark-extension-cjk-friendly-gfm-strikethrough-2.0.1.tgz", + "integrity": "sha512-wVC0zwjJNqQeX+bb07YTPu/CvSAyCTafyYb7sMhX1r62/Lw5M/df3JyYaANyp8g15c1ypJRFSsookTqA1IDsUg==", + "dev": true, + "license": "MIT", + "dependencies": { + "devlop": "^1.1.0", + "get-east-asian-width": "^1.4.0", + "micromark-extension-cjk-friendly-util": "3.0.1", + "micromark-util-character": "^2.1.1", + "micromark-util-chunked": "^2.0.1", + "micromark-util-resolve-all": "^2.0.1", + "micromark-util-symbol": "^2.0.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "micromark": "^4.0.0", + "micromark-util-types": "^2.0.0" + }, + "peerDependenciesMeta": { + "micromark-util-types": { + "optional": true + } + } + }, + "node_modules/micromark-extension-cjk-friendly-util": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/micromark-extension-cjk-friendly-util/-/micromark-extension-cjk-friendly-util-3.0.1.tgz", + "integrity": "sha512-GcbXqTTHOsiZHyF753oIddP/J2eH8j9zpyQPhkof6B2JNxfEJabnQqxbCgzJNuNes0Y2jTNJ3LiYPSXr6eJA8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.4.0", + "micromark-util-character": "^2.1.1", + "micromark-util-symbol": "^2.0.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependenciesMeta": { + "micromark-util-types": { + "optional": true + } + } + }, + "node_modules/micromark-extension-gfm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-3.0.0.tgz", + "integrity": "sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w==", + "dev": true, + "license": "MIT", + "dependencies": { + "micromark-extension-gfm-autolink-literal": "^2.0.0", + "micromark-extension-gfm-footnote": "^2.0.0", + "micromark-extension-gfm-strikethrough": "^2.0.0", + "micromark-extension-gfm-table": "^2.0.0", + "micromark-extension-gfm-tagfilter": "^2.0.0", + "micromark-extension-gfm-task-list-item": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-autolink-literal": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-2.1.0.tgz", + "integrity": "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==", + "dev": true, + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw==", + "dev": true, + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-strikethrough": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-2.1.0.tgz", + "integrity": "sha512-ADVjpOOkjz1hhkZLlBiYA9cR2Anf8F4HqZUO6e5eDcPQd0Txw5fxLzzxnEkSkfnD0wziSGiv7sYhk/ktvbf1uw==", + "dev": true, + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-table": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-2.1.2.tgz", + "integrity": "sha512-pRzm4kDTu0MjlmBkxmS9yYhw60nncfcEwu9NNdPFSQEFXS95ZKyIIyTSHu/o3ReBUrLKYEq+7YaXCRn/bPB4MA==", + "dev": true, + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-tagfilter": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-2.0.0.tgz", + "integrity": "sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg==", + "dev": true, + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-task-list-item": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-2.1.0.tgz", + "integrity": "sha512-qIBZhqxqI6fjLDYFTBIa4eivDMnP+OZqsNwmQ3xNLE4Cxwc+zfQEfbs6tzAo2Hjq+bh6q5F+Z8/cksrLFYWQQw==", + "dev": true, + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdx-expression": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-expression/-/micromark-extension-mdx-expression-3.0.1.tgz", + "integrity": "sha512-dD/ADLJ1AeMvSAKBwO22zG22N4ybhe7kFIZ3LsDI0GlsNr2A3KYxb0LdC1u5rj4Nw+CHKY0RVdnHX8vj8ejm4Q==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-mdx-expression": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-mdx-jsx": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-jsx/-/micromark-extension-mdx-jsx-3.0.2.tgz", + "integrity": "sha512-e5+q1DjMh62LZAJOnDraSSbDMvGJ8x3cbjygy2qFEi7HCeUT4BDKCvMozPozcD6WmOt6sVvYDNBKhFSz3kjOVQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "micromark-factory-mdx-expression": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdx-md": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-md/-/micromark-extension-mdx-md-2.0.0.tgz", + "integrity": "sha512-EpAiszsB3blw4Rpba7xTOUptcFeBFi+6PY8VnJ2hhimH+vCQDirWgsMpz7w1XcZE7LVrSAUGb9VJpG9ghlYvYQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdxjs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-mdxjs/-/micromark-extension-mdxjs-3.0.0.tgz", + "integrity": "sha512-A873fJfhnJ2siZyUrJ31l34Uqwy4xIFmvPY1oj+Ean5PHcPBYzEsvqvWGaWcfEIr11O5Dlw3p2y0tZWpKHDejQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn": "^8.0.0", + "acorn-jsx": "^5.0.0", + "micromark-extension-mdx-expression": "^3.0.0", + "micromark-extension-mdx-jsx": "^3.0.0", + "micromark-extension-mdx-md": "^2.0.0", + "micromark-extension-mdxjs-esm": "^3.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdxjs-esm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-mdxjs-esm/-/micromark-extension-mdxjs-esm-3.0.0.tgz", + "integrity": "sha512-DJFl4ZqkErRpq/dAPyeWp15tGrcrrJho1hKK5uBS70BCtfrIFg81sqcTVu3Ta+KD1Tk5vAtBNElWxtAa+m8K9A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-position-from-estree": "^2.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-factory-destination": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz", + "integrity": "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.1.tgz", + "integrity": "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-mdx-expression": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-factory-mdx-expression/-/micromark-factory-mdx-expression-2.0.3.tgz", + "integrity": "sha512-kQnEtA3vzucU2BkrIa8/VaSAsP+EJ3CKOvhMuJgOEGg9KDC6OAY6nSnNDVRiVNRqj7Y4SlSzcStaH/5jge8JdQ==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-position-from-estree": "^2.0.0", + "vfile-message": "^4.0.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.1.0.tgz", + "integrity": "sha512-fS8hnLIjnjvdQIj39Geug8wWsR0HrYZ43KShKxNfwT7t2LOHo/LbWZEzEaSOjwtjdCsjoE6syHhonEzW0zv0+Q==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.1.tgz", + "integrity": "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.1.tgz", + "integrity": "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.1.tgz", + "integrity": "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.1.tgz", + "integrity": "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.1.tgz", + "integrity": "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-chunked": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.2.tgz", + "integrity": "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-string": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-2.0.1.tgz", + "integrity": "sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-edit-map": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-edit-map/-/micromark-util-edit-map-1.0.0.tgz", + "integrity": "sha512-Pa2ljlsEL6sVwFaYeyrOLSYbQt73JvGbPOYFq+9AElXiSnfI5Q4465RRZ1sHv7lDjDOnGRDDaqPXqClqWK/q1Q==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", + "integrity": "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-events-to-acorn": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-util-events-to-acorn/-/micromark-util-events-to-acorn-2.0.3.tgz", + "integrity": "sha512-jmsiEIiZ1n7X1Rr5k8wVExBQCg5jy4UXVADItHmNk1zkwEVhBuIUKRu3fqv+hs4nxLISi2DQGlqIOGiFxgbfHg==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/unist": "^3.0.0", + "devlop": "^1.0.0", + "estree-util-visit": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "vfile-message": "^4.0.0" + } + }, + "node_modules/micromark-util-html-tag-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.1.tgz", + "integrity": "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.1.tgz", + "integrity": "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.1.tgz", + "integrity": "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", + "integrity": "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.1.0.tgz", + "integrity": "sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-types": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.3.tgz", + "integrity": "sha512-oxB2Ik03hI0gv+VNn9tnh1t1YEe9MDPptViAEgfdf3YQHsn0pzGTgCdlSCJXcwhqm8phaEuM7zeEu3QQzVBrPg==", + "dev": true, + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nprogress": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/nprogress/-/nprogress-0.2.0.tgz", + "integrity": "sha512-I19aIingLgR1fmhftnbWWO3dXc0hSxqHQHQb3H8m+K3TnEn/iSeTZZOyvKXWqQESMwuUVnatlCnZdLBZZt2VSA==", + "dev": true, + "license": "MIT" + }, + "node_modules/oniguruma-parser": { + "version": "0.12.2", + "resolved": "https://registry.npmjs.org/oniguruma-parser/-/oniguruma-parser-0.12.2.tgz", + "integrity": "sha512-6HVa5oIrgMC6aA6WF6XyyqbhRPJrKR02L20+2+zpDtO5QAzGHAUGw5TKQvwi5vctNnRHkJYmjAhRVQF2EKdTQw==", + "dev": true, + "license": "MIT" + }, + "node_modules/oniguruma-to-es": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/oniguruma-to-es/-/oniguruma-to-es-4.3.6.tgz", + "integrity": "sha512-csuQ9x3Yr0cEIs/Zgx/OEt9iBw9vqIunAPQkx19R/fiMq2oGVTgcMqO/V3Ybqefr1TBvosI6jU539ksaBULJyA==", + "dev": true, + "license": "MIT", + "dependencies": { + "oniguruma-parser": "^0.12.2", + "regex": "^6.1.0", + "regex-recursion": "^6.0.2" + } + }, + "node_modules/parse-entities": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz", + "integrity": "sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.0", + "character-entities-legacy": "^3.0.0", + "character-reference-invalid": "^2.0.0", + "decode-named-character-reference": "^1.0.0", + "is-alphanumerical": "^2.0.0", + "is-decimal": "^2.0.0", + "is-hexadecimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/parse-entities/node_modules/@types/unist": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", + "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", + "dev": true, + "license": "MIT" + }, + "node_modules/parse5": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.1.0.tgz", + "integrity": "sha512-sq3qBFpee5jeocbFbO48VHlGr8sT7mbXS0vxlKN6sottBtJzrOG3998sr0llSZcQAGgfpdmEYAIZE0P9z0dE8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "entities": "^9.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/property-information": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-7.2.0.tgz", + "integrity": "sha512-IAtzIB6sUiWaJYrX9smp3V46pBGbBeLFRGdh25kg1334VcBlD8HzhPeNIWQH9zhGmo2itIe25EHt9dQP7G5hmg==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/react": { + "version": "19.3.1", + "resolved": "https://registry.npmjs.org/react/-/react-19.3.1.tgz", + "integrity": "sha512-QNhWdt7nrgVWWQG58QuGsAR3jv3Hbe4mb4vz7w2csnHavciZJgda1CPNaldF6eVxv90ud3jUXpKlHL69gO38yw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.3.1", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.3.1.tgz", + "integrity": "sha512-Y0bARz53lhEC1NMCASs0/j+faB/zqt3R8hpvg1e8+61ZVJ4UL0UdC4cBbXfZMcIuhrfgeILTLe+ZeBCavQPtKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "scheduler": "^0.28.0" + }, + "peerDependencies": { + "react": "^19.3.1" + } + }, + "node_modules/react-lazy-with-preload": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/react-lazy-with-preload/-/react-lazy-with-preload-2.2.1.tgz", + "integrity": "sha512-ONSb8gizLE5jFpdHAclZ6EAAKuFX2JydnFXPPPjoUImZlLjGtKzyBS8SJgJq7CpLgsGKh9QCZdugJyEEOVC16Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/react-reconciler": { + "version": "0.33.0", + "resolved": "https://registry.npmjs.org/react-reconciler/-/react-reconciler-0.33.0.tgz", + "integrity": "sha512-KetWRytFv1epdpJc3J4G75I4WrplZE5jOL7Yq0p34+OVOKF4Se7WrdIdVC45XsSSmUTlht2FM/fM1FZb1mfQeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "scheduler": "^0.27.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "peerDependencies": { + "react": "^19.2.0" + } + }, + "node_modules/react-reconciler/node_modules/scheduler": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", + "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/react-refresh": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.19.0.tgz", + "integrity": "sha512-5HdstYgoJBbBofjapO5qCPcovECJCBfvjLb+iTcsmqDFVMRdqy0qs7iiInBJTUXmIFc2bjoc5b2HVLp2zRtYhQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-render-to-markdown": { + "version": "19.1.0", + "resolved": "https://registry.npmjs.org/react-render-to-markdown/-/react-render-to-markdown-19.1.0.tgz", + "integrity": "sha512-dF9b3tO41ezqdmHP8X92kbHbMexJ6iC7iHw4ykC8fwiO7DgpFc9PhMoKlI+BcPzRxGcWgQSdrixVB9RykhjJpQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "react-reconciler": "0.33.0" + }, + "peerDependencies": { + "react": ">=19" + } + }, + "node_modules/react-router": { + "version": "7.18.5", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.18.5.tgz", + "integrity": "sha512-dTgvcXDOrLPrlLHrcZAqWveiK1hMSqN+rGAJzegP8HtmNkfRxOoshGWBNX3ZJ0cZjOHkRdlUJtaIhZMLLyGyPw==", + "dev": true, + "license": "MIT", + "dependencies": { + "cookie": "^1.0.1", + "set-cookie-parser": "^2.6.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "react": ">=18", + "react-dom": ">=18" + }, + "peerDependenciesMeta": { + "react-dom": { + "optional": true + } + } + }, + "node_modules/react-router-dom": { + "version": "7.18.5", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-7.18.5.tgz", + "integrity": "sha512-9HcfDMU6JJ/fp6gJZYGwNE6RknVGS4kBjH1HkNw959mCJi70TVwdipQWv2FEol9uGpi+SZTFtVpJPtEQfayeOw==", + "dev": true, + "license": "MIT", + "dependencies": { + "react-router": "7.18.5" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "react": ">=18", + "react-dom": ">=18" + } + }, + "node_modules/recma-build-jsx": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/recma-build-jsx/-/recma-build-jsx-1.0.0.tgz", + "integrity": "sha512-8GtdyqaBcDfva+GUKDr3nev3VpKAhup1+RvkMvUxURHpW7QyIvk9F5wz7Vzo06CEMSilw6uArgRqhpiUcWp8ew==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "estree-util-build-jsx": "^3.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/recma-jsx": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/recma-jsx/-/recma-jsx-1.0.1.tgz", + "integrity": "sha512-huSIy7VU2Z5OLv6oFLosQGGDqPqdO1iq6bWNAdhzMxSJP7RAso4fCZ1cKu8j9YHCZf3TPrq4dw3okhrylgcd7w==", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn-jsx": "^5.0.0", + "estree-util-to-js": "^2.0.0", + "recma-parse": "^1.0.0", + "recma-stringify": "^1.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + }, + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/recma-parse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/recma-parse/-/recma-parse-1.0.0.tgz", + "integrity": "sha512-OYLsIGBB5Y5wjnSnQW6t3Xg7q3fQ7FWbw/vcXtORTnyaSFscOtABg+7Pnz6YZ6c27fG1/aN8CjfwoUEUIdwqWQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "esast-util-from-js": "^2.0.0", + "unified": "^11.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/recma-stringify": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/recma-stringify/-/recma-stringify-1.0.0.tgz", + "integrity": "sha512-cjwII1MdIIVloKvC9ErQ+OgAtwHBmcZ0Bg4ciz78FtbT8In39aAYbaA7zvxQ61xVMSPE8WxhLwLbhif4Js2C+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "estree-util-to-js": "^2.0.0", + "unified": "^11.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/regex": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/regex/-/regex-6.1.0.tgz", + "integrity": "sha512-6VwtthbV4o/7+OaAF9I5L5V3llLEsoPyq9P1JVXkedTP33c7MfCG0/5NOPcSJn0TzXcG9YUrR0gQSWioew3LDg==", + "dev": true, + "license": "MIT", + "dependencies": { + "regex-utilities": "^2.3.0" + } + }, + "node_modules/regex-recursion": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/regex-recursion/-/regex-recursion-6.0.2.tgz", + "integrity": "sha512-0YCaSCq2VRIebiaUviZNs0cBz1kg5kVS2UKUfNIx8YVs1cN3AV7NTctO5FOKBA+UT2BPJIWZauYHPqJODG50cg==", + "dev": true, + "license": "MIT", + "dependencies": { + "regex-utilities": "^2.3.0" + } + }, + "node_modules/regex-utilities": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/regex-utilities/-/regex-utilities-2.3.0.tgz", + "integrity": "sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng==", + "dev": true, + "license": "MIT" + }, + "node_modules/rehype-external-links": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/rehype-external-links/-/rehype-external-links-3.0.0.tgz", + "integrity": "sha512-yp+e5N9V3C6bwBeAC4n796kc86M4gJCdlVhiMTxIrJG5UHDMh+PJANf9heqORJbt1nrCbDwIlAZKjANIaVBbvw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@ungap/structured-clone": "^1.0.0", + "hast-util-is-element": "^3.0.0", + "is-absolute-url": "^4.0.0", + "space-separated-tokens": "^2.0.0", + "unist-util-visit": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-raw": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/rehype-raw/-/rehype-raw-7.0.0.tgz", + "integrity": "sha512-/aE8hCfKlQeA8LmyeyQvQF3eBiLRGNlfBJEvWH7ivp9sBqs7TNqBL5X3v157rM4IFETqDnIOO+z5M/biZbo9Ww==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-raw": "^9.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-recma": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/rehype-recma/-/rehype-recma-1.0.0.tgz", + "integrity": "sha512-lqA4rGUf1JmacCNWWZx0Wv1dHqMwxzsDWYMTowuplHF3xH0N/MmrZ/G3BDZnzAkRmxDadujCjaKM2hqYdCBOGw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/hast": "^3.0.0", + "hast-util-to-estree": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-cjk-friendly": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/remark-cjk-friendly/-/remark-cjk-friendly-2.3.4.tgz", + "integrity": "sha512-xhBzjXkumeN5tKXXg6k/THSCZCP6AYV6+wSxnCsPTkxHt60bb9Yk0IWhdD87CFaDCiTqd86LKHtVObcJTmgcow==", + "dev": true, + "license": "MIT", + "dependencies": { + "mdast-util-to-markdown-cjk-friendly": "1.0.2", + "micromark-extension-cjk-friendly": "2.0.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/mdast": "^4.0.0", + "unified": "^11.0.0" + }, + "peerDependenciesMeta": { + "@types/mdast": { + "optional": true + } + } + }, + "node_modules/remark-cjk-friendly-gfm-strikethrough": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/remark-cjk-friendly-gfm-strikethrough/-/remark-cjk-friendly-gfm-strikethrough-2.3.3.tgz", + "integrity": "sha512-UBujdRd1TTgr79iPdXhCnB9ag7kaM54FBCzTa/jLCNfZFGPyZCZb8G6yoCA0k4M3HTKDvilIVKvPxU6K/05dDw==", + "dev": true, + "license": "MIT", + "dependencies": { + "mdast-util-to-markdown-cjk-friendly-gfm-strikethrough": "1.0.2", + "micromark-extension-cjk-friendly-gfm-strikethrough": "2.0.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/mdast": "^4.0.0", + "unified": "^11.0.0" + }, + "peerDependenciesMeta": { + "@types/mdast": { + "optional": true + } + } + }, + "node_modules/remark-gfm": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-4.0.1.tgz", + "integrity": "sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-gfm": "^3.0.0", + "micromark-extension-gfm": "^3.0.0", + "remark-parse": "^11.0.0", + "remark-stringify": "^11.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-mdx": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/remark-mdx/-/remark-mdx-3.1.1.tgz", + "integrity": "sha512-Pjj2IYlUY3+D8x00UJsIOg5BEvfMyeI+2uLPn9VO9Wg4MEtN/VTIq2NEJQfde9PnX15KgtHyl9S0BcTnWrIuWg==", + "dev": true, + "license": "MIT", + "dependencies": { + "mdast-util-mdx": "^3.0.0", + "micromark-extension-mdxjs": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-parse": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz", + "integrity": "sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-rehype": { + "version": "11.1.2", + "resolved": "https://registry.npmjs.org/remark-rehype/-/remark-rehype-11.1.2.tgz", + "integrity": "sha512-Dh7l57ianaEoIpzbp0PC9UKAdCSVklD8E5Rpw7ETfbTl3FqcOOgq5q2LVDhgGCkaBv7p24JXikPdvhhmHvKMsw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "mdast-util-to-hast": "^13.0.0", + "unified": "^11.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-stringify": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-11.0.0.tgz", + "integrity": "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-to-markdown": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/scheduler": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.28.0.tgz", + "integrity": "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw==", + "dev": true, + "license": "MIT" + }, + "node_modules/scroll-into-view-if-needed": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/scroll-into-view-if-needed/-/scroll-into-view-if-needed-3.1.0.tgz", + "integrity": "sha512-49oNpRjWRvnU8NyGVmUaYG4jtTkNonFZI86MmGRDqBphEK2EXT9gdEUoQPZhuBM8yWHxCWbobltqYO5M4XrUvQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "compute-scroll-into-view": "^3.0.2" + } + }, + "node_modules/set-cookie-parser": { + "version": "2.7.3", + "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.3.tgz", + "integrity": "sha512-irZHtTqurLna57KQG/s+fzOwv/J6XSH4cs8M/vsrY3uCaQ3S6+QJguAH0WdN47SXlk82voMZ4naWMQf8fdEoIg==", + "dev": true, + "license": "MIT" + }, + "node_modules/shiki": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/shiki/-/shiki-4.6.0.tgz", + "integrity": "sha512-imjLsVHgPuWhvpU3P+zOekvMYqnel7QmKsFC4Sn0fCuAvaaalBsxwKEsrU8Ko5BdFnZA6Go/8a0RzahS634UgA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@shikijs/core": "4.6.0", + "@shikijs/engine-javascript": "4.6.0", + "@shikijs/engine-oniguruma": "4.6.0", + "@shikijs/langs": "4.6.0", + "@shikijs/themes": "4.6.0", + "@shikijs/types": "4.6.0", + "@shikijs/vscode-textmate": "^10.1.0", + "@types/hast": "^3.0.5" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/source-map": { + "version": "0.7.6", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.6.tgz", + "integrity": "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">= 12" + } + }, + "node_modules/space-separated-tokens": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz", + "integrity": "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/stringify-entities": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.4.tgz", + "integrity": "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==", + "dev": true, + "license": "MIT", + "dependencies": { + "character-entities-html4": "^2.0.0", + "character-entities-legacy": "^3.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/style-to-js": { + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/style-to-js/-/style-to-js-1.1.21.tgz", + "integrity": "sha512-RjQetxJrrUJLQPHbLku6U/ocGtzyjbJMP9lCNK7Ag0CNh690nSH8woqWH9u16nMjYBAok+i7JO1NP2pOy8IsPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "style-to-object": "1.0.14" + } + }, + "node_modules/style-to-js/node_modules/inline-style-parser": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.2.7.tgz", + "integrity": "sha512-Nb2ctOyNR8DqQoR0OwRG95uNWIC0C1lCgf5Naz5H6Ji72KZ8OcFZLz2P5sNgwlyoJ8Yif11oMuYs5pBQa86csA==", + "dev": true, + "license": "MIT" + }, + "node_modules/style-to-js/node_modules/style-to-object": { + "version": "1.0.14", + "resolved": "https://registry.npmjs.org/style-to-object/-/style-to-object-1.0.14.tgz", + "integrity": "sha512-LIN7rULI0jBscWQYaSswptyderlarFkjQ+t79nzty8tcIAceVomEVlLzH5VP4Cmsv6MtKhs7qaAiwlcp+Mgaxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "inline-style-parser": "0.2.7" + } + }, + "node_modules/style-to-object": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/style-to-object/-/style-to-object-2.0.2.tgz", + "integrity": "sha512-QmbXPUylqU80HAgTOJeEOdbA40W9KtxmjMhQy+tq+b8F2aoh3aQLuy82hh2jm2qsC9hR9eI0+eQidTMydKcqRA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inline-style-parser": "0.2.9" + } + }, + "node_modules/toggle-selection": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/toggle-selection/-/toggle-selection-1.0.6.tgz", + "integrity": "sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/trim-lines": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", + "integrity": "sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/trough": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/trough/-/trough-2.2.0.tgz", + "integrity": "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/tslib": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.9.1.tgz", + "integrity": "sha512-a4Ux9VKuIMHsOS7ENbEq6hK2ZGYIT27Rtl0E4eD9KqwxhKiQeMZE/9xb3IdYwWaSOyIBQLExhQgXDTYoiCxnWA==", + "dev": true, + "license": "0BSD", + "optional": true + }, + "node_modules/unhead": { + "version": "2.1.17", + "resolved": "https://registry.npmjs.org/unhead/-/unhead-2.1.17.tgz", + "integrity": "sha512-HLMKXOszRhAPBrr6VlqCeVeJq2kbC4kXwzGLEZvvojPLWNYTJw22xG7Bfwhsvs31+IBet3Wl8ADg9dwYdyphfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "hookable": "^6.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/harlan-zw" + } + }, + "node_modules/unified": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", + "integrity": "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "bail": "^2.0.0", + "devlop": "^1.0.0", + "extend": "^3.0.0", + "is-plain-obj": "^4.0.0", + "trough": "^2.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-is": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.1.tgz", + "integrity": "sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-position": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-position/-/unist-util-position-5.0.0.tgz", + "integrity": "sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-position-from-estree": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unist-util-position-from-estree/-/unist-util-position-from-estree-2.0.0.tgz", + "integrity": "sha512-KaFVRjoqLyF6YXCbVLNad/eS4+OfPQQn2yOd7zF/h5T/CSL2v8NpN6a5TPvtbXthAGw5nG+PuTtq+DdIZr+cRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-stringify-position": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", + "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-5.2.0.tgz", + "integrity": "sha512-fpsYs/VVoZxfGY9/j3FQpZmN8wdLJaV1Pt38AaJNZu9uKIHcFU3halCdBveEq1HGKO1OEfJY65w1zvoFV1b3qQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-children": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/unist-util-visit-children/-/unist-util-visit-children-3.0.0.tgz", + "integrity": "sha512-RgmdTfSBOg04sdPcpTSD1jzoNBjt9a80/ZCzp5cI9n1qPzLZWF9YdvWGN2zmTumP1HWhXKdUWexjy/Wy/lJ7tA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-parents": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-6.1.0.tgz", + "integrity": "sha512-vKXxz6O38sKDj1mtx/k2cWvkkFBuYGx6uGrlzBkVeWCRF/F8sdI0j3Wbkz/5rVNRRuVc8UrgRdbCxNhmYXgP3w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", + "integrity": "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-location": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-5.0.4.tgz", + "integrity": "sha512-vEvxatYZCIpNvzGk2UD+u4I/sZkSKoOsdRCbkOIsN73QpJbriJ1qvvBAL1PeI0y55Leoqf7pORbaSPpG/smmOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-message": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.4.tgz", + "integrity": "sha512-66ycRswIkNmHH9PHku7ADtv5RVwgIrOcuv3J8Svz6h4qgGRrKVa+9/neuKyJZUXyOpYwLTvS+whZ4R2urHHgQg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/web-namespaces": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/web-namespaces/-/web-namespaces-2.0.1.tgz", + "integrity": "sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/zwitch": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", + "integrity": "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + } + } +} diff --git a/docs/package.json b/docs/package.json new file mode 100644 index 0000000..3aeecf4 --- /dev/null +++ b/docs/package.json @@ -0,0 +1,15 @@ +{ + "name": "@flanksource/telltale-docs", + "private": true, + "type": "module", + "scripts": { + "dev": "rspress dev", + "build": "npm run check:catalog && rspress build", + "check:catalog": "go -C .. run ./hack/docs-catalog --check", + "generate:catalog": "go -C .. run ./hack/docs-catalog", + "preview": "rspress preview" + }, + "devDependencies": { + "@rspress/core": "2.1.0" + } +} diff --git a/docs/replay.md b/docs/replay.md new file mode 100644 index 0000000..0d0f901 --- /dev/null +++ b/docs/replay.md @@ -0,0 +1,19 @@ +# Replay recorded traces + +`telltale trace replay` reads the saved captures of one or more trace/run IDs. It never executes the recorded commands. Find IDs with `telltale trace list`, then pass them as positional arguments: + +```sh +telltale trace list +telltale trace replay buildhost20261107T123000Z +telltale trace replay buildhost20261107T123000Z buildhost20261107T123000Z_2 --ui +telltale trace replay buildhost20261107T123000Z --tui +telltale trace replay buildhost20261107T123000Z --format ndjson --sort timestamp +``` + +Replace the example IDs with ones from your data directory. Every form accepts `--data-dir` when the recordings live outside the default cache directory. Unknown IDs, duplicate selections, invalid paths and recordings without a readable manifest fail before any records are printed. + +Terminal output reads every page, merging captures by `--sort seq` (the default) or `--sort timestamp`. Sequence numbers belong to individual streams. NDJSON records include `runId`, `kind`, `stream`, `seq`, `timestamp`, `summary` and the original payload under `record`. `--follow` reads and follows selected captures until their streams seal; live arrivals preserve each stream's order. + +`--ui` opens a loopback viewer restricted to the selected runs and serves until Ctrl+C. A **Run** selector switches between multiple recordings and retains the selection in the URL. Kind-specific rendering opens in **Captures**; **Raw stream** opens the record table and JSON details on demand. `--tui` opens the full-screen kind monitors and separate stdout/stderr tabs. Tab/Shift+Tab switches kinds; `[` and `]` switch selected runs, retaining each run's tab state. Older combined recordings display their saved output in Stdout and explicitly report that separate stderr is unavailable. Attached processes without recorded output show an unavailable message. Choose one of `--ui` and `--tui`. + +New trace IDs concatenate up to nine characters from the short hostname with a compact UTC timestamp: `buildhost20261107T123000Z`. Hostname dashes are removed, and the timestamp contains no dashes. Runs created on the same host in the same second receive `_2`, `_3`, and subsequent suffixes. Existing recordings keep their IDs and remain selectable. Listing and retention use recorded start times across hostnames and ID formats; stores without a manifest use their directory modification time for retention. diff --git a/docs/rspress.config.ts b/docs/rspress.config.ts new file mode 100644 index 0000000..2ecaffe --- /dev/null +++ b/docs/rspress.config.ts @@ -0,0 +1,68 @@ +import { defineConfig } from '@rspress/core'; + +export default defineConfig({ + root: '.', + title: 'Telltale', + description: 'Trace applications and inspect the processes running on your host.', + outDir: 'doc_build', + route: { + extensions: ['.md', '.mdx'], + exclude: ['plans/**', 'doc_build/**'], + }, + builderConfig: { + plugins: [{ + name: 'telltale-docs-watch', + setup(api) { + api.modifyRsbuildConfig((config) => { + const watches = config.dev?.watchFiles; + if (!watches) return; + for (const watch of Array.isArray(watches) ? watches : [watches]) { + if (watch.type !== 'restart') continue; + const ignored = watch.options?.ignored; + watch.options = { + ...watch.options, + ignored: [ + ...(ignored ? (Array.isArray(ignored) ? ignored : [ignored]) : []), + /(^|[/\\])(node_modules|\.git|\.rspress|doc_build|plans)([/\\]|$)/, + ], + }; + } + }); + }, + }], + }, + themeConfig: { + nav: [ + { text: 'Guide', link: '/getting-started' }, + { text: 'GitHub', link: 'https://github.com/flanksource/telltale' }, + ], + sidebar: { + '/': [ + { + text: 'Get started', + items: [ + { text: 'Introduction', link: '/' }, + { text: 'Quick start', link: '/getting-started' }, + ], + }, + { + text: 'Guides', + items: [ + { text: 'Web workspace', link: '/web-ui' }, + { text: 'OpenTelemetry receiver', link: '/otlp-receiver' }, + { text: 'Active operation profiles', link: '/traces/profiles' }, + { text: 'Resource monitoring rules', link: '/rules' }, + { text: 'SQL Server and JVM captures', link: '/sqlserver-jvm' }, + ], + }, + { + text: 'Trace kinds', + items: [ + { text: 'Overview', link: '/traces/' }, + ...['exec', 'goprof', 'http', 'http_top', 'jms', 'jvm_jfr', 'llm', 'logs', 'process', 'prometheus', 'query', 'spans', 'sql', 'sql_deadlock', 'sql_plancache', 'sql_top', 'sql_xevent', 'stacks', 'terminal'].map((kind) => ({ text: kind, link: `/traces/${kind}` })), + ], + }, + ], + }, + }, +}); diff --git a/docs/rules.md b/docs/rules.md index 091721a..d196923 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -94,7 +94,7 @@ timeout: 5s ## Concurrent browser, test and build jobs -[The concurrency example](../examples/concurrency-rules.yaml) uses `groupBy: host` to detect excessive independent jobs across repositories: +[The concurrency example](https://github.com/flanksource/telltale/blob/main/examples/concurrency-rules.yaml) uses `groupBy: host` to detect excessive independent jobs across repositories: | Rule | Allowed concurrent jobs | Counted roots | | --- | --- | --- | diff --git a/docs/sqlserver-jvm.md b/docs/sqlserver-jvm.md index 831c0d3..63f8046 100644 --- a/docs/sqlserver-jvm.md +++ b/docs/sqlserver-jvm.md @@ -4,7 +4,7 @@ ## SQL Server -Configure a SQL Server connection using the [connection configuration](../README.md) and pass its `connection://name` reference. These captures need neither a local PID nor a command. An empty database filter selects the connection's database; `--databases '*'` selects all databases. Filters accept exact names, `*` wildcards, and `!` exclusions. +Configure a SQL Server connection using the [connection configuration](https://github.com/flanksource/telltale#connections) and pass its `connection://name` reference. These captures need neither a local PID nor a command. An empty database filter selects the connection's database; `--databases '*'` selects all databases. Filters accept exact names, `*` wildcards, and `!` exclusions. ```sh telltale run sql_xevent --connection connection://orders --session-name diagnostic --min-duration 10ms --duration 30s @@ -41,7 +41,7 @@ telltale agents install java --pid 1234 --directory /home/example/.local/share/t telltale run jvm_jfr --seam agent --pid 1234 --duration 30s --every 10s ``` -Installation stages a versioned artifact under the chosen absolute directory, verifies its SHA-256 on the destination, rechecks process identity, and invokes the Java Attach helper. Local discovery files and staged credentials are private. Repeating a compatible local install reuses the authenticated agent; an incompatible agent requires restarting the JVM. `--dry-run` returns the destination and artifact hash without copying or attaching. +Installation stages a versioned artifact under the chosen absolute directory, verifies its SHA-256 on the destination, rechecks process identity, and invokes the Java Attach helper. Local discovery files and staged credentials are private. Repeating a compatible local install reuses the authenticated agent. Add `--upgrade` to replace an idle attached runtime without restarting its JVM; stop captures and restore patches first. Agents installed before the upgrade-capable launcher need one JVM restart to adopt it. `--dry-run` returns the destination and artifact hash without copying or attaching. See [runtime upgrades](jvm-diagnostics-port.md#upgrade-an-attached-runtime). Remote agents dial out to a Telltale server. Create an agent-scope token as the user that runs the hub (it is minted into the global config dir, `~/.config/telltale/agent-tokens`), save the printed value in a private file, and give that file to the installer with `--token-file`. The token value is never a command argument. The hub URL must use HTTP or HTTPS and be reachable both from the target JVM and the installer. Use HTTPS outside a trusted development network. If the hub protects its agent-list API, provide an API-scope token through `TELLTALE_API_TOKEN` to verify readiness. @@ -53,10 +53,14 @@ telltale agents install java --destination kubernetes --context development --na telltale agents list --server https://telltale.example.com ``` -SSH uses native SSH configuration, batch authentication, and optionally `--ssh-config`. Kubernetes uses native kubeconfig and requires explicit namespace, pod, and container. Destinations require POSIX `ps`, `install`, `dd`, `mv`, `rm`, `uname`, and `sha256sum` (or `shasum` on macOS). The installer copies and attaches the agent; profiling then uses the agent protocol. +SSH uses native SSH configuration, batch authentication, and optionally `--ssh-config`. Kubernetes uses native kubeconfig, or `--kubeconfig` and `--context`, and requires explicit namespace, pod, and container. + +`telltale agents install java` is the same command as `telltale jvm deploy`. Instead of a hub, a Kubernetes JVM can verify short-lived tokens itself: name a target (`telltale agents install java --namespace shop sts/orders`), or add `--issuer connection:///`, and the command deploys a Deployer built from the flags, signed by `connection://telltale/issuer` unless you name another, reached later as `--agent deployer///` over a port-forward. Neither a hub nor a token file is needed. See [Deployers](jvm-diagnostics-port.md#deployers). Destinations require POSIX `ps`, `install`, `dd`, `mv`, `rm`, `uname`, and `sha256sum` (or `shasum` on macOS). The installer copies and attaches the agent; profiling then uses the agent protocol. Start remote captures in the connected hub's browser, selecting the agent seam and an agent ID or resource selector such as `service.name=orders`. Remote agent identities are namespaced and cannot be confused with a controller's local PID. A JVM disconnect releases its recordings; the dial-out connection retries with bounded backoff. Recording blobs are limited to 64 MiB. +`telltale agents list` prints tables of locally discovered and connected agents plus saved, successfully deployed targets, grouped by namespace, cloud, or host. The tables show runtime, connection state, deployment state, and endpoint. Use `--format json` for structured output. Connected entries have `connected: true` and their live handshake details. Targets awaiting a connection have `connected: false` and a `known` record with their runtime language, names, last deployment state, and endpoint. Listing these records uses saved metadata; it does not open port-forwards or contact every target. `telltale agents get ` also shows these saved details. Use a JVM command such as `telltale jvm status --agent orders` to connect and inspect a target. With `--server`, listing reads the selected server's inventory from `GET /api/agents`. The browser's Agents page uses this same inventory and grouping, including saved targets and their JVM diagnostics link. + ## HTTP API and browser `GET /api/traces/start` includes schemas and defaults for all three kinds. `POST /api/traces` accepts `pid`, `duration`, and `every`; `command` is optional for connection or agent captures. For example: @@ -69,6 +73,12 @@ Start remote captures in the connected hub's browser, selecting the agent seam a {"traces":{"sql_deadlock":{"enabled":true,"connection":"connection://orders","from":"2026-10-28T08:00:00Z","to":"2026-10-28T09:00:00Z"}}} ``` +A kind may instead be an array of such objects, each its own capture with its own target and params; refusals of an element are reported at `/traces//`: + +```json +{"traces":{"sql_deadlock":[{"enabled":true,"connection":"connection://orders"},{"enabled":true,"connection":"connection://billing"}]}} +``` + Use the run's existing result/export routes to read stored rows and the checkpoint route for JFR. The generated `/api/openapi.json` also exposes `agents install java` and the single-kind run commands. Installer API calls execute on the server and therefore use its filesystem, SSH configuration, and kubeconfig. ## Verification diff --git a/docs/top.md b/docs/top.md new file mode 100644 index 0000000..337a123 --- /dev/null +++ b/docs/top.md @@ -0,0 +1,36 @@ +# Completed sample Top views + +Top reads one completed sample before applying search, column filters, ordering and pagination. A successful empty poll replaces the previous sample. Interrupted reads publish no completion record. History and native JSON exports retain every captured row. Recordings without completion metadata show “Complete samples unavailable” and remain readable in History. + +The kind catalog advertises Top capabilities for `sql_top`, `sql_plancache`, `process`, `goprof`, `jvm_jfr`, `prometheus`, `stacks` and `query`. SQL activity defaults to elapsed time, plan-cache to interval CPU, process observations to CPU, and stacks to waiting time. Prometheus combines the latest completed scrape of each endpoint. Query captures retain the originating profile's columns, filter grammar and order; repeated reads remain incremental windows. + +Use `-i` or `--interactive` to fill a command's arguments in Clicky's JSON Schema form before it runs. Saved connections appear in the Connection chooser, limited to the types that kind accepts: PostgreSQL and SQL Server for `sql_top`, and SQL Server for `sql_plancache`. Existing flags and arguments prefill the form. Tab moves between fields, Enter edits a field, Ctrl+S submits, and Ctrl+C cancels before capture starts. The web Arm captures form uses the same schema metadata and saved-connection lookup. + +```sh +.bin/telltale run sql_top -i +.bin/telltale run sql_top -i --duration 30s --poll 2s +.bin/telltale run sql_plancache -i +``` + +```sh +make build +.bin/telltale records sql_top --run --view top --filter 'elapsedMs=>1s' --limit 20 +.bin/telltale records sql_top --run --view top --frame --sort elapsedMs --desc --json +.bin/telltale trace replay --tui +.bin/telltale trace replay --ui +``` + +In the web viewer, choose a source, sample and metric in Top. CPU profiles use flamegraphs and memory uses treemaps. Process graphs aggregate exclusive process usage through ownership groups; resident memory sums process RSS rather than physical host memory. Search, column filters, metric, zoom, sample, sort and selected row are saved in the URL. Pause, pagination and detail inspection pin the current selection. Resume live explicitly to return to the newest completed sample. SQL detail includes copyable SQL and a blocker lookup within the same sample; an omitted blocker is identified explicitly. + +The TUI uses `s` to change sort, `S` to reverse, `/` for search, `f` for a native `column=expression` filter, `m` for metric, `o` for source, `c` for capture, and `p/n` for historical samples. Enter opens scrollable detail and freezes that sample. `b` follows a blocker, `h` opens History, `r` resumes live, and `?` lists controls. Standalone process Top preserves grouping and guarded process actions; `d` opens frozen detail, `P` pauses/resumes, and `?` shows help. + +The records route accepts `view=frames` to list completion metadata and `view=top` with `frame`, `source`, `metric`, `q`, repeatable `filter`, `where`, `sort`, `desc`, `limit` and `cursor`. Its Top response includes `selection`, the exact frame selection to send when continuing a page or freezing multiple endpoint samples. `Accept: application/schema+json` with `view=top` returns the same native column contract used for reads. Unavailable SQL measurements are null; measured zero remains zero. + +```sh +go run ./hack/top-fixture +.bin/telltale serve --dev --port 8080 --data-dir .tmp/top-ux-review/data +``` + +The fixture creates an embedded local PostgreSQL database, records a lock holder and waiter, then closes the database. Its output names the new recording. It writes only under `.tmp/top-ux-review/`. + +The reusable implementations live in Clicky (`tui/table`), commons-db (`recordstore/sampleframe`, typed query presentation and SQL activity), and Clicky UI (`TopView`, `WeightedProfile`). Consumers must use released package versions for production. Development uses the existing Go workspace and `serve --dev` sibling-source resolution; local replacement directives must not be published. diff --git a/docs/traces/exec.mdx b/docs/traces/exec.mdx new file mode 100644 index 0000000..b245cd5 --- /dev/null +++ b/docs/traces/exec.mdx @@ -0,0 +1,12 @@ +# exec + +import { KindReference } from '../_components/KindReference'; + +Records Clicky executions observed in Telltale itself or an instrumented agent, including output tails and environment variable names. The default `self` seam observes Telltale; use `agent` for a connected agent. + +```sh +telltale run exec --pid 1234 --name git --duration 1m +``` + +`--name` filters executable names with a glob, and `--tail-bytes` limits each output tail (default 4096 bytes). Select an agent with `--pid`, `--agent`, or `--selector`; `--wait` waits for a matching agent. Run `telltale run exec --help` for all options. + diff --git a/docs/traces/goprof.mdx b/docs/traces/goprof.mdx new file mode 100644 index 0000000..1eca299 --- /dev/null +++ b/docs/traces/goprof.mdx @@ -0,0 +1,12 @@ +# goprof + +import { KindReference } from '../_components/KindReference'; + +Reads Go pprof profiles at checkpoints from a command, an attached process, or an agent. The default process source is a [gops agent](https://github.com/google/gops) in the target process, discovered through its port file. + +```sh +telltale run goprof --pid 1234 --duration 1m --every 15s +``` + +`--profile` selects `heap` (default), `cpu`, `allocs`, `mutex`, `block`, or `goroutine`; `--source` selects `gops`, `pprof`, or `file`. `--seconds` sets the CPU profile window. Checkpoints occur at `--every` and once when an attached run stops. Run `telltale run goprof --help` for all options. + diff --git a/docs/traces/http.mdx b/docs/traces/http.mdx new file mode 100644 index 0000000..1f1918a --- /dev/null +++ b/docs/traces/http.mdx @@ -0,0 +1,20 @@ +# http + +import { KindReference } from '../_components/KindReference'; + +Records HTTP exchanges from Telltale, an instrumented agent, or a local proxy. The default seam is `self`; choose `proxy` to route a command through the capture. + +```sh +telltale run http --seam proxy --mode mitm -- curl -sS https://example.com +``` + +`--mode connect` records HTTPS tunnels by host; `mitm` decrypts exchanges using an ephemeral CA exported to the command. `--hosts` and `--paths` filter traffic, `--bodies` limits retained body size, and `--redact` adds headers to redact. The proxy exports standard proxy and CA environment variables; the application must honor them. Run `telltale run http --help` for all options. + +For an attached JVM, `--direction incoming`, `outgoing`, or `both` selects servlet requests and HTTP client calls, including Spring MVC and WebFlux. Hooks observe subsequent operations on existing application instances without restarting the JVM. Asynchronous calls and streamed bodies complete when their body completes, closes, fails, or is cancelled. Body capture is bounded by `--bodies`, with header redaction applied before records leave the JVM. + +```sh +telltale run http --agent java/checkout --allow-mutation --direction both --duration 1m +``` + +Use `instrument.jvm.http` in a TraceDefinition. For the current active operations without retained bodies, use [http_top](/traces/http_top). + diff --git a/docs/traces/http_top.mdx b/docs/traces/http_top.mdx new file mode 100644 index 0000000..f5d701e --- /dev/null +++ b/docs/traces/http_top.mdx @@ -0,0 +1,33 @@ +# http_top + +import { KindReference } from '../_components/KindReference'; + +Samples active incoming and outgoing HTTP exchanges in an attached JVM. Each snapshot records the operation identity, elapsed time, direction, method, route or URL, and available thread and trace correlation. HTTP bodies belong to the `http` capture; this view contains metadata only. + +```bash +telltale run http_top --agent java/checkout --allow-mutation --direction both --poll 5s --top 50 --duration 1m +``` + +The agent observes subsequent operations on existing servlet deployments and HTTP client instances. Spring MVC and WebFlux retain their route and handler metadata when available. An asynchronous request or streamed response remains active until its body completes, closes, fails, or is cancelled; receiving response headers does not by itself finish it. Operations whose starts were not observed at attachment are outside the capture boundary. + +Profiles carry the snapshots using producer timestamps and sequence numbers. An empty complete snapshot clears the view; missing snapshots leave the previous reading stale. Elapsed values are ages at a point in time and must not be summed across snapshots. Top limits and dropped observations are reported separately. + +Definitions use the JVM instrumentation section: + +```yaml +apiVersion: telltale.flanksource.com/v1alpha1 +kind: TraceDefinition +metadata: {name: service-http} +spec: + instrument: + jvm: + httpTop: + - agent: {selector: {service.name: checkout}, wait: true} + direction: both + poll: 5s + top: 50 +``` + +Attaching and arming require mutation permission. This enables observation hooks without replacing application beans, tracing providers, or normal exporters. Inspect the connected JVM's capabilities for its supported libraries and versions. + + diff --git a/docs/traces/index.md b/docs/traces/index.md new file mode 100644 index 0000000..b675f0e --- /dev/null +++ b/docs/traces/index.md @@ -0,0 +1,33 @@ +# Trace kinds + +Each trace kind records a particular source. Run one with `telltale run `, or combine captures using `telltale trace run --trace`. Run `telltale run --help` for the complete, schema-derived flags for your installed version. + +The target seam determines what is observed: `process` follows a command or PID, `host` observes host resources or remote services, `connection` uses a configured database, `proxy` routes a command through a local proxy, `agent` observes an instrumented agent, and `self` observes Telltale's own instrumented sources. A self capture does not automatically record the command's activity. + +| Kind | Records | +| --- | --- | +| [exec](/traces/exec) | Instrumented command executions | +| [goprof](/traces/goprof) | Go profiles at checkpoints | +| [http](/traces/http) | HTTP exchanges | +| [http_top](/traces/http_top) | Active incoming and outgoing JVM HTTP exchanges | +| [jms](/traces/jms) | JVM message sends, receives, and processing | +| [jvm_jfr](/traces/jvm_jfr) | JVM profiling samples | +| [jvm_notifications](/traces/jvm_notifications) | JMX notifications and garbage collections | +| [jvm_ognl](/traces/jvm_ognl) | Periodic expression samples | +| [jvm_redefine](/traces/jvm_redefine) | Temporary Java replacements and restoration | +| [jvm_trace](/traces/jvm_trace) | Method values and nested calls | +| [llm](/traces/llm) | Instrumented LLM provider calls | +| [logs](/traces/logs) | Application logs | +| [process](/traces/process) | Process startup metadata, resource deltas and numeric metrics | +| [prometheus](/traces/prometheus) | Metrics scrapes | +| [query](/traces/query) | Query profile or provider results | +| [spans](/traces/spans) | Application or imported spans | +| [sql](/traces/sql) | Instrumented or proxied SQL statements | +| [sql_deadlock](/traces/sql_deadlock) | SQL Server deadlocks | +| [sql_plancache](/traces/sql_plancache) | SQL Server plan cost deltas | +| [sql_top](/traces/sql_top) | Running database requests or active JVM SQL executions | +| [sql_xevent](/traces/sql_xevent) | SQL Server Extended Events | +| [stacks](/traces/stacks) | Process stacks at checkpoints | +| [terminal](/traces/terminal) | Terminal events supplied by a host | + +Runs are stored in the data directory. Use `telltale trace list --kind ` to find runs, and `--follow`, `--tui`, or `--ui` to inspect captures while recording. Some kinds require an instrumented application, a connection, or external tools; their pages explain the relevant prerequisites. diff --git a/docs/traces/jms.mdx b/docs/traces/jms.mdx new file mode 100644 index 0000000..033f032 --- /dev/null +++ b/docs/traces/jms.mdx @@ -0,0 +1,32 @@ +# jms + +import { KindReference } from '../_components/KindReference'; + +Records incoming and outgoing JMS operations inside an attached JVM: producer sends, consumer receive waits, and message processing. Spring templates and annotated listeners use the same capture. Records preserve each stage separately, including outgoing replies and redelivery attempts. + +```bash +telltale run jms --agent java/checkout --allow-mutation --direction both --duration 1m +``` + +Message bodies are disabled by default. Records include available destination, message and correlation IDs, redelivery state, duration, outcome, exception phase, Spring handler, and existing trace correlation. Missing metadata stays absent. The observer never acknowledges messages, changes transactions, deserializes object messages, or consumes message-body cursors. + +An asynchronous send completes at its completion callback or synchronous failure. Successful send completion does not prove broker delivery or transaction commit. Listener failures handled by Spring remain observable, and receive timeouts are distinguished from successful message receipt. Provider metadata that cannot be observed appears in `metadataErrors`; incomplete operations retain an explicit outcome. + +Outcomes distinguish `success`, `error`, `timeout`, and `incomplete`. Unsupported body representations are reported as `bodyUnsupported`, and failures reading optional metadata appear in `metadataErrors` rather than silently replacing values. + +The JVM bounds active operations and pending asynchronous messages at 8,192 each. Exceeding either limit fails the capture explicitly. + +```yaml +apiVersion: telltale.flanksource.com/v1alpha1 +kind: TraceDefinition +metadata: {name: queue-traffic} +spec: + instrument: + jvm: + jms: + - agent: {selector: {service.name: checkout}, wait: true} + direction: both + destinations: [orders] +``` + + diff --git a/docs/traces/jvm_jfr.mdx b/docs/traces/jvm_jfr.mdx new file mode 100644 index 0000000..3924836 --- /dev/null +++ b/docs/traces/jvm_jfr.mdx @@ -0,0 +1,12 @@ +# jvm_jfr + +import { KindReference } from '../_components/KindReference'; + +Profiles a local JVM or a connected Java agent with Flight Recorder or async-profiler. Checkpoints decode new samples into the run store. + +```sh +telltale run jvm_jfr --pid 1234 --duration 30s --every 10s +``` + +The process seam requires a compatible JDK's `jcmd`, attach permission, and JFR. Select `--engine async-profiler --event cpu` (or `wall`, `alloc`, or `lock`) to use async-profiler; local captures require `asprof`. `--threads`, `--interval`, and `--max-stack-depth` control sampling and decoded output. See [SQL Server and JVM captures](/sqlserver-jvm#local-jvm) for prerequisites and [Java agent installation](/sqlserver-jvm#java-agent-installation). + diff --git a/docs/traces/jvm_notifications.mdx b/docs/traces/jvm_notifications.mdx new file mode 100644 index 0000000..829a722 --- /dev/null +++ b/docs/traces/jvm_notifications.mdx @@ -0,0 +1,13 @@ +# jvm_notifications + +import { KindReference } from '../_components/KindReference'; + +Subscribe to JMX notifications from an MBean name or ObjectName pattern. Garbage collector notifications include decoded GC details. The bounded buffer records dropped-event counts when consumers fall behind. + +```sh +telltale run jvm_notifications --agent 1234 --duration 10s --mbean 'java.lang:type=GarbageCollector,*' +``` + +Use `--types` to select notification type prefixes and `--capacity` to size the buffer. Stopping or disconnecting the capture removes its listeners. See [JVM diagnostics](/jvm-diagnostics-port). + + diff --git a/docs/traces/jvm_ognl.mdx b/docs/traces/jvm_ognl.mdx new file mode 100644 index 0000000..77ac1f8 --- /dev/null +++ b/docs/traces/jvm_ognl.mdx @@ -0,0 +1,13 @@ +# jvm_ognl + +import { KindReference } from '../_components/KindReference'; + +Evaluate an OGNL expression periodically in a connected JVM and record scalar values or bounded object graphs. Select an application loader hash when the expression references application classes. + +```sh +telltale run jvm_ognl --agent 1234 --allow-mutation --duration 5s --expression '@java.lang.Runtime@getRuntime().freeMemory()' +``` + +Expressions can invoke application code and require both agent and caller mutation consent. `--interval-ms` sets the sample interval. See [JVM diagnostics](/jvm-diagnostics-port) for loader selection and one-shot expressions or Groovy scripts. + + diff --git a/docs/traces/jvm_redefine.mdx b/docs/traces/jvm_redefine.mdx new file mode 100644 index 0000000..b6062d9 --- /dev/null +++ b/docs/traces/jvm_redefine.mdx @@ -0,0 +1,13 @@ +# jvm_redefine + +import { KindReference } from '../_components/KindReference'; + +Compile replacement Java source against a live class and apply it for a capture. Stopping the capture restores its original bytes. Use the JVM diagnostics class browser to inspect source and preview compilation before applying a replacement. + +```sh +telltale run jvm_redefine --agent 1234 --allow-mutation --duration 10s --class example.Worker --method compute --source 'public static double compute(long value) { return 7.0; }' +``` + +The agent must allow mutation. The replacement must match the selected method or class structure; compile errors are returned without applying bytecode. A renewable JVM lease restores abandoned patches. See [JVM diagnostics](/jvm-diagnostics-port). + + diff --git a/docs/traces/jvm_trace.mdx b/docs/traces/jvm_trace.mdx new file mode 100644 index 0000000..1299fef --- /dev/null +++ b/docs/traces/jvm_trace.mdx @@ -0,0 +1,27 @@ +# jvm_trace + +import { KindReference } from '../_components/KindReference'; + +Watch arguments, return values and exceptions, or trace nested method calls in a connected Java agent. Select one to eight methods and use a loader hash when a class name is ambiguous. + +```sh +telltale run jvm_trace --agent 1234 --allow-mutation --duration 10s --targets '[{"class":"example.Worker","method":"compute"}]' --args --return --exception --calls +``` + +## Keeping only the calls you want + +A hot method fills the event budget with calls nobody is looking at. Root-call filters keep only the outermost calls that qualify, each with everything nested in it, and count the rest as `filtered` in the capture report: + +- `minCost` keeps calls that ran at least this long, such as `250ms`. +- `exceptionsOnly` keeps calls that threw. +- `match` keeps calls whose rendered argument (`on: arg`, `index`) or return value (`on: return`) passes every predicate. `path` names a value inside it by dotted child labels, and `op` is `equals`, `contains` or `regex` (a Java regular expression found anywhere in the value). A match needs `--args` or `--return` to be recording that value. + +```sh +telltale run jvm_trace --agent 1234 --allow-mutation --duration 5m --targets '[{"class":"example.Dispatcher","method":"send"}]' --args --calls --min-cost 250ms --match '[{"on":"arg","index":0,"op":"contains","value":"SubmitTask"}]' +``` + +The agent holds each call on its own thread until it ends. A held call is bounded by the event limit and the probe's byte budget; one that outgrows them keeps its first events and its closing events, and its last event carries `omitted` with the count left out between them. Filters need probe API v10, which the agent advertises on its `jvm_trace` probe: an older agent attached to the JVM is refused before anything is instrumented, because it would ignore the filters and return every call. + +The agent must be installed with mutation enabled. Captures restore original bytecode when stopped, disconnected, expired or completed at their event limit. The JVM also owns a renewable lease for abandoned captures. See [JVM diagnostics](/jvm-diagnostics-port) for installation, leases and the standalone WAR. + + diff --git a/docs/traces/llm.mdx b/docs/traces/llm.mdx new file mode 100644 index 0000000..5f9e57c --- /dev/null +++ b/docs/traces/llm.mdx @@ -0,0 +1,12 @@ +# llm + +import { KindReference } from '../_components/KindReference'; + +Records Captain provider calls in Telltale or an instrumented agent. The default `self` seam observes Telltale itself. + +```sh +telltale run llm --pid 1234 --include-prompts --duration 1m +``` + +Prompts and responses are excluded by default; enable them separately with `--include-prompts` and `--include-responses`. Use `--models` to filter model names by glob. Agent selection supports `--pid`, `--agent`, and `--selector`. Run `telltale run llm --help` for all options. + diff --git a/docs/traces/logs.mdx b/docs/traces/logs.mdx new file mode 100644 index 0000000..a5cdf6d --- /dev/null +++ b/docs/traces/logs.mdx @@ -0,0 +1,12 @@ +# logs + +import { KindReference } from '../_components/KindReference'; + +Records application logs from Telltale, an instrumented agent, or a configured host source. The default seam is `self`. + +```sh +telltale run logs --seam agent --agent my-agent --duration 1m +``` + +Options include `--min-level`, `--match`, and per-logger `--levels`; host-source options include `--backend`, `--connection`, `--query`, `--namespace`, `--pod-selector`, and `--container`. Available settings depend on the selected source. Run `telltale run logs --help` for the complete flags. + diff --git a/docs/traces/process.mdx b/docs/traces/process.mdx new file mode 100644 index 0000000..1be1f51 --- /dev/null +++ b/docs/traces/process.mdx @@ -0,0 +1,35 @@ +# process + +import { KindReference } from '../_components/KindReference'; + +Capture a process tree or the host with one kind and three tables: + +| Result | Storage | Contents | +| --- | --- | --- | +| Primary | `process` | Stable PID/start identity, parent, startup command, CWD, selected masked environment and grouping tags; observed exit updates the same row. | +| `facets` | `process_facets` | Timestamp, tick, process identity, facet, entity, `added`/`removed`, descriptor and direction evidence. | +| `metrics` | `process_metrics` | One numeric column per metric, keyed by process identity, observation time and tick. Unavailable values are NULL; measured zero stays zero. | + +```sh +telltale run process --pid 1234 --facets files,sockets,connections --duration 30s +telltale run process --seam host --duration 1m +telltale records process --run +telltale records process --run --result metrics --stats cpuPercent,rssBytes --group-by entity +telltale records process --run --result facets --follow +telltale trace replay --tui +``` + +Files and receiving sockets can be selected independently. Receiving sockets include TCP listeners and bound UDP sockets. Connections count established TCP peers, collapsed across concurrent sockets to the same remote endpoint: the first adds a peer and closing the last removes it. Listener history supplies inbound inference; a newly observed peer without a listener supplies outbound inference after the initial observation. Other initial peers remain unknown. UDP sockets are not TCP connections. Polling can miss short-lived resources, and unreadable descriptor observations do not remove resources. Observed process exits remove held resources with `reason: observed_exit`. + +Each observation commits identities, deltas and metrics in one transaction. Failed writes leave the observation state unchanged for retry. Metric columns are seeded, grow for newly encountered numeric metrics, and have a bounded count. Group totals are derived from startup grouping information and metric rows; group snapshots and full descriptor lists are not repeated in storage. + +The storage benchmark uses 20 processes with 100 file descriptors each, changes one file per tick and writes 100 observations to SQLite. On an Apple M2 Pro, repeated snapshots used 168,346 bytes per tick and 6.71 ms per write; the unified tables used 33,260 bytes and 5.99 ms. This is about 80% less disk use for that workload. Allocated bytes were slightly higher for the unified writer. Process and descriptor polling are measured separately; delta storage does not reduce the cost of discovering resources, and savings depend on resource churn. + +```sh +go test ./pkg/tracing/processcapture -run '^$' -bench BenchmarkProcessObservationStorage -benchtime=100x -benchmem +go test ./pkg/tracing/processcapture -run '^$' -bench BenchmarkProcessCollection -benchtime=3x -benchmem +``` + +`process_tree`, `process_samples` and `process_groups` recordings require the build that recorded them. This build leaves those files untouched and reports their format as unsupported. + + diff --git a/docs/traces/profiles.mdx b/docs/traces/profiles.mdx new file mode 100644 index 0000000..dd5c5bc --- /dev/null +++ b/docs/traces/profiles.mdx @@ -0,0 +1,84 @@ +--- +title: Active operation profiles +description: OTLP Profiles transport for current HTTP requests and SQL operations +--- + +Telltale carries JVM `http_top` and `sql_top` snapshots using the development OTLP Profiles schema. Each profile is a complete observation of the operations retained by one capture at one sampling tick. An operation sample represents one active operation and contains its current elapsed nanoseconds. These values are gauges: repeated observations of the same operation must never be summed as elapsed intervals. + +Telltale convention version `2` carries empty observations with one explicit zero-valued control sample. This works around Collector 0.163.0 discarding zero-sample profiles; released 0.165.0 has the same [receiver guard](https://github.com/open-telemetry/opentelemetry-collector/blob/e883d737e49935984df6b064e512eed6cdc0d289/receiver/otlpreceiver/internal/profiles/otlp.go#L33-L46). Telltale preserves the control sample in the native profile and excludes it from operation rows. Generic Profiles consumers see that sample, and Collector sample counters count it as one. + +## Wire contract + +The implementation pins `go.opentelemetry.io/collector/pdata/pprofile` to `v0.163.0`, Collector pdata to `v1.69.0`, Java `io.opentelemetry.proto:opentelemetry-proto` to `1.11.1-alpha`, and Java protobuf to `4.36.2`. Java production sources compile for Java 8. Profiles use the `opentelemetry.proto.profiles.v1development` package and development version `1`. Changes to these pins require rerunning the Java golden and actual Collector gates. + +The instrumentation scope is `com.flanksource.telltale.top`, version `1`. The sample type is `telltale.http.active.elapsed` or `telltale.sql.active.elapsed`, with unit `ns`. Every dictionary contains reserved zero entries for strings, attributes, mappings, locations, functions, links, and stacks. Real stack frames occupy the function, location, and stack tables; trace and span IDs occupy a native link entry. Elapsed values, timestamps, sequence numbers, and thread IDs use protobuf integers, preserving values above JavaScript's exact integer range. + +Each profile carries these attributes: + +| Attribute | Type | Meaning | +| --- | --- | --- | +| `telltale.schema.version` | integer | Telltale convention version, currently `2`; version `1` is rejected | +| `telltale.kind` | string | `http_top` or `sql_top` | +| `telltale.capture.id` | string | Capture identity | +| `telltale.source.instance` | string | JVM source identity; changes when the JVM restarts | +| `telltale.snapshot.sequence` | positive integer | Increasing sequence within a source and capture | +| `telltale.snapshot.observed` | nonnegative integer | Number of matching operations before row limiting | +| `telltale.snapshot.limit` | nonnegative integer | Retention limit; zero means no row limit | +| `telltale.snapshot.dropped` | nonnegative integer | Events or operations lost by the observer | +| `telltale.snapshot.complete` | boolean | Whether this tick finished successfully | +| `telltale.checkpoint.id` | string | Optional capture checkpoint, empty when absent | +| `telltale.snapshot.poll_interval_ms` | positive integer | Optional expected sampling interval for freshness display | + +The native profile UUID identifies a snapshot. `time_unix_nano` and `duration_nano` define the producer's observation bounds. Each operation sample has exactly one elapsed value and one observation timestamp inside those bounds. `telltale.operation.id` identifies the request or SQL execution across ticks. HTTP attributes describe method, direction, URL, route, phase, start time, and thread. SQL attributes describe JDBC or R2DBC, statement or explicit unavailability, driver or pool context, start time, and thread. Resource attributes describe the JVM and application. The Telltale convention version is independent of instrumentation scope version `1` and OTLP development version `1`. + +An empty observation has observed count zero, normal producer metadata, and exactly one control sample. Its sole sample attribute is `telltale.sample.kind="snapshot_control"`. It has one elapsed value of `0`, one timestamp equal to the snapshot's completed-at time, zero stack and link references, and no operation ID or HTTP/SQL operation attributes. Decoding produces zero operation rows. A complete empty observation clears the current table. + +The decoder rejects zero-sample version-2 profiles, missing or unknown marker kinds, extra control attributes, mixed control/operation samples, nonzero control values, invalid control timestamps, stack/link references, and control samples with a nonzero observed count. Encoders reject zero retained operations with a positive observed count. The convention does not downgrade to version 1. Collector filters and transforms must retain the control sample and its marker attribute. + +A missing message leaves the previous observation available with its receive timestamp. Truncation retains the profile's observed count and row limit. An incomplete profile remains available in history and does not replace the newest complete observation. Dropped events are displayed separately from truncation and completeness. + +## Transport and persistence + +The injected JVM sends binary `ProfilesData` in the existing authenticated capture blob transport with content type `application/x-protobuf;message=opentelemetry.proto.profiles.v1development.ProfilesData`. Decoding binds its kind, capture ID, and source identity to the authenticated capture. Direct agent and standalone ingestion share a durable profile UUID and source/capture/sequence journal, committing identities, rows, and frame metadata atomically. Producer sequences and completeness survive interleaved sources and reordered delivery; delivery gaps do not make a complete observation incomplete. The transport never reconstructs active operations from completed spans. + +Standalone producers send OTLP export requests directly to `POST /v1/profiles`, or use the OTLP Profiles gRPC service on the configured OTLP listener. HTTP accepts protobuf or JSON and optional gzip. Send `OTLP-Profiles-Development-Version: 1`; unsupported, repeated, and malformed version metadata fail before payload decoding. HTTP responses identify development version `1`. API-scoped bearer tokens protect standalone ingestion; agent-scoped tokens are rejected. The standalone receiver uses the producer's resource, source, and capture metadata and appears under the standing `receiver` run. + +For a saved binary golden request, direct HTTP ingestion is reproducible with: + +```sh +curl --fail-with-body \ + -H "Authorization: Bearer $TELLTALE_API_TOKEN" \ + -H 'Content-Type: application/x-protobuf' \ + -H 'OTLP-Profiles-Development-Version: 1' \ + --data-binary @pkg/otlp/profiles/testdata/java-http-top-active.pb \ + http://localhost:8080/v1/profiles +``` + +Adapt the URL to the configured Telltale HTTP address. An HTTP success means the direct receiver committed the snapshot, its retained operation rows, and identity metadata atomically. A retry of the same content is idempotent; conflicting content for the same UUID or source/capture/sequence receives OTLP partial rejection. Old snapshots remain history and cannot replace a newer producer sequence. Retries do not refresh the newest snapshot's receive timestamp. + +The receiver stores the redacted native profile once in the frame summary, alongside its canonical digest and status metadata. Operation rows contain the typed projection. Native periods, dropped attributes, resource and scope schemas, dictionaries, trace links, and stack detail supported by the pinned pdata schema survive readback. Unrecognized future wire fields are outside that preservation contract. The digest resolves dictionary references to semantic values and sorts unordered samples and attributes, so Collector reserialization remains idempotent while changed native metadata conflicts. Redaction rebuilds dictionary references so unused strings cannot retain secret values. Native payloads embedded in `original_payload` are rejected because they cannot participate safely in this structured convention. + +Rows, frames, and the small identity ledger expire using receive time and the configured receiver retention. Producer clock skew does not expire newly received observations. Reads and Top use the same record store as captures and the standing receiver run; there is no parallel snapshot database. + +## Verification and Collector compatibility + +The ordinary Go codec tests read checked binary fixtures emitted by the Java encoder. Receiver tests cover direct protobuf, JSON, gzip, gRPC, API authentication, durable retries, conflicts, reordered delivery, empty clearing, native readback, and retention. Control-sample tests check the exact shape and reject malformed markers before publication. + +```sh +go test ./pkg/otlp/profiles +go test ./internal/otelreceiver -run TestReceiver -ginkgo.focus='Profiles protocol boundaries|durable Profiles snapshots|OTLP Profiles receiver' +``` + +Regenerate the Java fixtures from `sdk/java` when deliberately updating the shared wire contract: + +```sh +TELLTALE_UPDATE_PROFILE_GOLDEN=true ./gradlew test --tests com.flanksource.telltale.observer.profiles.ProfileEncoderTest +``` + +The actual Collector compatibility gate starts the released `otlpreceiver` and `otlpexporter` components from a separately pinned module in `hack/collector-fixture`, tests request-sized and item-sized export queues, and sends network requests through both components. It verifies that active, updated, and empty observations arrive intact and that the native control sample produces zero operation rows in the latest Top view: + +```sh +go test ./internal/otelreceiver -run TestReceiver -ginkgo.focus='released Collector Profiles pipeline' +``` + +The control-sample workaround was approved on 2026-11-15. The correct upstream fix must distinguish empty telemetry envelopes from profiles that have metadata and zero samples. In 0.165.0, the receiver still has a [zero-sample early return](https://github.com/open-telemetry/opentelemetry-collector/blob/e883d737e49935984df6b064e512eed6cdc0d289/receiver/otlpreceiver/internal/profiles/otlp.go#L33-L46). The exporter's [item count remains the sample count](https://github.com/open-telemetry/opentelemetry-collector/blob/e883d737e49935984df6b064e512eed6cdc0d289/exporter/exporterhelper/xexporterhelper/profiles.go#L98-L100), so [an item-sized queue discards zero-sample requests](https://github.com/open-telemetry/opentelemetry-collector/blob/e883d737e49935984df6b064e512eed6cdc0d289/exporter/exporterhelper/internal/queue/memory_queue.go#L61-L66). The default request-sized queue counts each request as one. [Batch merging and splitting also omit zero-sample data](https://github.com/open-telemetry/opentelemetry-collector/blob/e883d737e49935984df6b064e512eed6cdc0d289/exporter/exporterhelper/xexporterhelper/profiles_batch.go#L31-L79). These paths all require profile-aware handling before the control sample can be removed. The pinned 0.163.0 pipeline uses receiver and queued exporter components; its stock batch processor lacks Profiles support. diff --git a/docs/traces/prometheus.mdx b/docs/traces/prometheus.mdx new file mode 100644 index 0000000..0506abd --- /dev/null +++ b/docs/traces/prometheus.mdx @@ -0,0 +1,17 @@ +# prometheus + +import { KindReference } from '../_components/KindReference'; + +Scrapes Prometheus exporters using a managed VictoriaMetrics process. Captures report target health and store the metric samples directly in VictoriaMetrics. + +```sh +telltale run prometheus --endpoints http://localhost:8080/metrics --interval 5s --duration 1m +telltale run prometheus --endpoints port-forward://metrics-exporter.monitoring:9100/metrics --duration 1m +telltale run prometheus --endpoints proxy://metrics-exporter.monitoring:9100/metrics --duration 1m +telltale run prometheus --endpoints https://metrics.example.com/metrics --duration 1m +``` + +Mix direct HTTP(S) URLs and Kubernetes URLs in a comma-separated `--endpoints` list. Inline Kubernetes URLs use the credentials of the Telltale process running the capture. Replace `metrics-exporter`, `monitoring`, and `9100` with the Service name, namespace, and Service port. `portforward://` and `port-forward://` both open a tunnel for the capture. `proxy://` uses the Kubernetes API server's service proxy. Use `?kind=pod` with a pod name and port to target a specific pod. Telltale keeps the input URL in the `telltale_endpoint` label. + +The first capture installs and starts VictoriaMetrics. `--endpoints` accepts multiple URLs, `--interval` defaults to 15 seconds, `--timeout` bounds a scrape, and `--labels` adds labels to samples. Query samples at the URL shown by the run or through the read-only `telltale-prometheus` connection. Start `telltale serve --prometheus` with the same `--data-dir` to reuse the instance and keep samples queryable between captures. Samples persist across restarts. + diff --git a/docs/traces/query.mdx b/docs/traces/query.mdx new file mode 100644 index 0000000..1379f65 --- /dev/null +++ b/docs/traces/query.mdx @@ -0,0 +1,12 @@ +# query + +import { KindReference } from '../_components/KindReference'; + +Reads a saved commons-db query profile or a provider configured in place over a run's window. The host seam does not require a command or PID. + +```sh +telltale run query --profile pod-cpu-by-pod --every 30s --duration 10m +``` + +`--profile` selects a saved profile; `--provider` accepts typed provider configuration as JSON. `--from` and `--to` set the window, `--every` polls for newer rows, and `--until-stopped` reads once at the end. Supported providers include SQL, Prometheus, OpenSearch, HTTP, PostgREST, BigQuery, Jaeger, and OpenTelemetry. Run `telltale run query --help` for all options. + diff --git a/docs/traces/spans.mdx b/docs/traces/spans.mdx new file mode 100644 index 0000000..7fc6311 --- /dev/null +++ b/docs/traces/spans.mdx @@ -0,0 +1,20 @@ +# spans + +import { KindReference } from '../_components/KindReference'; + +Records SDK spans in Telltale or an instrumented agent, or imports span documents from an OpenSearch-backed OpenTelemetry connection using the host seam. + +```sh +telltale run spans --seam host --opensearch-connection traces --opensearch-from now-1h +``` + +Use `--opensearch-follow-index` to follow new documents, `--opensearch-to` to bound a window, and `--opensearch-search` for a structured JSON search. For continuous OTLP ingestion without a capture run, see [Receive OpenTelemetry traces](/otlp-receiver). The [web workspace](/web-ui) can search and inspect recorded spans. + +An attached JVM can mirror spans from its existing OpenTelemetry exporters without replacing its tracer provider or exporter configuration. It observes subsequent exported spans, preserving the application's trace IDs, resource attributes, events, links, and status. It cannot recover spans already exported before attachment or spans discarded by the application's sampling policy. Collector processing performed after export is outside this mirror. + +```sh +telltale run spans --agent java/checkout --allow-mutation --duration 1m +``` + +Definitions use `instrument.jvm.spans`. JVM capabilities report the available OpenTelemetry exporters; an unsupported exporter fails explicitly. + diff --git a/docs/traces/sql.mdx b/docs/traces/sql.mdx new file mode 100644 index 0000000..2042914 --- /dev/null +++ b/docs/traces/sql.mdx @@ -0,0 +1,20 @@ +# sql + +import { KindReference } from '../_components/KindReference'; + +Records SQL statements from Telltale, an instrumented agent, or a local PostgreSQL proxy. The default seam is `self`; use `proxy` for a command's database traffic. + +```sh +telltale run sql --seam proxy --env ORDERS_DB_URL -- go test ./... +``` + +The proxy reads the upstream DSN from the named environment variable and exports a proxied DSN under the same name. Alternatively use `--dsn` or the standard database environment variables. `--min-duration` filters statements and `--max-statements` limits retained rows. `--params` includes bind values, which can contain sensitive data. Run `telltale run sql --help` for all options. + +An attached JVM observes existing JDBC and R2DBC connections, including calls made through Spring data access layers. The command requires an explicit connection selection: + +```sh +telltale run sql --agent java/checkout --allow-mutation --connections '*' --duration 1m +``` + +In a TraceDefinition, `instrument.jvm.sql` defaults omitted `connections` to `['*']`. The `self` selector is Go-only. Go and Telltale connection selection keeps its existing defaults. SQL prepared before attachment may have unavailable text; the observer records that reason with the observed execution timing. + diff --git a/docs/traces/sql_deadlock.mdx b/docs/traces/sql_deadlock.mdx new file mode 100644 index 0000000..decd929 --- /dev/null +++ b/docs/traces/sql_deadlock.mdx @@ -0,0 +1,12 @@ +# sql_deadlock + +import { KindReference } from '../_components/KindReference'; + +Reads SQL Server deadlock graphs from `system_health`, either historically or by following new deadlocks. It uses a configured SQL Server connection and does not require a local process. + +```sh +telltale run sql_deadlock --connection connection://orders --duration 1m --poll 5s +``` + +Set `--from` to an RFC 3339 timestamp for a historical read; `--to` is exclusive and defaults to server time. `--databases` filters participants, defaulting to the connected database. Reading requires the appropriate server-state and event-file permissions. See [SQL Server captures](/sqlserver-jvm#sql-server) for retention, shutdown, and permission details. + diff --git a/docs/traces/sql_plancache.mdx b/docs/traces/sql_plancache.mdx new file mode 100644 index 0000000..21cdf15 --- /dev/null +++ b/docs/traces/sql_plancache.mdx @@ -0,0 +1,12 @@ +# sql_plancache + +import { KindReference } from '../_components/KindReference'; + +Samples SQL Server's plan cache, recording interval execution and resource deltas, recompiles, and evictions. Each poll ranks plans by interval CPU. + +```sh +telltale run sql_plancache --connection connection://warehouse --duration 2m --poll 30s --top 200 +``` + +`--databases` and `--text` filter with wildcard and exclusion patterns; `--min-executions` sets a minimum interval execution count. `--top` defaults to 200 and `--poll` to 30 seconds. This connection capture does not need a local process. Run `telltale run sql_plancache --help` for all options. + diff --git a/docs/traces/sql_top.mdx b/docs/traces/sql_top.mdx new file mode 100644 index 0000000..221dd3b --- /dev/null +++ b/docs/traces/sql_top.mdx @@ -0,0 +1,24 @@ +# sql_top + +import { KindReference } from '../_components/KindReference'; + +Samples active SQL executions in database connections or attached JVMs. The agent seam observes JDBC and R2DBC executions with an agent target and mutation permission: + +```bash +telltale run sql_top --agent java/checkout --allow-mutation --poll 5s --top 50 --duration 1m +``` + +JDBC rows describe the execute-call lifetime; R2DBC rows distinguish execution, result consumption, and results awaiting consumption. Spring JDBC templates, Spring Data JDBC, JPA/Hibernate, and reactive templates use these same execution boundaries. SQL prepared before attachment may have unavailable text, which is reported explicitly while retaining its observed timing. + +The common `active` result powers Top for both connection and JVM captures. Native database detail remains available for connection captures. JVM observations do not invent database session IDs, blockers, waits, transaction ages, or server CPU measurements. Empty complete Profiles snapshots clear the active view; sampled elapsed ages are never summed across snapshots. + +Samples what a Postgres or SQL Server is running right now, top style. Each poll reads `pg_stat_activity` or `sys.dm_exec_requests` on the connection and records one row per running request, longest first: session, user, application, state, wait, blocker, elapsed time, transaction age and SQL. SQL Server rows also carry CPU, reads, writes, granted memory and the query hash. The engine follows the connection's type. + +```sh +telltale run sql_top --connection connection://orders --duration 2m --poll 2s +``` + +Sessions that hold an open transaction are always recorded, even when idle, because they are usually the blockers. `--idle` records every idle session as well. `--databases` and `--text` filter with wildcard and exclusion patterns, and `--min-elapsed` drops short requests. `--top` keeps the longest requests per poll and defaults to 50; `--poll` defaults to 5 seconds. Every recorded column can be filtered and sorted, for example `blocked`, `waitType` or `elapsedMs`. This connection capture does not need a local process. Run `telltale run sql_top --help` for all options. + +In a TraceDefinition, `collect.postgres.top` and `collect.sqlserver.top` arm it against that section's default connection. + diff --git a/docs/traces/sql_xevent.mdx b/docs/traces/sql_xevent.mdx new file mode 100644 index 0000000..c4a269c --- /dev/null +++ b/docs/traces/sql_xevent.mdx @@ -0,0 +1,12 @@ +# sql_xevent + +import { KindReference } from '../_components/KindReference'; + +Captures SQL Server Extended Events through a configured connection, either by creating an owned session or reading the existing `system_health` session. + +```sh +telltale run sql_xevent --connection connection://orders --session-name diagnostic --duration 30s +``` + +An owned session requires permission to create and alter server event sessions; Telltale drains and drops only its own session. Use `--session system_health` to read the existing session without changing it. Filters include `--events`, `--databases`, `--users`, `--apps`, `--hosts`, `--types`, and `--tables`. See [SQL Server captures](/sqlserver-jvm#sql-server) for permissions and behavior. + diff --git a/docs/traces/stacks.mdx b/docs/traces/stacks.mdx new file mode 100644 index 0000000..b0c8d6c --- /dev/null +++ b/docs/traces/stacks.mdx @@ -0,0 +1,12 @@ +# stacks + +import { KindReference } from '../_components/KindReference'; + +Dumps Go goroutines or, for other processes, threads using the OS sampler. Observe a command, an attached process, or an agent. + +```sh +telltale run stacks --pid 1234 --duration 1m --every 10s +``` + +Checkpoints run at `--every` and once when an attached run stops. An attached process remains running after capture. Use `--seam agent` with an agent selector for a connected agent, or `--seam self` for Telltale. Run `telltale run stacks --help` for all options. + diff --git a/docs/traces/terminal.mdx b/docs/traces/terminal.mdx new file mode 100644 index 0000000..d11ceea --- /dev/null +++ b/docs/traces/terminal.mdx @@ -0,0 +1,12 @@ +# terminal + +import { KindReference } from '../_components/KindReference'; + +Records terminal events supplied by a host such as a test runner. Arming this kind for a normal command does not feed events into it: the command's output is stored separately as the run's output. + +```sh +telltale run terminal -- true +``` + +The example arms an empty terminal stream. A host feeding events can configure `--width`, `--height`, `--max-bytes`, `--snapshot-interval`, and the command named in the cast header. The supported seam is `self`. Run `telltale run terminal --help` for all options. + diff --git a/docs/vitest.config.ts b/docs/vitest.config.ts new file mode 100644 index 0000000..293a48f --- /dev/null +++ b/docs/vitest.config.ts @@ -0,0 +1,3 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ test: { include: ['docs/_components/**/*.test.tsx'] } }); diff --git a/examples/definitions/README.md b/examples/definitions/README.md new file mode 100644 index 0000000..df45cc2 --- /dev/null +++ b/examples/definitions/README.md @@ -0,0 +1,25 @@ +# Application and build trace definitions + +Each YAML file is a complete `TraceDefinition`. Pass its path to `telltale run --definition`, or copy it to `.telltale/traces/.yaml` in your project to select it by name. Run build commands in the target project's directory and adjust the definition path to point to your copy. + +| Runtime | Application | Build | +|---|---|---| +| Java | [java-app.yaml](java-app.yaml): Flight Recorder samples, thread identities, process resources and descriptors | [java-build.yaml](java-build.yaml): Gradle/Maven process tree and resource samples | +| Go | [go-app.yaml](go-app.yaml): gops goroutine dumps, heap profiles, process resources and descriptors | [go-build.yaml](go-build.yaml): compiler, linker and cgo process tree and resource samples | +| Node | [node-app.yaml](node-app.yaml): process tree, resources, open files and network sockets | [node-build.yaml](node-build.yaml): bundler, type checker and worker process tree and resource samples | + +```sh +telltale definitions validate ./examples/definitions/java-app.yaml +telltale run --definition ./examples/definitions/java-app.yaml --pid 1234 --duration 1m +telltale run --definition ./examples/definitions/go-app.yaml --pid 1234 --duration 1m +telltale run --definition ./examples/definitions/node-app.yaml -- node server.js +telltale run --definition ./examples/definitions/java-build.yaml -- ./gradlew --no-daemon build +telltale run --definition ./examples/definitions/go-build.yaml -- go build ./... +telltale run --definition ./examples/definitions/node-build.yaml -- pnpm run build +``` + +Replace `1234` with the application's PID. Java profiling requires a compatible JDK's `jcmd` on `PATH`, Flight Recorder support and permission to attach to that JVM; see [JVM capture prerequisites](../../docs/sqlserver-jvm.md#local-jvm). The Go application must already run `github.com/google/gops/agent`, with its port file readable by Telltale. Both app definitions read checkpoints every 10 seconds; `--every` overrides this. The Node definition uses OS process observation and does not collect V8 heap or CPU profiles. + +All six definitions collect process resources and descriptor facets using the host's process tools, including `lsof`. They restrict captured environment variables to the listed runtime settings. Builds also sample CPU, memory and disk IO every 250 milliseconds, and finish when the command exits. Sampling can miss processes that exit between observations. Use Gradle's `--no-daemon` so the build does not reuse a daemon outside the traced tree; likewise, detached workers from an earlier build are outside the captured descendants. + +To inspect results, run `telltale trace list`, then `telltale serve` with the same data directory and open the recorded run in the web UI. See [checkout.yaml](checkout.yaml) for proxies, connected agents, metrics and queries, or [logs-and-queries.yaml](logs-and-queries.yaml) for a host-only run. diff --git a/examples/definitions/checkout.yaml b/examples/definitions/checkout.yaml index e3c8f9c..df82c98 100644 --- a/examples/definitions/checkout.yaml +++ b/examples/definitions/checkout.yaml @@ -78,7 +78,7 @@ spec: - {} resources: - interval: 1s - facets: [ports] + facets: [sockets] logs: kubernetes: - namespace: checkout diff --git a/examples/definitions/go-app.yaml b/examples/definitions/go-app.yaml new file mode 100644 index 0000000..e7ea7fe --- /dev/null +++ b/examples/definitions/go-app.yaml @@ -0,0 +1,23 @@ +# Attach to the compiled application's PID, not the go run launcher: +# telltale run --definition ./examples/definitions/go-app.yaml --pid 1234 --duration 1m +# The application must start github.com/google/gops/agent; its port file must +# be readable under GOPS_CONFIG_DIR or the user's default gops config directory. +apiVersion: telltale.flanksource.com/v1alpha1 +kind: TraceDefinition +metadata: + name: go-app +spec: + description: Read a Go application's goroutines and heap over gops alongside its process tree + every: 10s + collect: + go: + gops: + - {} + pprof: + - source: gops + profile: heap + process: + tree: + facets: true + environment: + keys: [GOMAXPROCS, GOMEMLIMIT, GOGC, GOPS_CONFIG_DIR] diff --git a/examples/definitions/go-build.yaml b/examples/definitions/go-build.yaml new file mode 100644 index 0000000..f47a949 --- /dev/null +++ b/examples/definitions/go-build.yaml @@ -0,0 +1,17 @@ +# Run from the Go project's directory, adjusting the definition path: +# telltale run --definition ./examples/definitions/go-build.yaml -- go build ./... +# No gops agent or pprof endpoint is needed in the compiler or linker. +apiVersion: telltale.flanksource.com/v1alpha1 +kind: TraceDefinition +metadata: + name: go-build +spec: + description: Follow a Go build's compiler linker and cgo processes with CPU memory and disk IO samples + collect: + process: + tree: + facets: true + environment: + keys: [GOOS, GOARCH, CGO_ENABLED, GOMAXPROCS, GOMEMLIMIT] + resources: + - interval: 250ms diff --git a/examples/definitions/java-app.yaml b/examples/definitions/java-app.yaml new file mode 100644 index 0000000..87e1844 --- /dev/null +++ b/examples/definitions/java-app.yaml @@ -0,0 +1,22 @@ +# Attach to the application's JVM, not a Maven/Gradle launcher: +# telltale run --definition ./examples/definitions/java-app.yaml --pid 1234 --duration 1m +# Requires a compatible JDK's jcmd on PATH and permission to attach to the JVM. +apiVersion: telltale.flanksource.com/v1alpha1 +kind: TraceDefinition +metadata: + name: java-app +spec: + description: Profile a Java application with Flight Recorder and inspect its process tree + every: 10s + instrument: + jvm: + flightRecorder: + - settings: profile + threads: true + maxStackDepth: 128 + collect: + process: + tree: + facets: true + environment: + keys: [JAVA_HOME, LANG, TZ] diff --git a/examples/definitions/java-build.yaml b/examples/definitions/java-build.yaml new file mode 100644 index 0000000..381da05 --- /dev/null +++ b/examples/definitions/java-build.yaml @@ -0,0 +1,18 @@ +# Run from the Java project's directory, adjusting the definition path: +# telltale run --definition ./examples/definitions/java-build.yaml -- ./gradlew --no-daemon build +# Or: telltale run --definition ./examples/definitions/java-build.yaml -- mvn package +# --no-daemon avoids reusing a Gradle daemon outside the traced process tree. +apiVersion: telltale.flanksource.com/v1alpha1 +kind: TraceDefinition +metadata: + name: java-build +spec: + description: Follow a Java build's compiler and worker processes with CPU memory and disk IO samples + collect: + process: + tree: + facets: true + environment: + keys: [JAVA_HOME, GRADLE_USER_HOME, LANG] + resources: + - interval: 250ms diff --git a/examples/definitions/node-app.yaml b/examples/definitions/node-app.yaml new file mode 100644 index 0000000..69a0fef --- /dev/null +++ b/examples/definitions/node-app.yaml @@ -0,0 +1,16 @@ +# Run from the Node application's directory, adjusting the definition path: +# telltale run --definition ./examples/definitions/node-app.yaml -- node server.js +# Or attach: telltale run --definition ./examples/definitions/node-app.yaml --pid 1234 --duration 1m +# This records OS resources and descriptors; it does not collect V8 profiles. +apiVersion: telltale.flanksource.com/v1alpha1 +kind: TraceDefinition +metadata: + name: node-app +spec: + description: Follow a Node application's processes CPU memory disk IO open files and network sockets + collect: + process: + tree: + facets: true + environment: + keys: [NODE_ENV, UV_THREADPOOL_SIZE, LANG, TZ] diff --git a/examples/definitions/node-build.yaml b/examples/definitions/node-build.yaml new file mode 100644 index 0000000..5ae8823 --- /dev/null +++ b/examples/definitions/node-build.yaml @@ -0,0 +1,18 @@ +# Run from the Node project's directory, adjusting the definition path: +# telltale run --definition ./examples/definitions/node-build.yaml -- pnpm run build +# Or: telltale run --definition ./examples/definitions/node-build.yaml -- npm run build +# Keep build workers under this command; an existing detached daemon is outside its tree. +apiVersion: telltale.flanksource.com/v1alpha1 +kind: TraceDefinition +metadata: + name: node-build +spec: + description: Follow a Node build's bundler type checker and worker processes with CPU memory and disk IO samples + collect: + process: + tree: + facets: true + environment: + keys: [NODE_ENV, CI, UV_THREADPOOL_SIZE] + resources: + - interval: 250ms diff --git a/examples/deployers/orders.yaml b/examples/deployers/orders.yaml new file mode 100644 index 0000000..4b71cba --- /dev/null +++ b/examples/deployers/orders.yaml @@ -0,0 +1,52 @@ +# Deployers put telltale's JVM agent into the pods a selector picks. Install +# crds/telltale.flanksource.com_deployers.yaml first. The issuer connection +# holds the RSA signing key as its `signingKey` named secret; agents get only +# its public JWKS, never a shared token. +apiVersion: telltale.flanksource.com/v1alpha1 +kind: Deployer +metadata: + name: orders + namespace: orders +spec: + selector: + kind: StatefulSet + namespace: orders + labels: {app: orders} + container: tomcat + javaContainer: + kind: tomcat + # The Tomcat base URL the deploy uses per pod: the manager upload and + # /health. The apiserver proxy deletes Authorization, so the manager login + # needs a port-forward (or a direct URL); proxy:// is refused here. + url: 'portforward://{{.Pod.Name}}.{{.Pod.Namespace}}:{{.Port "http"}}?kind=pod' + # Executions may go through the apiserver proxy: their token travels in + # X-Telltale-Authorization. The WAR's context path (default + # /telltale-diag) is appended to both URLs. + executionUrl: 'proxy://{{.Pod.Name}}.{{.Pod.Namespace}}:{{.Port "http"}}?kind=pod' + connection: connection://orders/tomcat-manager + runtimeDir: /usr/local/tomcat/temp/telltale-diag + auth: + issuer: connection://orders/telltale-signing + jwksUrl: https://telltale.example/.well-known/jwks.json +--- +apiVersion: telltale.flanksource.com/v1alpha1 +kind: Deployer +metadata: + name: orders-batch + namespace: orders +spec: + selector: + kind: Deployment + namespace: orders + labels: {app: orders-batch} + container: batch + javaContainer: + kind: attach + url: 'portforward://{{.Pod.Name}}.{{.Pod.Namespace}}:7070?kind=pod' + pid: 1 + directory: /tmp/telltale + bind: 0.0.0.0 + port: 7070 + service: orders-batch + auth: + issuer: connection://orders/telltale-signing diff --git a/examples/jvm-diagnostics.fixture.yaml b/examples/jvm-diagnostics.fixture.yaml new file mode 100644 index 0000000..02cc7d6 --- /dev/null +++ b/examples/jvm-diagnostics.fixture.yaml @@ -0,0 +1,354 @@ +kind: Fixture +name: Interactive JVM diagnostics and protocol captures +params: + type: object + properties: + interactive: + type: boolean + title: Keep the demo open after verification + default: true +defaults: + timeout: 2m +steps: + - id: prepare + kind: exec + cwd: &root .. + timeout: 5m + script: | + set -eu + test -x .bin/telltale + command -v java >/dev/null + command -v javac >/dev/null + command -v jcmd >/dev/null + command -v curl >/dev/null + ./sdk/java/gradlew -p sdk/java jvmDemo >&2 + mkdir -p .tmp/jvm-demo + demo_dir=$(mktemp -d "$PWD/.tmp/jvm-demo/run.XXXXXX") + printf '::notice::Demo artifacts: %s\n' "$demo_dir" + printf '%s' "$demo_dir" + export: + DEMO_DIR: '{{.outputs.stdout}}' + + - id: server + kind: services + services: + web: + cwd: *root + exec: exec .bin/telltale serve --host 127.0.0.1 --port {{.port}} --otlp --otlp-grpc-listen 127.0.0.1:0 --data-dir "$DEMO_DIR/data" > "$DEMO_DIR/server.log" 2>&1 + ready: + http: http://127.0.0.1:{{.port}}/health + wait: + timeout: 30s + + - id: runtime + kind: services + services: + workload: + cwd: *root + exec: | + set -eu + IFS= read -r demo_classpath < sdk/java/build/jvm-demo/classpath.txt + printf '%s\n' "$$" > "$DEMO_DIR/jvm.pid" + exec java -javaagent:sdk/java/build/jvm-demo/opentelemetry-javaagent.jar -cp "$demo_classpath" Workload {{.port}} "$DEMO_DIR" > "$DEMO_DIR/workload.log" 2>&1 + env: + OTEL_SERVICE_NAME: jvm-demo + OTEL_TRACES_EXPORTER: otlp + OTEL_METRICS_EXPORTER: none + OTEL_LOGS_EXPORTER: none + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: http/protobuf + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: '$WEB_URL/v1/traces' + OTEL_BSP_SCHEDULE_DELAY: '100' + OTEL_TRACES_SAMPLER: always_on + ready: + http: http://127.0.0.1:{{.port}}/health + wait: + timeout: 1m + export: + JVM_PID: '{{.outputs.pid}}' + + - id: attach + kind: exec + cwd: *root + exec: .bin/telltale + args: [agents, install, java, --pid, '$JVM_PID', --directory, '$DEMO_DIR/agent', --extensions-dir, '$DEMO_DIR/extensions', --service, jvm-demo, --allow-mutation, --format, json] + cel: exitCode == 0 && json.agent != "" + export: + DEMO_AGENT: '{{.outputs.json.agent}}' + + - id: groovy + kind: telltale + server: '$WEB_URL' + agent: '{{.env.DEMO_AGENT}}' + operation: jvm.groovy.eval + allowMutation: true + params: + source: 'println "hello"; return 3 + 4' + cel: exitCode == 0 && json.value == 7 && json.stdout.contains("hello") + + - id: methods + kind: telltale + server: '$WEB_URL' + agent: '{{.env.DEMO_AGENT}}' + capture: jvm_trace + duration: 5s + allowMutation: true + params: + targets: [{class: Workload, method: compute}] + args: true + return: true + events: 10 + wait: &captureWait + until: status == "succeeded" + failWhen: status == "failed" + interval: 200ms + timeout: 15s + cel: exitCode == 0 && json.captures.exists(c, c.kind == "jvm_trace" && c.summary.count == 10) + + - id: restored + kind: telltale + server: '$WEB_URL' + agent: '{{.env.DEMO_AGENT}}' + operation: jvm.classes.patches + cel: exitCode == 0 && size(json) == 0 + + - id: catalog + kind: telltale + server: '$WEB_URL' + agent: '{{.env.DEMO_AGENT}}' + operation: jvm.catalog + cel: >- + exitCode == 0 + && stdout.contains('"name":"jvm.groovy.eval"') + && stdout.contains('"name":"jvm.classes.source"') + + - id: compilePreview + kind: telltale + server: '$WEB_URL' + agent: '{{.env.DEMO_AGENT}}' + operation: jvm.classes.compile + params: &replacement + class: Workload + method: compute + source: 'public static double compute(long value) { return 7.0; }' + cel: json.success == true + + - id: redefine + kind: telltale + server: '$WEB_URL' + agent: '{{.env.DEMO_AGENT}}' + capture: jvm_redefine + duration: 1s + allowMutation: true + params: *replacement + wait: *captureWait + cel: json.captures.exists(c, c.kind == "jvm_redefine" && c.summary.count > 0) + + - id: originalMethod + kind: telltale + server: '$WEB_URL' + agent: '{{.env.DEMO_AGENT}}' + operation: jvm.ognl.eval + allowMutation: true + params: + expression: '@Workload@compute(1L)' + cel: json.value > 1.414 && json.value < 1.415 + + - id: httpCapture + kind: telltale + server: '$WEB_URL' + agent: '{{.env.DEMO_AGENT}}' + capture: http + duration: 1h + allowMutation: true + params: + features: [http] + direction: both + paths: [/demo/*] + bodies: 4KiB + wait: &armed + until: status == "running" + failWhen: status == "failed" + interval: 200ms + timeout: 15s + + - id: sqlCapture + kind: telltale + server: '$WEB_URL' + agent: '{{.env.DEMO_AGENT}}' + capture: sql + duration: 1h + allowMutation: true + params: + connections: ['*'] + wait: *armed + + - id: jmsCapture + kind: telltale + server: '$WEB_URL' + agent: '{{.env.DEMO_AGENT}}' + capture: jms + duration: 1h + allowMutation: true + params: + direction: both + destinations: [demo.orders] + body: true + maxBodyBytes: 4096 + wait: *armed + + - id: spansCapture + kind: telltale + server: '$WEB_URL' + agent: '{{.env.DEMO_AGENT}}' + capture: spans + duration: 1h + allowMutation: true + wait: *armed + + - id: httpTop + kind: telltale + server: '$WEB_URL' + agent: '{{.env.DEMO_AGENT}}' + capture: http_top + duration: 1h + allowMutation: true + params: + direction: both + paths: [/demo/*] + poll: 1s + top: 50 + wait: *armed + + - id: sqlTop + kind: telltale + server: '$WEB_URL' + agent: '{{.env.DEMO_AGENT}}' + capture: sql_top + duration: 1h + allowMutation: true + params: + text: [DEMO_HOLD] + poll: 1s + top: 50 + wait: *armed + + - id: holdOperations + kind: exec + exec: curl + args: [-fsS, --max-time, '30', -X, POST, '$WORKLOAD_URL/control/hold'] + cel: json.holding && json.httpEntered == 2 && json.jdbcEntered && json.r2dbcEntered && json.jmsEntered + + - id: activeSnapshots + kind: exec + cwd: *root + exec: go + args: &checkArgs + - run + - ./hack/jvm-demo + - --server + - '$WEB_URL' + - --workload + - '$WORKLOAD_URL' + - --agent + - '{{.env.DEMO_AGENT}}' + - --http-top + - '{{.httpTop.runId}}' + - --sql-top + - '{{.sqlTop.runId}}' + - --http + - '{{.httpCapture.runId}}' + - --sql + - '{{.sqlCapture.runId}}' + - --jms + - '{{.jmsCapture.runId}}' + - --spans + - '{{.spansCapture.runId}}' + - --artifacts + - '$DEMO_DIR/evidence' + - --phase + - '$CHECK_PHASE' + env: {CHECK_PHASE: active} + cel: exitCode == 0 && json.phase == "active" + + - id: updatedSnapshots + kind: exec + cwd: *root + exec: go + args: *checkArgs + env: {CHECK_PHASE: updated} + cel: exitCode == 0 && json.phase == "updated" + + - id: releaseOperations + kind: exec + exec: curl + args: [-fsS, --max-time, '30', -X, POST, '$WORKLOAD_URL/control/release'] + cel: '!json.holding && json.completedCycles > holdOperations.json.completedCycles' + + - id: clearedSnapshots + kind: exec + cwd: *root + exec: go + args: *checkArgs + env: {CHECK_PHASE: cleared} + cel: exitCode == 0 && json.phase == "cleared" + + - id: completedProtocols + kind: exec + cwd: *root + exec: go + args: *checkArgs + env: {CHECK_PHASE: completed} + cel: exitCode == 0 && json.phase == "completed" + + - id: interactiveTraffic + kind: exec + if: params.interactive + exec: curl + args: [-fsS, --max-time, '10', -X, POST, '$WORKLOAD_URL/control/traffic?enabled=true'] + cel: json.trafficEnabled + + - id: exitDemo + kind: form + if: params.interactive + timeout: 55m + title: Exit JVM diagnostics demo + description: | + Diagnostics and protocol checks passed, including HTTP/JMS directions, + JDBC/R2DBC, existing OpenTelemetry exporter mirroring and Profiles Top + active/update/clear snapshots. Local Spring traffic is running. + Keep this form open while exploring: + {{.env.WEB_URL}}/jvm/{{.env.DEMO_AGENT}} + + Dedicated capture workspaces: + {{.env.WEB_URL}}/jvm/{{.env.DEMO_AGENT}}/instrument?class=Workload&method=compute + {{.env.WEB_URL}}/jvm/{{.env.DEMO_AGENT}}/traces + {{.env.WEB_URL}}/jvm/{{.env.DEMO_AGENT}}/notifications + + Live protocol captures and Top snapshots: + HTTP Top: {{.env.WEB_URL}}/?viewer=1&run={{.httpTop.runId}} + SQL Top: {{.env.WEB_URL}}/?viewer=1&run={{.sqlTop.runId}} + HTTP: {{.env.WEB_URL}}/?viewer=1&run={{.httpCapture.runId}} + SQL: {{.env.WEB_URL}}/?viewer=1&run={{.sqlCapture.runId}} + JMS: {{.env.WEB_URL}}/?viewer=1&run={{.jmsCapture.runId}} + Mirrored spans: {{.env.WEB_URL}}/?viewer=1&run={{.spansCapture.runId}} + Received traces: {{.env.WEB_URL}}/otel + + Workload state: {{.env.WORKLOAD_URL}}/control/state + + Try JMX Memory attributes and gc(), Workload class/method source, + compile-preview and short redefinition captures, Threads and Metrics. + The Extensions install operation accepts sdk/java/build/libs/telltale-example-extension.jar. + + Choose Exit and submit to stop the demo server and JVM. + Cancelling also stops both processes. Artifacts remain in: + {{.env.DEMO_DIR}} + schema: + type: object + required: [action] + properties: + action: + type: string + title: Action + enum: [exit] + default: + action: exit diff --git a/examples/jvm-snapshot.fixture.md b/examples/jvm-snapshot.fixture.md new file mode 100644 index 0000000..18bd4ab --- /dev/null +++ b/examples/jvm-snapshot.fixture.md @@ -0,0 +1,26 @@ +--- +cwd: .. +timeout: 15m +--- + +# JVM support bundle acceptance + +Build and live checks run sequentially to keep the binary and embedded assets consistent. + +### command: Snapshot acceptance + +```yaml +timeout: 1200 +``` + +```bash +set -e +make build +./sdk/java/gradlew -p sdk/java test --tests com.flanksource.telltale.SettingsTest --tests com.flanksource.telltale.diag.jvm.ResourceSettingsTest --tests com.flanksource.telltale.diag.jvm.NativeTelemetryServiceTest --tests com.flanksource.telltale.CatalogSnapshotTest +go test ./internal/jvm/snapshot -count=1 +go test ./internal/cli ./internal/server -ginkgo.focus='JVM command tree|HTTP routing' +pnpm exec vitest run webapp/src/jvm/JvmSnapshot.test.tsx packages/api/test/agents.spec.ts +make lint +``` + +- cel: exitCode == 0 diff --git a/go.mod b/go.mod index 54acc40..20c04c2 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/flanksource/telltale -go 1.27.1 +go 1.27.4 require ( charm.land/bubbletea/v2 v2.0.2 @@ -13,52 +13,56 @@ require ( github.com/charmbracelet/x/ansi v0.11.6 github.com/creack/pty v1.1.24 github.com/elazarl/goproxy v1.7.2 - github.com/flanksource/captain v0.0.73 - github.com/flanksource/clicky v1.21.90 - github.com/flanksource/commons v1.63.0 - github.com/flanksource/commons-db v0.1.70 + github.com/flanksource/captain v0.0.83 + github.com/flanksource/clicky v1.21.95 + github.com/flanksource/commons v1.64.1 + github.com/flanksource/commons-db v0.1.72 github.com/flanksource/commons-db/cmd/query v0.1.70 - github.com/flanksource/commons-db/recordstore v0.1.70 - github.com/flanksource/commons-db/tracing v0.1.70 - github.com/flanksource/deps/start v0.0.0-20261028154603-ac4c84178264 - github.com/flanksource/gomplate/v3 v3.24.90 + github.com/flanksource/commons-db/recordstore v0.1.72 + github.com/flanksource/deps v1.0.51 + github.com/flanksource/gavel/sdk v0.0.0-20261108204313-d40d4b480181 + github.com/flanksource/gomplate/v3 v3.24.91 github.com/flanksource/sqlglot-go v1.1.1 - github.com/flanksource/telltale/sdk v0.0.0 - github.com/flanksource/telltale/sdk/captain v0.0.0 - github.com/flanksource/telltale/sdk/clicky v0.0.0 - github.com/flanksource/telltale/sdk/commonsdb v0.0.0 + github.com/flanksource/telltale/sdk v0.0.2 + github.com/flanksource/telltale/sdk/captain v0.0.2 + github.com/flanksource/telltale/sdk/clicky v0.0.2 + github.com/flanksource/telltale/sdk/commonsdb v0.0.2 + github.com/gofrs/flock v0.13.0 + github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/cel-go v0.31.0 github.com/google/gops v0.3.29 github.com/google/pprof v0.0.0-20260507013755-92041b743c96 - github.com/google/uuid v1.6.0 + github.com/google/uuid v1.6.1-0.20241114170450-2d3c2a9cc518 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 github.com/grafana/jfr-parser v0.19.0 github.com/jackc/pgx/v5 v5.10.0 github.com/klauspost/compress v1.18.7 github.com/maruel/panicparse/v2 v2.5.0 github.com/microsoft/go-mssqldb v1.10.0 - github.com/onsi/ginkgo/v2 v2.32.1 - github.com/onsi/gomega v1.42.1 + github.com/onsi/ginkgo/v2 v2.33.1 + github.com/onsi/gomega v1.44.0 github.com/pierrec/lz4/v4 v4.1.26 github.com/shirou/gopsutil/v4 v4.26.7 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 + github.com/stretchr/testify v1.12.1 github.com/ulikunitz/xz v0.5.15 go.opentelemetry.io/collector/pdata v1.69.0 + go.opentelemetry.io/collector/pdata/pprofile v0.163.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 - go.opentelemetry.io/otel v1.45.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 - go.opentelemetry.io/otel/metric v1.45.0 - go.opentelemetry.io/otel/sdk v1.45.0 - go.opentelemetry.io/otel/trace v1.45.0 + go.opentelemetry.io/otel v1.47.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 + go.opentelemetry.io/otel/metric v1.47.0 + go.opentelemetry.io/otel/sdk v1.47.0 + go.opentelemetry.io/otel/trace v1.47.0 golang.org/x/crypto v0.58.0 - golang.org/x/mod v0.41.0 + golang.org/x/mod v0.42.0 golang.org/x/sync v0.24.0 golang.org/x/sys v0.49.0 golang.org/x/term v0.47.0 golang.org/x/text v0.43.0 - google.golang.org/grpc v1.84.0 + google.golang.org/grpc v1.85.0 google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/sqlserver v1.6.4 @@ -67,12 +71,14 @@ require ( k8s.io/apiextensions-apiserver v0.36.2 k8s.io/apimachinery v0.36.2 k8s.io/client-go v0.36.2 + k8s.io/streaming v0.36.2 sigs.k8s.io/yaml v1.6.0 ) require ( ariga.io/atlas v0.38.0 // indirect - cel.dev/expr v0.25.2 // indirect + cel.dev/cel-go v0.33.2 // indirect + cel.dev/expr v0.25.3 // indirect cloud.google.com/go v0.123.0 // indirect cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect @@ -90,11 +96,10 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect - github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 // indirect github.com/ClickHouse/ch-go v0.71.0 // indirect github.com/ClickHouse/clickhouse-go/v2 v2.46.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect @@ -108,11 +113,14 @@ require ( github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b // indirect github.com/alecthomas/chroma/v2 v2.23.1 // indirect github.com/alpkeskin/gotoon v0.1.1 // indirect - github.com/antchfx/xmlquery v1.5.1 // indirect - github.com/antchfx/xpath v1.3.6 // indirect + github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect + github.com/antchfx/xmlquery v1.5.2 // indirect + github.com/antchfx/xpath v1.3.9 // indirect + github.com/anthropics/anthropic-sdk-go v1.26.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/apache/arrow/go/v15 v15.0.2 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect + github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aws/aws-sdk-go-v2 v1.42.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 // indirect @@ -143,9 +151,7 @@ require ( github.com/bmatcuk/doublestar v1.3.4 // indirect github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect - github.com/caarlos0/env/v11 v11.3.1 // indirect github.com/catppuccin/go v0.3.0 // indirect - github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 // indirect @@ -155,7 +161,7 @@ require ( github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 // indirect github.com/charmbracelet/ultraviolet v0.0.0-20260205113103-524a6607adb8 // indirect github.com/charmbracelet/x/cellbuf v0.0.15 // indirect - github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect + github.com/charmbracelet/x/exp/strings v0.0.0-20260122224438-b01af16209d9 // indirect github.com/charmbracelet/x/term v0.2.2 // indirect github.com/charmbracelet/x/termios v0.1.1 // indirect github.com/charmbracelet/x/windows v0.2.2 // indirect @@ -163,23 +169,12 @@ require ( github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/cloudflare/circl v1.6.3 // indirect github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect - github.com/containerd/errdefs v1.0.0 // indirect - github.com/containerd/errdefs/pkg v0.3.0 // indirect - github.com/containerd/log v0.1.0 // indirect - github.com/containerd/platforms v0.2.1 // indirect + github.com/coder/websocket v1.8.14 // indirect github.com/cyphar/filepath-securejoin v0.6.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deckarep/golang-set/v2 v2.8.0 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect - github.com/docker/go-connections v0.7.0 // indirect - github.com/docker/go-sdk/client v0.1.0-alpha013 // indirect - github.com/docker/go-sdk/config v0.1.0-alpha013 // indirect - github.com/docker/go-sdk/container v0.1.0-alpha016 // indirect - github.com/docker/go-sdk/context v0.1.0-alpha013 // indirect - github.com/docker/go-sdk/image v0.1.0-alpha015 // indirect - github.com/docker/go-sdk/network v0.1.0-alpha013 // indirect - github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.10.2 // indirect github.com/eko/gocache/lib/v4 v4.2.3 // indirect @@ -192,20 +187,22 @@ require ( github.com/exaring/otelpgx v0.11.1 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect github.com/fergusstrange/embedded-postgres v1.34.0 // indirect - github.com/flanksource/clicky/valkey v1.21.89 // indirect - github.com/flanksource/deps v1.0.49 // indirect + github.com/firebase/genkit/go v1.13.1 // indirect + github.com/flanksource/clicky/valkey v1.21.94 // indirect + github.com/flanksource/gomplate/v4 v4.0.0-20261111091247-6aab4d0463bd // indirect github.com/flanksource/is-healthy v1.0.93 // indirect github.com/flanksource/kubectl-neat v1.0.4 // indirect + github.com/flanksource/sandbox-runtime v1.0.2 // indirect github.com/fxamacker/cbor/v2 v2.9.1 // indirect github.com/geoffgarside/ber v1.2.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect - github.com/glebarez/sqlite v1.11.0 // indirect + github.com/gliderlabs/ssh v0.3.8 // indirect github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.7.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.9.0 // indirect github.com/go-git/go-git/v5 v5.19.2 // indirect - github.com/go-jose/go-jose/v4 v4.1.4 // indirect + github.com/go-jose/go-jose/v4 v4.1.9 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect @@ -231,8 +228,6 @@ require ( github.com/gobwas/glob v0.2.3 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/goccy/go-yaml v1.19.2 // indirect - github.com/gofrs/flock v0.13.0 // indirect - github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect @@ -243,12 +238,13 @@ require ( github.com/google/go-cmp v0.7.0 // indirect github.com/google/go-github/v57 v57.0.0 // indirect github.com/google/go-querystring v1.2.0 // indirect - github.com/google/jsonschema-go v0.4.2 // indirect + github.com/google/jsonschema-go v0.4.3 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.17 // indirect github.com/googleapis/gax-go/v2 v2.22.0 // indirect github.com/gosimple/slug v1.15.0 // indirect github.com/gosimple/unidecode v1.0.1 // indirect + github.com/govalues/decimal v0.1.36 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/hairyhenderson/toml v0.4.2-0.20210923231440-40456b8e66cf // indirect github.com/hairyhenderson/yaml v0.0.0-20220618171115-2d35fca545ce // indirect @@ -276,7 +272,7 @@ require ( github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/kr/fs v0.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/labstack/echo/v4 v4.16.3 // indirect + github.com/labstack/echo/v4 v4.16.4 // indirect github.com/labstack/gommon v0.5.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect @@ -293,16 +289,7 @@ require ( github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/go-archive v0.1.0 // indirect - github.com/moby/moby/api v1.52.0 // indirect - github.com/moby/moby/client v0.1.0 // indirect - github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/spdystream v0.5.1 // indirect - github.com/moby/sys/sequential v0.6.0 // indirect - github.com/moby/sys/user v0.4.0 // indirect - github.com/moby/sys/userns v0.1.0 // indirect - github.com/moby/term v0.5.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect @@ -313,13 +300,13 @@ require ( github.com/ncruces/go-strftime v1.0.0 // indirect github.com/nukilabs/ftoa v1.0.0 // indirect github.com/nukilabs/unicodeid v0.1.0 // indirect - github.com/ohler55/ojg v1.28.1 // indirect - github.com/oklog/ulid/v2 v2.1.1 // indirect + github.com/ohler55/ojg v1.28.9 // indirect + github.com/oklog/ulid/v2 v2.1.2 // indirect + github.com/openai/openai-go v1.12.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opensearch-project/opensearch-go/v2 v2.3.0 // indirect github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect - github.com/patrickmn/go-cache v2.1.0+incompatible // indirect + github.com/patrickmn/go-cache v2.1.1-0.20191004192108-46f407853014+incompatible // indirect github.com/paulmach/orb v0.12.0 // indirect github.com/pb33f/ordered-map/v2 v2.3.1 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect @@ -341,7 +328,7 @@ require ( github.com/robertkrimen/otto v0.5.1 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/samber/lo v1.53.0 // indirect - github.com/samber/oops v1.22.0 // indirect + github.com/samber/oops v1.23.2 // indirect github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 // indirect github.com/segmentio/asm v1.2.1 // indirect github.com/segmentio/encoding v0.5.4 // indirect @@ -351,7 +338,6 @@ require ( github.com/skeema/knownhosts v1.3.2 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect - github.com/stretchr/testify v1.12.1 // indirect github.com/t14raptor/go-fast v0.1.0 // indirect github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.2.0 // indirect @@ -362,10 +348,11 @@ require ( github.com/tj/go-naturaldate v1.3.0 // indirect github.com/tklauser/go-sysconf v0.3.16 // indirect github.com/tklauser/numcpus v0.11.0 // indirect - github.com/ugorji/go/codec v1.3.1 // indirect + github.com/ugorji/go/codec v1.3.2 // indirect github.com/valkey-io/valkey-go v1.0.75 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect + github.com/woodsbury/decimal128 v1.4.1 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect @@ -374,7 +361,7 @@ require ( github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/xuri/efp v0.0.3-0.20260924042702-217b81952dec // indirect - github.com/xuri/excelize/v2 v2.11.1-0.20261029001137-c5fd73bde8f6 // indirect + github.com/xuri/excelize/v2 v2.11.1 // indirect github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yuin/goldmark v1.7.17 // indirect @@ -387,10 +374,11 @@ require ( go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/collector/featuregate v1.69.0 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.45.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect + go.opentelemetry.io/otel/log v1.47.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.47.0 // indirect go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect @@ -400,14 +388,15 @@ require ( golang.org/x/exp v0.0.0-20260527015227-08cc5374adb3 // indirect golang.org/x/net v0.62.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/telemetry v0.0.0-20260924152758-ed294f943157 // indirect + golang.org/x/telemetry v0.0.0-20261009151841-97e8ea77aa2a // indirect golang.org/x/time v0.15.0 // indirect - golang.org/x/tools v0.51.0 // indirect + golang.org/x/tools v0.52.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/api v0.286.0 // indirect + google.golang.org/genai v1.57.0 // indirect google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect @@ -418,7 +407,6 @@ require ( k8s.io/component-base v0.36.2 // indirect k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260501160325-927ab1f70cd6 // indirect - k8s.io/streaming v0.36.2 // indirect k8s.io/utils v0.0.0-20260626114624-be93311217bd // indirect layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf // indirect modernc.org/libc v1.74.1 // indirect diff --git a/go.sum b/go.sum index 641eb8b..50bac77 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,9 @@ ariga.io/atlas v0.38.0 h1:MwbtwVtDWJFq+ECyeTAz2ArvewDnpeiw/t/sgNdDsdo= ariga.io/atlas v0.38.0/go.mod h1:D7XMK6ei3GvfDqvzk+2VId78j77LdqHrqPOWamn51/s= -cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= -cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cel.dev/cel-go v0.33.2 h1:eAcEdBguc6AFrGbPOyE0g168Uey4kKFW9J6qsBLSeMU= +cel.dev/cel-go v0.33.2/go.mod h1:DnVip7tpJSsgZymwfT+m1tnEVy3ivAjSMXPx12YrMkU= +cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0= +cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= charm.land/bubbletea/v2 v2.0.2 h1:4CRtRnuZOdFDTWSff9r8QFt/9+z6Emubz3aDMnf/dx0= charm.land/bubbletea/v2 v2.0.2/go.mod h1:3LRff2U4WIYXy7MTxfbAQ+AdfM3D8Xuvz2wbsOD9OHQ= charm.land/lipgloss/v2 v2.0.2 h1:xFolbF8JdpNkM2cEPTfXEcW1p6NRzOWTSamRfYEw8cs= @@ -39,8 +41,6 @@ dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 h1:zvXfGJCWvywnCA814d8ZiVyt+fm9nnTE8xSb99zRyfo= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 h1:u93s+zU2JD62im61Bm5CZIc1ZrOJaIAWEg0WOrMVkEo= @@ -59,8 +59,6 @@ github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.4.0 h1:E4MgwLB github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.4.0/go.mod h1:Y2b/1clN4zsAoUd/pgNAQHjLDnTis/6ROkUfyob6psM= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA= -github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= -github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI= @@ -72,8 +70,8 @@ github.com/ClickHouse/clickhouse-go/v2 v2.46.0 h1:s3eRy+hYmu5uzotB6ZhDofgHu8kDgG github.com/ClickHouse/clickhouse-go/v2 v2.46.0/go.mod h1:giJfUVlMkcfUEPVfRpt51zZaGEx9i17gCos8gBl392c= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0 h1:yzIYdwuro811Z27D3T80Wkd3rqZzb0K43nner7Eh1yE= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 h1:bN1gA3of5bXtbnLsRPrwfmbbe7A5UWFlcTHseujLnpc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0/go.mod h1:Yj5vHEz/aAepZGliRJsA6uvHAVAQyEwajq9ORCHPxzM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ= @@ -117,10 +115,12 @@ github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwTo github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= -github.com/antchfx/xmlquery v1.5.1 h1:T9I4Ns1EXiWHy0IqKupGhnfTQtJwlGrpXtauYOoNv78= -github.com/antchfx/xmlquery v1.5.1/go.mod h1:bVqnl7TaDXSReKINrhZz+2E/PbCu2tUahb+wZ7WZNT8= -github.com/antchfx/xpath v1.3.6 h1:s0y+ElRRtTQdfHP609qFu0+c6bglDv20pqOViQjjdPI= -github.com/antchfx/xpath v1.3.6/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= +github.com/antchfx/xmlquery v1.5.2 h1:RRKPmhm4gZOSSqDkbNVBHC+NwbE2grfPFssBI+/zkrg= +github.com/antchfx/xmlquery v1.5.2/go.mod h1:2H6ruVAe9IVOxkuhoWR60XHc62EDYWL56IqCMZPTzac= +github.com/antchfx/xpath v1.3.9 h1:muXJSod6jRzhsiUjvduNSxsRSptFZ2GOJ1oMD1QkPSA= +github.com/antchfx/xpath v1.3.9/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= +github.com/anthropics/anthropic-sdk-go v1.26.0 h1:oUTzFaUpAevfuELAP1sjL6CQJ9HHAfT7CoSYSac11PY= +github.com/anthropics/anthropic-sdk-go v1.26.0/go.mod h1:qUKmaW+uuPB64iy1l+4kOSvaLqPXnHTTBKH6RVZ7q5Q= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE= @@ -208,12 +208,8 @@ github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= -github.com/caarlos0/env/v11 v11.3.1 h1:cArPWC15hWmEt+gWk7YBi7lEXTXCvpaSdCiZE2X5mCA= -github.com/caarlos0/env/v11 v11.3.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= github.com/catppuccin/go v0.3.0 h1:d+0/YicIq+hSTo5oPuRi5kOpqkVA5tAsU6dNhvRu+aY= github.com/catppuccin/go v0.3.0/go.mod h1:8IHJuMGaUUjQM82qBrGNBv7LFq6JI3NnQCF6MOlZjpc= -github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= -github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -241,8 +237,8 @@ github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86 h1:JSt3B+U9 github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86/go.mod h1:2P0UgXMEa6TsToMSuFqKFQR+fZTO9CNGUNokkPatT/0= github.com/charmbracelet/x/exp/golden v0.0.0-20250806222409-83e3a29d542f h1:pk6gmGpCE7F3FcjaOEKYriCvpmIN4+6OS/RD0vm4uIA= github.com/charmbracelet/x/exp/golden v0.0.0-20250806222409-83e3a29d542f/go.mod h1:IfZAMTHB6XkZSeXUqriemErjAWCCzT0LwjKFYCZyw0I= -github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 h1:qko3AQ4gK1MTS/de7F5hPGx6/k1u0w4TeYmBFwzYVP4= -github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0/go.mod h1:pBhA0ybfXv6hDjQUZ7hk1lVxBiUbupdw5R31yPUViVQ= +github.com/charmbracelet/x/exp/strings v0.0.0-20260122224438-b01af16209d9 h1:JevRYfkTT0sN9OIXAOncYNC0cTP1Gml/0mCSnsmRkRk= +github.com/charmbracelet/x/exp/strings v0.0.0-20260122224438-b01af16209d9/go.mod h1:/ehtMPNh9K4odGFkqYJKpIYyePhdp1hLBRvyY4bWkH8= github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= github.com/charmbracelet/x/termios v0.1.1 h1:o3Q2bT8eqzGnGPOYheoYS8eEleT5ZVNYNy8JawjaNZY= @@ -263,14 +259,8 @@ github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= -github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= -github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= -github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= -github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= -github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= -github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= -github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g= +github.com/coder/websocket v1.8.14/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= @@ -292,22 +282,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= -github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= -github.com/docker/go-sdk/client v0.1.0-alpha013 h1:FtIFLXjwSkQuOA4k6eE4Erz1nfW0d5Aq/lxID/d4Lpw= -github.com/docker/go-sdk/client v0.1.0-alpha013/go.mod h1:6AWYuKRnvTXp+rWYHDrWUui0Aykgl8JdjttukngnKE0= -github.com/docker/go-sdk/config v0.1.0-alpha013 h1:AbVVhOjtuyzlireZ8kJrzd3Xlrqz6CafsBQjLGKOipA= -github.com/docker/go-sdk/config v0.1.0-alpha013/go.mod h1:Eq39AVh06RUJ50fLvJHFKM80P9kZWTiog1x7NzrndBg= -github.com/docker/go-sdk/container v0.1.0-alpha016 h1:qvE173H6pDykJl5zXufDhp0nK4HSPjkfd5gjA+fPjvM= -github.com/docker/go-sdk/container v0.1.0-alpha016/go.mod h1:O0mTuDUPrl6x8BwwL53k1Q0C6wDoUp4xAmE1uLzjszw= -github.com/docker/go-sdk/context v0.1.0-alpha013 h1:he+yf7bEAOvDdeRkkd+vNFHiGhmZV2oSx0j7s9lI7e8= -github.com/docker/go-sdk/context v0.1.0-alpha013/go.mod h1:a+3Mdx3Tss56tN5bajB3YfW29ZHdm6nenlB6yxIaeF4= -github.com/docker/go-sdk/image v0.1.0-alpha015 h1:y0KqSQadpLMURL0jgA8HO79M0nHwQXEjmalPxxAqNwo= -github.com/docker/go-sdk/image v0.1.0-alpha015/go.mod h1:ujOXFWntnGkARHfXiIK/nonx6+8eQLqjWMb9wL35s0I= -github.com/docker/go-sdk/network v0.1.0-alpha013 h1:vl0F5LZLmuF9yBwJ9/6kPJqHOgevSFHNK82t9a2tejQ= -github.com/docker/go-sdk/network v0.1.0-alpha013/go.mod h1:kgEPlBmaYjQwO+a+6IfQmLvo6BC9x8106TbJBBEXF6w= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/ebitengine/purego v0.10.2 h1:W809HbnvzAxgdm+aOvlSekrM16wGCdT/e76+9tS7gzE= @@ -342,36 +318,42 @@ github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeO github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/fergusstrange/embedded-postgres v1.34.0 h1:c6RKhPKFsLVU+Tdxsx8q0UxCHsvZZ/iShAnljRBXs6s= github.com/fergusstrange/embedded-postgres v1.34.0/go.mod h1:w0YvnCgf19o6tskInrOOACtnqfVlOvluz3hlNLY7tRk= -github.com/flanksource/captain v0.0.73 h1:kxV+OkhanWTr0DcnWeUlALHSSY8pn1FNg5cq39JBkuw= -github.com/flanksource/captain v0.0.73/go.mod h1:M6+b1SjeyS88NSrLZKrzuVt/LHUUH/hqzDCcwVKLKpY= -github.com/flanksource/clicky v1.21.90 h1:8xkBJAwaeVwK2H4O3sfXeZ4LPanNAfd8krfgI35J278= -github.com/flanksource/clicky v1.21.90/go.mod h1:aigyzJt9FSoGV1v7q9nb7D74vQJHaBYJJ3ShMwq8r34= -github.com/flanksource/clicky/valkey v1.21.89 h1:1OCDksleAUViqtn0KGPt533TH6yA0jvQc/wrEnPQj5w= -github.com/flanksource/clicky/valkey v1.21.89/go.mod h1:rPqFN4S/9KctKpCKaYJrjHpS97TWW6i4BYabB+t9Heg= -github.com/flanksource/commons v1.63.0 h1:gI5DecATKbay5Q1o1hYRxQUSnbC+yWP7BmdHQjrpvTY= -github.com/flanksource/commons v1.63.0/go.mod h1:RlVSQ+Jq96jKaJWf3WnwVO4DmDTdzkm3sremzJuYuVw= -github.com/flanksource/commons-db v0.1.70 h1:u5Ch8NCL9c2Gtqzn9OayKUkF6c03t5z7QLF9PE2Yxag= -github.com/flanksource/commons-db v0.1.70/go.mod h1:grmVCAlfxqk8OuQRi1Jmgs9k3QELNVCDDJl3I3EsNAQ= +github.com/firebase/genkit/go v1.13.1 h1:6fgQ0ogxG+SIgtYQD/yf8T0+39lacioB5voFdFYk1tI= +github.com/firebase/genkit/go v1.13.1/go.mod h1:nWewix7d2O+oikJ035XPmY4mrO2phXwmk6NekKmaDLU= +github.com/flanksource/captain v0.0.83 h1:QeHvs/OodVPethKOFpN6IuEiyrkoYngSaolRSWLL5Lc= +github.com/flanksource/captain v0.0.83/go.mod h1:6Hg3PYn8rARRpYpi8E+EWCO5kgFrEeLD6xNaexTCkpE= +github.com/flanksource/clicky v1.21.95 h1:cBuTKkvbWHjpDRswJ0iOPJzNT5QYiyqt31qmJOA7rBg= +github.com/flanksource/clicky v1.21.95/go.mod h1:H4HeUqMEzAz14ewqX5CNu/VnlaZHrTK/1otCgs1O1m4= +github.com/flanksource/clicky/valkey v1.21.94 h1:JPcksQ4qLy2MJnFL2QrfNoYHcouS2X0yr4t77qvb/Qo= +github.com/flanksource/clicky/valkey v1.21.94/go.mod h1:2uuZIuCWcLGd4TuehgVcUpKTvS6+2vx6gFeWlybgHlY= +github.com/flanksource/commons v1.64.1 h1:qm6ZnsoTw9JxzzH3R3WJs9kKig0+nfMBXJomWgCi/VU= +github.com/flanksource/commons v1.64.1/go.mod h1:XauWx50brQc4kK6t9OwK9JVMXy/Aq99mX6w6txASBag= +github.com/flanksource/commons-db v0.1.72 h1:N8r/W+uIOfwnQW9I4yZLoPhBOPPRHZJpf7RfZt8E6Ak= +github.com/flanksource/commons-db v0.1.72/go.mod h1:5Bs4q2IBmJxCJUwqJvL1df8g5ODVJ/jF6aELHb4UNMY= github.com/flanksource/commons-db/cmd/query v0.1.70 h1:ONcy93yi4wUYahI0LdoVmbp7FaWCOGkob9LDNj2f3eM= github.com/flanksource/commons-db/cmd/query v0.1.70/go.mod h1:tJGkIRrcKd0oA6C9/5yMLLi7UOTDf8qV4EeY+8JMoS0= -github.com/flanksource/commons-db/recordstore v0.1.70 h1:tuIrunjt6TaPIiGi/elWoTfPirY0nKYqsJ01yQTwaYU= -github.com/flanksource/commons-db/recordstore v0.1.70/go.mod h1:F1o5vrRFDnBYEFuD4enn2eN84V6XOFpIplbEVUL7tFk= -github.com/flanksource/commons-db/tracing v0.1.70 h1:+Kq9vNBJKpwUrn/uTiJhREiVuxhlXOG0fuwkZiSoX04= -github.com/flanksource/commons-db/tracing v0.1.70/go.mod h1:bNv3jEZzUzriVamZzjfpDPg10Ri6NbTW9E6yGZTlhMM= -github.com/flanksource/deps v1.0.49 h1:Lg9r8rd6HHSM4tvXwkxWk9QL7cLlTDt+B04FSCi2Oy8= -github.com/flanksource/deps v1.0.49/go.mod h1:/AdWJJhDH/Phs+NxR2RLiwM3sHgBzb+LNgqAwtcyYgU= -github.com/flanksource/deps/start v0.0.0-20261028154603-ac4c84178264 h1:ehw9wz74TiCvqKFgixIAJuYypXR5NH1u0Pvsu7QNkfg= -github.com/flanksource/deps/start v0.0.0-20261028154603-ac4c84178264/go.mod h1:2di5Ubu3eec1O3dAjFm2UgdRdomLw8WP7jMS3D0wjxI= -github.com/flanksource/gomplate/v3 v3.24.90 h1:MAUlwU7REhSL62i4n74yvo+T4LA7HuRpJ9tKH9dbMzk= -github.com/flanksource/gomplate/v3 v3.24.90/go.mod h1:tcjpBIaHboWVPUYt973dGLCWgTD1sOHdiFgA/AaPBPQ= +github.com/flanksource/commons-db/recordstore v0.1.72 h1:iz6WhPHmYkWHhRHY5xze635bXz3VPidqgAh3ETdf2io= +github.com/flanksource/commons-db/recordstore v0.1.72/go.mod h1:0hrPbUxv/FB7rrB8k3eJIRTgh5HW9fLXgSlLb7d2PoY= +github.com/flanksource/deps v1.0.51 h1:i5iOzDTVTLdvXfMfan0tNZQAtaYxcipk0nJwE9FhMEk= +github.com/flanksource/deps v1.0.51/go.mod h1:/AdWJJhDH/Phs+NxR2RLiwM3sHgBzb+LNgqAwtcyYgU= +github.com/flanksource/gavel/sdk v0.0.0-20261108204313-d40d4b480181 h1:hlXFHLeWpgFvcE2pAyJy7yv8YSG3T9h5SV9cPtiLaOc= +github.com/flanksource/gavel/sdk v0.0.0-20261108204313-d40d4b480181/go.mod h1:FiYJ6Ili2N+mPfVSdjpGeIPDxs9v0mqAfpve6nDpqa0= +github.com/flanksource/gomplate/v3 v3.24.91 h1:rqkTic5mJ2QZ75EQl2J4vHb/MCvSd++egEVC1bROokw= +github.com/flanksource/gomplate/v3 v3.24.91/go.mod h1:SrN9fb0X+3FKs8QvdZY+sUD3w9w/YuZUX2fAyqJidvw= +github.com/flanksource/gomplate/v4 v4.0.0-20261111091247-6aab4d0463bd h1:886yMGt65MmSy49HLEXTPV7ID60AdFJAbCkQntIjfmY= +github.com/flanksource/gomplate/v4 v4.0.0-20261111091247-6aab4d0463bd/go.mod h1:tZtKBQN7i3sD6tynPjAeOJnh8b1GHivVC52L7Wo+oXs= github.com/flanksource/is-healthy v1.0.93 h1:hpXkPfMXG+zawNs0HTjc/3ZIjrW7obAPDR/VoTJk59Q= github.com/flanksource/is-healthy v1.0.93/go.mod h1:Ee0lzss7zeLlfYOcr4r3eOUJqTD3fjVnxT5cv79UVF8= github.com/flanksource/kubectl-neat v1.0.4 h1:t5/9CqgE84oEtB0KitgJ2+WIeLfD+RhXSxYrqb4X8yI= github.com/flanksource/kubectl-neat v1.0.4/go.mod h1:Un/Voyh3cmiZNKQrW/TkAl28nAA7vwnwDGVjRErKjOw= +github.com/flanksource/sandbox-runtime v1.0.2 h1:dEaVyUpeaCAn3GOiWcNDrPguaey1Gx9eTsPAc27W3Eg= +github.com/flanksource/sandbox-runtime v1.0.2/go.mod h1:x5Ywc1MrP3lTgmj0VUCeFJzrDZ3mBhBV4Gdc/ioV76M= github.com/flanksource/sqlglot-go v1.1.1 h1:9AlEb41+Ll+W/PQwl7ctZMKo3phXF5f2jznsAcUmt9Y= github.com/flanksource/sqlglot-go v1.1.1/go.mod h1:aDnAJW6L2s2EXgiM8blbZ+ldV76Rdyu87KkPZJxFQIY= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ= github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= @@ -399,8 +381,8 @@ github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMj github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY= github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuwuB/jWEHjk= -github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= -github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-jose/go-jose/v4 v4.1.9 h1:pKfeQkig1NmgK4hJrZYXLEYURXfoJB61Larw0uIW4gY= +github.com/go-jose/go-jose/v4 v4.1.9/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -521,8 +503,8 @@ github.com/google/go-querystring v1.2.0/go.mod h1:8IFJqpSRITyJ8QhQ13bmbeMBDfmeEJ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gops v0.3.29 h1:n98J2qSOK1NJvRjdLDcjgDryjpIBGhbaqph1mXKL0rY= github.com/google/gops v0.3.29/go.mod h1:8N3jZftuPazvUwtYY/ncG4iPrjp15ysNKLfq+QQPiwc= -github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8= -github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= +github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0= +github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20260507013755-92041b743c96 h1:YDDnaZ9afWajDboPMt9Vikqca/yWAX7KAxVzb4lJU1M= @@ -530,8 +512,8 @@ github.com/google/pprof v0.0.0-20260507013755-92041b743c96/go.mod h1:MxpfABSjhmI github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.1-0.20241114170450-2d3c2a9cc518 h1:UBg1xk+oAsIVbFuGg6hdfAm7EvCv3EL80vFxJNsslqw= +github.com/google/uuid v1.6.1-0.20241114170450-2d3c2a9cc518/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4= github.com/google/wire v0.7.0/go.mod h1:n6YbUQD9cPKTnHXEBN2DXlOp/mVADhVErcMFb0v3J18= github.com/googleapis/enterprise-certificate-proxy v0.3.17 h1:73NfMHdiqo9JFU9+7a5ExpVa10/R29pXfZIaW559nrg= @@ -544,6 +526,8 @@ github.com/gosimple/slug v1.15.0 h1:wRZHsRrRcs6b0XnxMUBM6WK1U1Vg5B0R7VkIf1Xzobo= github.com/gosimple/slug v1.15.0/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ= github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o= github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= +github.com/govalues/decimal v0.1.36 h1:dojDpsSvrk0ndAx8+saW5h9WDIHdWpIwrH/yhl9olyU= +github.com/govalues/decimal v0.1.36/go.mod h1:Ee7eI3Llf7hfqDZtpj8Q6NCIgJy1iY3kH1pSwDrNqlM= github.com/grafana/jfr-parser v0.19.0 h1:GNLFIJ2i7y/pyismwi5GjiVmJDSzF2JoUEOp8cammXY= github.com/grafana/jfr-parser v0.19.0/go.mod h1:A2KH/ezc9ufSMk3+639UwB2nOwU17X4jWJLqfm7Ph+A= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 h1:QGLs/O40yoNK9vmy4rhUGBVyMf1lISBGtXRpsu/Qu/o= @@ -631,8 +615,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/labstack/echo/v4 v4.16.3 h1:wAzM49wiBxHpZJQ621lminvdxbwfd1Cn6KwRThKV6nA= -github.com/labstack/echo/v4 v4.16.3/go.mod h1:V/EQaTVHYJH92QhbQe7kFQ4mjgcjxLQQUQtAN87zZdw= +github.com/labstack/echo/v4 v4.16.4 h1:sQ9yq/FgkY7zWZKaRIvNJAQ1ybWVJrm+z37SWI8keRY= +github.com/labstack/echo/v4 v4.16.4/go.mod h1:V/EQaTVHYJH92QhbQe7kFQ4mjgcjxLQQUQtAN87zZdw= github.com/labstack/gommon v0.5.0 h1:6VSQ2NOzsnEJ5W6+84E0RbcaDDmgB6NIAzWCczTEe6c= github.com/labstack/gommon v0.5.0/go.mod h1:Rzlg7HHy1maLfzBYGg9NZcVuz1sA68HHhLjhcEllYE0= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= @@ -675,26 +659,8 @@ github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4 github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= -github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= -github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo= -github.com/moby/moby/api v1.52.0 h1:00BtlJY4MXkkt84WhUZPRqt5TvPbgig2FZvTbe3igYg= -github.com/moby/moby/api v1.52.0/go.mod h1:8mb+ReTlisw4pS6BRzCMts5M49W5M7bKt1cJy/YbAqc= -github.com/moby/moby/client v0.1.0 h1:nt+hn6O9cyJQqq5UWnFGqsZRTS/JirUqzPjEl0Bdc/8= -github.com/moby/moby/client v0.1.0/go.mod h1:O+/tw5d4a1Ha/ZA/tPxIZJapJRUS6LNZ1wiVRxYHyUE= -github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= -github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= -github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= -github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= -github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= -github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= -github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= -github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= -github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= -github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -722,24 +688,24 @@ github.com/nukilabs/ftoa v1.0.0 h1:AiCb9begI2rwjDQm7/H+ce1ctdkixnOOw/bpurZTySc= github.com/nukilabs/ftoa v1.0.0/go.mod h1:ZJE2wT+SozGrxoj3oRB6Q6Kl3IMpCzdNOhymdmLGrHU= github.com/nukilabs/unicodeid v0.1.0 h1:+CtAaV9CkgVA3Jy9V29yb57DPCLsnWUJrP9LxhkD89w= github.com/nukilabs/unicodeid v0.1.0/go.mod h1:3qxjfcmcgVYBg9dMRNSniAVfqxwOed7JU7P2rPkJrRg= -github.com/ohler55/ojg v1.28.1 h1:Xy93DelhLSZNeWv8GPKtP6qMqkUlZlAxBP/AQcC5RfY= -github.com/ohler55/ojg v1.28.1/go.mod h1:/Y5dGWkekv9ocnUixuETqiL58f+5pAsUfg5P8e7Pa2o= -github.com/oklog/ulid/v2 v2.1.1 h1:suPZ4ARWLOJLegGFiZZ1dFAkqzhMjL3J1TzI+5wHz8s= -github.com/oklog/ulid/v2 v2.1.1/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= -github.com/onsi/ginkgo/v2 v2.32.1 h1:6tlvcDm/3sE8lGJbZ4+d4mO3RLy24/tQWOFzVSQNIfw= -github.com/onsi/ginkgo/v2 v2.32.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= -github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= -github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= +github.com/ohler55/ojg v1.28.9 h1:VL2vdN1CjniaNABNOFjorAW+/o7eBQDc4z5jO49jkHM= +github.com/ohler55/ojg v1.28.9/go.mod h1:/Y5dGWkekv9ocnUixuETqiL58f+5pAsUfg5P8e7Pa2o= +github.com/oklog/ulid/v2 v2.1.2 h1:IEclFb9JNvzYA6MW2SCxbLzcHTVsfqm3PrqGQJH5zec= +github.com/oklog/ulid/v2 v2.1.2/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= +github.com/onsi/ginkgo/v2 v2.33.1 h1:62XfWJbEIjPSCSDYrsCPP/SHcmH7O0M8ne8zfqQ2iz4= +github.com/onsi/ginkgo/v2 v2.33.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= +github.com/onsi/gomega v1.44.0 h1:eAiGl3Pw5jz5GQdDff0BcxYpAX1JxW8xD7mFUuwNfZQ= +github.com/onsi/gomega v1.44.0/go.mod h1:e/C2HwaZ1DhvjzXXuFhcR7hY7Sh9pl7MmoWKEjzwcdA= +github.com/openai/openai-go v1.12.0 h1:NBQCnXzqOTv5wsgNC36PrFEiskGfO5wccfCWDo9S1U0= +github.com/openai/openai-go v1.12.0/go.mod h1:g461MYGXEXBVdV5SaR/5tNzNbSfwTBBefwc+LlDCK0Y= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= -github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/opensearch-project/opensearch-go/v2 v2.3.0 h1:nQIEMr+A92CkhHrZgUhcfsrZjibvB3APXf2a1VwCmMQ= github.com/opensearch-project/opensearch-go/v2 v2.3.0/go.mod h1:8LDr9FCgUTVoT+5ESjc2+iaZuldqE+23Iq0r1XeNue8= github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c= github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM= -github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= -github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/patrickmn/go-cache v2.1.1-0.20191004192108-46f407853014+incompatible h1:IWzUvJ72xMjmrjR9q3H1PF+jwdN0uNQiR2t1BLNalyo= +github.com/patrickmn/go-cache v2.1.1-0.20191004192108-46f407853014+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/paulmach/orb v0.12.0 h1:z+zOwjmG3MyEEqzv92UN49Lg1JFYx0L9GpGKNVDKk1s= github.com/paulmach/orb v0.12.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= @@ -748,6 +714,8 @@ github.com/pb33f/ordered-map/v2 v2.3.1/go.mod h1:qxFQgd0PkVUtOMCkTapqotNgzRhMPL7 github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pgplex/pgparser v0.2.0 h1:BB7I/Mrwetiw07PTq/AlXcKfajlCwoJfwB7KZ7V9irI= +github.com/pgplex/pgparser v0.2.0/go.mod h1:DWhK7oZJn/16KTFeOQ+mk78+sRnEYya1XPghIoP9NFU= github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= @@ -791,8 +759,8 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samber/lo v1.53.0 h1:t975lj2py4kJPQ6haz1QMgtId2gtmfktACxIXArw3HM= github.com/samber/lo v1.53.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= -github.com/samber/oops v1.22.0 h1:fmWRC3YRUXFzpZ9Vs4nuckrsnCHv1cFV/bEm00Wvm0Y= -github.com/samber/oops v1.22.0/go.mod h1:8ZDRxwQdphVhmLtEX9I6134LHJe5yeCV8cTfHz3m91Y= +github.com/samber/oops v1.23.2 h1:DIfPsYJ8p+diZKBZEfZsGDsJtl/zkYX3O813sVqJ82w= +github.com/samber/oops v1.23.2/go.mod h1:34UjnFO2/j7LvV4dCc+Ipe4o1a1HiCdbFFF5N3kh7HM= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= @@ -861,8 +829,8 @@ github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYI github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= -github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= -github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +github.com/ugorji/go/codec v1.3.2 h1:zkEASHHyEClGeURfgNT9PJZVfAbs9oEX9QXggwWNJbc= +github.com/ugorji/go/codec v1.3.2/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/valkey-io/valkey-go v1.0.75 h1:cfq9DODW2ntuUgyHJmFWb4/p+xpLpQB1t5SQyWM9uJ4= @@ -871,6 +839,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/woodsbury/decimal128 v1.4.1 h1:tlcaayBsiNfwo4dG9GtTfBM/3MgC1c7aAaTwspSaFKw= +github.com/woodsbury/decimal128 v1.4.1/go.mod h1:BP46FUrVjVhdTbKT+XuQh2xfQaGki9LMIRJSFuh6THU= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= @@ -891,8 +861,8 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/xuri/efp v0.0.3-0.20260924042702-217b81952dec h1:sZZokRLgr8nyrzkDDCQcvEYBsTrZr6eqXa/R9qALoQA= github.com/xuri/efp v0.0.3-0.20260924042702-217b81952dec/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= -github.com/xuri/excelize/v2 v2.11.1-0.20261029001137-c5fd73bde8f6 h1:zGu5f3tQT1WHoaP60eteDfGgqCT+APDwHsvAkWsjb84= -github.com/xuri/excelize/v2 v2.11.1-0.20261029001137-c5fd73bde8f6/go.mod h1:NQLBThVu4BB7tYM8wJIBj2dZsptXbDW4WyrFlL8YLZQ= +github.com/xuri/excelize/v2 v2.11.1 h1:tm/FVufArvfUiJXJtGv8jna/zZS1t9Uji2ImsnS9iBo= +github.com/xuri/excelize/v2 v2.11.1/go.mod h1:NQLBThVu4BB7tYM8wJIBj2dZsptXbDW4WyrFlL8YLZQ= github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 h1:+C0TIdyyYmzadGaL/HBLbf3WdLgC29pgyhTjAT/0nuE= github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= @@ -936,34 +906,38 @@ go.opentelemetry.io/collector/internal/testutil v0.163.0 h1:JR65ZHtuOP1E5cJZKl4u go.opentelemetry.io/collector/internal/testutil v0.163.0/go.mod h1:RUvWdMZGEBicc0C3teDcnTrifUX8HB4AJ4l8pMONURU= go.opentelemetry.io/collector/pdata v1.69.0 h1:u0a2n1kOS/OGwIEQMOKCnl+bbmFG5wGO/OfFXajICpA= go.opentelemetry.io/collector/pdata v1.69.0/go.mod h1:5U4aIRhQKQgo3DifeWgVdWBhSBbYV2BQ6Nz4bJc7dfE= -go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw= -go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s= +go.opentelemetry.io/collector/pdata/pprofile v0.163.0 h1:+JzjYkCizAR00yyCV6HFPm7SREi9R/TOnaeWQyExmt0= +go.opentelemetry.io/collector/pdata/pprofile v0.163.0/go.mod h1:aXV1wYxl0VXf8NfrrfBaqYoOqkkIgQJ2N/KdgtBxLl8= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0 h1:9jR0ZPRok9ryaOQ2Wx8rg5F7Aon59mxrqbVI60/vlBk= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0/go.mod h1:VSme3o2fvSg5bVg0dRzyHaj4Z5EVhG+g2Fde6LKzmQA= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 h1:oECp5f+hN7nkwjU/8BxQ/q23bGPb8FIrD839owX222E= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0/go.mod h1:DqEFwLumhzMBDQv9PcWbyoDxHI/4lAk6CM4nJBH39sc= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= -go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= -go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 h1:fG5MCxGz8+2VtrN/WgqSpJFctVz24gpxj8CxkKmc8Ww= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0/go.mod h1:BmAYTn+3ysbRe+IU2msxmf5Rx3g6DHvex+tWI3LdhYI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 h1:lgh3PiVrRUWMLOVSkQicxzZll5NjF1r+AtsX1XRIHw0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0/go.mod h1:5Cnhth3m/AgOeTgE3ex12pPmiu/gGtZit03kSzx9X7s= +go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= +go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 h1:UFpxOpYPmMNUtOWhdb+nC1WELIgVf8z9higURrbzYU4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0/go.mod h1:YBGjxe3lt0jtXoEXxGrGhv/jM7oUBUDVK7zhAvNyEjc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 h1:aXZXsZ012wOgVkqaiQv+Z/d8V0xKjmg70F6m4CE94lc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0/go.mod h1:ziu1gUIJhAaxM3EV1e3Ralk5AAyL1UOYz3lJJ4VSI38= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0 h1:TC+BewnDpeiAmcscXbGMfxkO+mwYUwE/VySwvw88PfA= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0/go.mod h1:J/ZyF4vfPwsSr9xJSPyQ4LqtcTPULFR64KwTikGLe+A= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 h1:lsA/S1bxgdbyFGkTj+3meEdJ6ADVU7QoFstV6MXgE68= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0/go.mod h1:L7u+MirGoB1bjeLH66+xDykF4RC8C3RN7lIFpBiewUo= -go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= -go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= -go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns= -go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI= -go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= -go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= -go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= -go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= -go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= -go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= +go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= +go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= +go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= +go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/metric/x v0.69.0 h1:DjRLr15H83v+hCW7JA9NoJvOkYTtmq5YoDRbe9deYpM= +go.opentelemetry.io/otel/metric/x v0.69.0/go.mod h1:uVvsMPMFFyj/HUQfrUnH3JjnOQ1dwFDorgFLRBasM0k= +go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= +go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= +go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= +go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= +go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= +go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= go.opentelemetry.io/proto/slim/otlp v1.11.1 h1:qf2OA3kQcIDXh3q7b6kc0IzRe8BFXlVIlH5uaOACrVQ= @@ -1016,8 +990,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= -golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= +golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= +golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1069,7 +1043,6 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1084,8 +1057,8 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= -golang.org/x/telemetry v0.0.0-20260924152758-ed294f943157 h1:DftxASHKl6JolFPqoN+2WrFO53JvkHwRXvFqf8gsivA= -golang.org/x/telemetry v0.0.0-20260924152758-ed294f943157/go.mod h1:i+ivNqjDnTF3WTElsdk5g9V5DTSBYgdNo7xTU9SDwYA= +golang.org/x/telemetry v0.0.0-20261009151841-97e8ea77aa2a h1:wnveMBKHwx8/nIb8WYT5P9vtefdXOvIWxzycCkc5n6Q= +golang.org/x/telemetry v0.0.0-20261009151841-97e8ea77aa2a/go.mod h1:i+ivNqjDnTF3WTElsdk5g9V5DTSBYgdNo7xTU9SDwYA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1125,8 +1098,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI= -golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA= +golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs= +golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1139,22 +1112,24 @@ google.golang.org/api v0.286.0 h1:TdTXMvzYKnWV1/lPbCdbXRqBrkDqjPto22H2xeZZ8LI= google.golang.org/api v0.286.0/go.mod h1:NlOlUIr8MPoIhT9Bb/oUnRuHbJOLwxb6JSYJM8Yz+jQ= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genai v1.57.0 h1:qTyG2ynz5dQy2jF4CvZdLHHVslhR0heMue+zM1a4GNM= +google.golang.org/genai v1.57.0/go.mod h1:A3kkl0nyBjyFlNjgxIwKq70julKbIxpSxqKO5gw/gmk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= -google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc= -google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc h1:4bNTbnb44EqGVy9HaQxSY2jnafSUdgV3qHtedvNpDKg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= -google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= +google.golang.org/grpc v1.85.0 h1:BPBCfuVRQ1qvBhKDOq9uvQllMvO59rKsYhXpp7Dd8vw= +google.golang.org/grpc v1.85.0/go.mod h1:0rQrOD58yjal/0EaVbGBANindixsCZNQPDV3VKtaKuQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1253,8 +1228,6 @@ modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= mvdan.cc/sh/v3 v3.13.0 h1:dSfq/MVsY4w0Vsi6Lbs0IcQquMVqLdKLESAOZjuHdLg= mvdan.cc/sh/v3 v3.13.0/go.mod h1:KV1GByGPc/Ho0X1E6Uz9euhsIQEj4hwyKnodLlFLoDM= -pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= -pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2Fol2CS0QHMNs/WI1MOSGzCm1KhM5ec= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= diff --git a/go.work.ci b/go.work.ci index 44ee714..4425b46 100644 --- a/go.work.ci +++ b/go.work.ci @@ -1,4 +1,4 @@ -go 1.27.1 +go 1.27.4 use ( . diff --git a/hack/collector-fixture/README.md b/hack/collector-fixture/README.md new file mode 100644 index 0000000..3686a53 --- /dev/null +++ b/hack/collector-fixture/README.md @@ -0,0 +1,15 @@ +# Collector Profiles compatibility fixture + +This isolated module pins the real OTLP receiver and queued OTLP exporter to Collector `0.163.0`. The parent Ginkgo integration test starts the fixture, sends HTTP Profiles to `/v1development/profiles`, and verifies snapshots delivered over gRPC into Telltale. It does not replace either Collector component with a proxy. + +Run from the repository root: + +```sh +go test ./internal/otelreceiver -run TestReceiver -ginkgo.focus='released Collector Profiles pipeline' +``` + +The gate checks active, updated and empty observations through request-sized and item-sized export queues. Telltale convention version `2` encodes an empty observation with one explicit zero-valued `snapshot_control` sample because the released Collector discards zero-sample profiles. The test verifies that the native control survives and that the latest Top view contains zero operation rows. Generic Profiles consumers and Collector sample counters still see the control sample; see the [wire contract](../../docs/traces/profiles.mdx). + +The parent process isolates this module with `GOWORK=off` so local workspace modules do not substitute the pinned Collector dependencies. Compatibility assertions remain enabled. + +`main.go` waits for standard input to close, then shuts down the receiver and drains the exporter. Dependencies remain isolated from the application module. diff --git a/hack/collector-fixture/go.mod b/hack/collector-fixture/go.mod new file mode 100644 index 0000000..a896251 --- /dev/null +++ b/hack/collector-fixture/go.mod @@ -0,0 +1,96 @@ +module github.com/flanksource/telltale/collector-profile-gate + +go 1.26.0 + +require ( + go.opentelemetry.io/collector/component v1.69.0 + go.opentelemetry.io/collector/component/componenttest v0.163.0 + go.opentelemetry.io/collector/config/configgrpc v1.69.0 + go.opentelemetry.io/collector/config/configoptional v1.69.0 + go.opentelemetry.io/collector/config/configtls v1.69.0 + go.opentelemetry.io/collector/exporter v1.69.0 + go.opentelemetry.io/collector/exporter/otlpexporter v0.163.0 + go.opentelemetry.io/collector/exporter/xexporter v0.163.0 + go.opentelemetry.io/collector/receiver v1.69.0 + go.opentelemetry.io/collector/receiver/otlpreceiver v0.163.0 + go.opentelemetry.io/collector/receiver/xreceiver v0.163.0 +) + +require ( + github.com/Microsoft/go-winio v0.6.3 // indirect + github.com/cenkalti/backoff/v7 v7.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/foxboron/go-tpm-keyfiles v0.0.0-20251226215517-609e4778396f // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/golang/snappy v1.0.0 // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/go-version v1.9.0 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.20.3 // indirect + github.com/knadh/koanf/maps v0.1.3 // indirect + github.com/knadh/koanf/providers/confmap v1.0.1 // indirect + github.com/knadh/koanf/v2 v2.3.8 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/pierrec/lz4/v4 v4.1.33 // indirect + github.com/rs/cors v1.11.1 // indirect + github.com/stretchr/testify v1.12.1 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/collector/client v1.69.0 // indirect + go.opentelemetry.io/collector/component/componentstatus v0.163.0 // indirect + go.opentelemetry.io/collector/config/configauth v1.69.0 // indirect + go.opentelemetry.io/collector/config/configcompression v1.69.0 // indirect + go.opentelemetry.io/collector/config/confighttp v0.163.0 // indirect + go.opentelemetry.io/collector/config/configmiddleware v1.69.0 // indirect + go.opentelemetry.io/collector/config/confignet v1.69.0 // indirect + go.opentelemetry.io/collector/config/configopaque v1.69.0 // indirect + go.opentelemetry.io/collector/config/configretry v1.69.0 // indirect + go.opentelemetry.io/collector/confmap v1.69.0 // indirect + go.opentelemetry.io/collector/consumer v1.69.0 // indirect + go.opentelemetry.io/collector/consumer/consumererror v0.163.0 // indirect + go.opentelemetry.io/collector/consumer/consumererror/xconsumererror v0.163.0 // indirect + go.opentelemetry.io/collector/consumer/xconsumer v0.163.0 // indirect + go.opentelemetry.io/collector/exporter/exporterhelper v0.163.0 // indirect + go.opentelemetry.io/collector/exporter/exporterhelper/xexporterhelper v0.163.0 // indirect + go.opentelemetry.io/collector/extension v1.69.0 // indirect + go.opentelemetry.io/collector/extension/extensionauth v1.69.0 // indirect + go.opentelemetry.io/collector/extension/extensionmiddleware v0.163.0 // indirect + go.opentelemetry.io/collector/extension/xextension v0.163.0 // indirect + go.opentelemetry.io/collector/featuregate v1.69.0 // indirect + go.opentelemetry.io/collector/internal/componentalias v0.163.0 // indirect + go.opentelemetry.io/collector/internal/sharedcomponent v0.163.0 // indirect + go.opentelemetry.io/collector/internal/statusutil v0.163.0 // indirect + go.opentelemetry.io/collector/internal/telemetry v0.163.0 // indirect + go.opentelemetry.io/collector/pdata v1.69.0 // indirect + go.opentelemetry.io/collector/pdata/pprofile v0.163.0 // indirect + go.opentelemetry.io/collector/pdata/xpdata v0.163.0 // indirect + go.opentelemetry.io/collector/pipeline v1.69.0 // indirect + go.opentelemetry.io/collector/pipeline/xpipeline v0.163.0 // indirect + go.opentelemetry.io/collector/receiver/receiverhelper v0.163.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect + go.opentelemetry.io/otel v1.47.0 // indirect + go.opentelemetry.io/otel/log v1.47.0 // indirect + go.opentelemetry.io/otel/metric v1.47.0 // indirect + go.opentelemetry.io/otel/sdk v1.47.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.47.0 // indirect + go.opentelemetry.io/otel/trace v1.47.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.28.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/crypto v0.58.0 // indirect + golang.org/x/net v0.61.0 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/text v0.43.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect + google.golang.org/grpc v1.84.0 // indirect + google.golang.org/protobuf v1.36.12 // indirect +) diff --git a/hack/collector-fixture/go.sum b/hack/collector-fixture/go.sum new file mode 100644 index 0000000..c5ad1cb --- /dev/null +++ b/hack/collector-fixture/go.sum @@ -0,0 +1,217 @@ +github.com/Microsoft/go-winio v0.6.3 h1:M/52tsoyTfwaxajLnXZm7PqkbMGRH8iSWP+8fcg9MQ4= +github.com/Microsoft/go-winio v0.6.3/go.mod h1:ma0QpxizD4fyyzJ1C7MIblymTD2Nxz5Mho/3CepwIZg= +github.com/cenkalti/backoff/v7 v7.0.1 h1:+ulHti5s1d9QnfyOnLWVmQ/SaZ+TJQF9OdOI0i9BGZI= +github.com/cenkalti/backoff/v7 v7.0.1/go.mod h1:qcKBGwsu4hpxHtQ8tWYsQ+ifzx2+sS+Xx/3jfe30lI8= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/foxboron/go-tpm-keyfiles v0.0.0-20251226215517-609e4778396f h1:RJ+BDPLSHQO7cSjKBqjPJSbi1qfk9WcsjQDtZiw3dZw= +github.com/foxboron/go-tpm-keyfiles v0.0.0-20251226215517-609e4778396f/go.mod h1:VHbbch/X4roIY22jL1s3qRbZhCiRIgUAF/PdSUcx2io= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/google/go-tpm-tools v0.4.7 h1:J3ycC8umYxM9A4eF73EofRZu4BxY0jjQnUnkhIBbvws= +github.com/google/go-tpm-tools v0.4.7/go.mod h1:gSyXTZHe3fgbzb6WEGd90QucmsnT1SRdlye82gH8QjQ= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= +github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4= +github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/knadh/koanf/maps v0.1.3 h1:P1z7EvTqdFBrPYbzSvorvrpib+sjkUMxf0FVvA5NKK4= +github.com/knadh/koanf/maps v0.1.3/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v1.0.1 h1:L15hbvMqlvhwUuCtL9BkL+rqiMAjk6cZc8O9XoDtE3A= +github.com/knadh/koanf/providers/confmap v1.0.1/go.mod h1:txHYHiI2hAtF0/0sCmcuol4IDcuQbKTybiB1nOcUo1A= +github.com/knadh/koanf/v2 v2.3.8 h1:NftEOf0Y456FqpLJj31RYYZQnfBQ+DDhsNQPqJ6nuk4= +github.com/knadh/koanf/v2 v2.3.8/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pierrec/lz4/v4 v4.1.33 h1:GjG1TJ1V4IzKP8L96muuuDNpTwd7D+l2ccXrjAbe014= +github.com/pierrec/lz4/v4 v4.1.33/go.mod h1:7SE9MC2STkNtL4PIwGhjmyVwvILaGI9/COYQNBhKM/c= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= +github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/collector/client v1.69.0 h1:JaWIacYOkEq2aajAuNltTy7elsIpSPdNMe/I3+b28bY= +go.opentelemetry.io/collector/client v1.69.0/go.mod h1:D3eMugzcsQlJyPJ0WXp2b7+yS72Uswi0e+WMQyNLzHo= +go.opentelemetry.io/collector/component v1.69.0 h1:ZbVd2N+sCv39wUWQ4qtFG7wuTaok1EHaThbtdE/6Rn4= +go.opentelemetry.io/collector/component v1.69.0/go.mod h1:F8t8JGwjKTtd84Qki+e2k0Emz7AgL00jRFdD5ZPWSe4= +go.opentelemetry.io/collector/component/componentstatus v0.163.0 h1:DxS+q1BA7uZ7Le6NEAl3gbazIEPV2RX5YVO2dtXnxak= +go.opentelemetry.io/collector/component/componentstatus v0.163.0/go.mod h1:4WGQJpbXRBCFnSiOAfqPLc0804wEblapy+XtmJAfMUM= +go.opentelemetry.io/collector/component/componenttest v0.163.0 h1:2KE+VpMlgKskjeM6Slrl5qAIju8UbjECYAwOmmj1dZo= +go.opentelemetry.io/collector/component/componenttest v0.163.0/go.mod h1:bq9Clgt1at53cjlMUH2drrseX+00WsZt0OpFAnrXeCU= +go.opentelemetry.io/collector/config/configauth v1.69.0 h1:Atid/XFWwn+89YZUBl/AeJlhxtHWV1+t2V5xwtwRunk= +go.opentelemetry.io/collector/config/configauth v1.69.0/go.mod h1:e2R0cdHcDwxZHCZzJSov/1t/MhsHFsOaYydDhk8pkh8= +go.opentelemetry.io/collector/config/configcompression v1.69.0 h1:v0cn/6Pmlxe/vlyGQioxA1uZUNFebU4WYmw8VTdy3sE= +go.opentelemetry.io/collector/config/configcompression v1.69.0/go.mod h1:SyPEhp8hwY8Yxuxbet2AIFDRYIEKl+tn5/jKnWtV1y0= +go.opentelemetry.io/collector/config/configgrpc v1.69.0 h1:DtlrUWhO10161NH5FtQyV1fVcMl7lK0PxnzUHgPCQgA= +go.opentelemetry.io/collector/config/configgrpc v1.69.0/go.mod h1:0kOEwUyFOyoKn4rk7GU9Ta/AkVSP1YNMLBgsAy0j8o4= +go.opentelemetry.io/collector/config/confighttp v0.163.0 h1:NZWIXcxNgc8NQNJ1mB4jwXzs1VV1RRmKYtZ3k9jZ2i4= +go.opentelemetry.io/collector/config/confighttp v0.163.0/go.mod h1:/dZNZWVXMatiCM/Erecq5aGxlrumR4ucXSk1RrcHYmI= +go.opentelemetry.io/collector/config/configmiddleware v1.69.0 h1:mgulrjsj+H9U+qPXNNbomyTdRBuJgsZhz0ev4ThrZeA= +go.opentelemetry.io/collector/config/configmiddleware v1.69.0/go.mod h1:SbasWsuiOUL8x7TKKm4lbitrP7T/L30T68kkDXejBWc= +go.opentelemetry.io/collector/config/confignet v1.69.0 h1:9CWjjmank0fU94zDiJ2rhQlKgp9WVGv+RRJCFxxEpQU= +go.opentelemetry.io/collector/config/confignet v1.69.0/go.mod h1:jLGx0okSewpXBdFA5i6tgZQcVHU9oBvYT7v5uMUO6ig= +go.opentelemetry.io/collector/config/configopaque v1.69.0 h1:4hIwoeIpV5uAPvpqXF4gXNJ/NprklevSBeTvStkya2o= +go.opentelemetry.io/collector/config/configopaque v1.69.0/go.mod h1:/qHp1tURH6QGTrCV0EAuLQlm1klwbjGLqyVOFr7H3Zs= +go.opentelemetry.io/collector/config/configoptional v1.69.0 h1:40LnYFA3CF7XkMztI6ldmDwWu9tIJOQKszTvMxfqR8o= +go.opentelemetry.io/collector/config/configoptional v1.69.0/go.mod h1:3WxT+wDaA98b6Tk2lwqD7MjBT/1naGDaTTS7AB1TOPM= +go.opentelemetry.io/collector/config/configretry v1.69.0 h1:MT0NCJnLy/66mvpGTNyjjxj5X0M9Yeb3qM0EsZ+R1TA= +go.opentelemetry.io/collector/config/configretry v1.69.0/go.mod h1:Y2Kdgd0HFBXYbGqUUUWGDLYviRHgl9aJVSyDfFYY7U8= +go.opentelemetry.io/collector/config/configtls v1.69.0 h1:pygxB6oHg0yWRG6xsw8NnbflLV/4nMGJCP894ABwpIY= +go.opentelemetry.io/collector/config/configtls v1.69.0/go.mod h1:hS53GM5G8ZNhlUvpiour3YUEQE2GUnTHD0t6ZjydEDw= +go.opentelemetry.io/collector/confmap v1.69.0 h1:PfCZR4752WXa0cDKEOphnboLNLTJhrvjvnM8OUD847w= +go.opentelemetry.io/collector/confmap v1.69.0/go.mod h1:B3mmFRAkl0x24l2/oM8KAApgI1JDEnew8BMzqJ08rMw= +go.opentelemetry.io/collector/consumer v1.69.0 h1:bgPqG4fRn9o70MSOB7rQPvlHRBQ1XmMtxqI9Wjk/PFI= +go.opentelemetry.io/collector/consumer v1.69.0/go.mod h1:eMzP5WIsb4UUqVTl3pvrOPk5NOuwJ/55EmS2Kzva/J0= +go.opentelemetry.io/collector/consumer/consumererror v0.163.0 h1:U110UnktprP2CqpOsOLeefMXdzawF8D2HXsZxaiGFsM= +go.opentelemetry.io/collector/consumer/consumererror v0.163.0/go.mod h1:aXYeSUvLOYnFxaYs4Ri0jvJcezT/lO64aAE3C7Ym4iw= +go.opentelemetry.io/collector/consumer/consumererror/xconsumererror v0.163.0 h1:LTcoc1NeNadGPJJee2d9tvxEW6m1oevrtBwfm5pNcdg= +go.opentelemetry.io/collector/consumer/consumererror/xconsumererror v0.163.0/go.mod h1:o1dq9Jp33SM4l7hH4r6JjzHlIaOF6Bb6M5koAH+vnTs= +go.opentelemetry.io/collector/consumer/consumertest v0.163.0 h1:skpDDMuulMi/RcDkv/jAbjt+BetkaMA8V2oOoK5uLMA= +go.opentelemetry.io/collector/consumer/consumertest v0.163.0/go.mod h1:SxKGOXtKptcQq/lpsrOE3qeNSoNEMvORunAtL7uvlpA= +go.opentelemetry.io/collector/consumer/xconsumer v0.163.0 h1:M9RkuJyJePExQDsyTORVJ/tzDNBDFEvcpD3495Nytxw= +go.opentelemetry.io/collector/consumer/xconsumer v0.163.0/go.mod h1:XZu88U9w24y7xqv/qj4H10T/NashmwBgxzk6jDWE/Qk= +go.opentelemetry.io/collector/exporter v1.69.0 h1:vGVgTXVeh4Z0Y+zLqfJEwQpPSDRA92inIcVQ0i+oTi0= +go.opentelemetry.io/collector/exporter v1.69.0/go.mod h1:OmPuMtTjgDAdYWzE6O8O0B/ieNqSEL33u6MlFot4bRU= +go.opentelemetry.io/collector/exporter/exporterhelper v0.163.0 h1:RN+eR4Oen/6gKafQ2g4BbwBLYhWtJGpY3cQi0vCeamQ= +go.opentelemetry.io/collector/exporter/exporterhelper v0.163.0/go.mod h1:4d+b8epG79x+AFcFnD7fimvegt21RzLoJ069Kun2G7U= +go.opentelemetry.io/collector/exporter/exporterhelper/xexporterhelper v0.163.0 h1:NtcDcP6HhAEEO5S5JcI4HM1wrhZDaFrpSpEY75B8K1k= +go.opentelemetry.io/collector/exporter/exporterhelper/xexporterhelper v0.163.0/go.mod h1:um4ArRlMBPmI+fknbug/UJ235kG623uWHxZVqjKSOso= +go.opentelemetry.io/collector/exporter/exportertest v0.163.0 h1:5hBAUbW3V3KBQj/85L7CixmH9TLbdCRkPHl9yN2euEM= +go.opentelemetry.io/collector/exporter/exportertest v0.163.0/go.mod h1:s52VuDhIk8SYIgh3gaRvwJJjRVuQax07tyLAkulFi+I= +go.opentelemetry.io/collector/exporter/otlpexporter v0.163.0 h1:BMB2qzPUEqd70muQnOoh0rgzJLVK/RHE7g30B4auVFI= +go.opentelemetry.io/collector/exporter/otlpexporter v0.163.0/go.mod h1:NpCPZ15WkwSknI4RyiAuB5GCMAky8vvDWH0s82CdTA8= +go.opentelemetry.io/collector/exporter/xexporter v0.163.0 h1:bifyKPc9BssXAz6uw63dQ8fph0IE0UsmZCyXij5p+s8= +go.opentelemetry.io/collector/exporter/xexporter v0.163.0/go.mod h1:PhDi1hcSec4lt99KV3n5pVtAYir8JryvGxCL/uOSiCQ= +go.opentelemetry.io/collector/extension v1.69.0 h1:Xp91fIIXeFCjY/2/8oX3EHcBj0laCr9EvQv/Pf67O0U= +go.opentelemetry.io/collector/extension v1.69.0/go.mod h1:VmW1iOuQrLXQFUnytFZeqI9JqV4drbIg/bxPduQRjzk= +go.opentelemetry.io/collector/extension/extensionauth v1.69.0 h1:QLivn2zII3pi8eYCz5m+bvgMd9vulngYS0VoS3VqL7Q= +go.opentelemetry.io/collector/extension/extensionauth v1.69.0/go.mod h1:kz7nMWwSv5JEzA8xoxa4nOHL0PTDqNG4+SJ6Y1/QZxo= +go.opentelemetry.io/collector/extension/extensionauth/extensionauthtest v0.163.0 h1:eDfg+kRLi73NuvGcM3ebQkksfpMOKOPpmOFiOwstytQ= +go.opentelemetry.io/collector/extension/extensionauth/extensionauthtest v0.163.0/go.mod h1:32Q4OqoxaBabtGTjFWu8eCkrdtILqqnyEQMKJ6y5f9M= +go.opentelemetry.io/collector/extension/extensionmiddleware v0.163.0 h1:YCOPfJKxhr9V6nsqzHpHwixF1nGAs3BPh3BCUHNZSO0= +go.opentelemetry.io/collector/extension/extensionmiddleware v0.163.0/go.mod h1:/XHpIhlYqoZSl8ToY0YeF9AvIRz/NO/9ecbHetnbjJw= +go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest v0.163.0 h1:sha79HBSfdRr3FOkzz+uql/V+W86/MmMHX6ppE8idJY= +go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest v0.163.0/go.mod h1:yeJHeEa0OTqiVAYJkHz+7k89RS8nkQPBUvXbBMXpd78= +go.opentelemetry.io/collector/extension/extensiontest v0.163.0 h1:rWPNjJSLkTeNVkNjp8oUGPmA1LHjHA24zutsUtTsebE= +go.opentelemetry.io/collector/extension/extensiontest v0.163.0/go.mod h1:A+OOc1FxAvNmkKk4unYbkEwTOSlNqjVKXkF2gbl8/pQ= +go.opentelemetry.io/collector/extension/xextension v0.163.0 h1:aMq+0hQ7BvAiv5HqPAkIdnWa0KFwuptAMfmgZ2KKCKk= +go.opentelemetry.io/collector/extension/xextension v0.163.0/go.mod h1:BlCqUiIC+fdFufVpHxvkwsJFCbXgg4V3baIkaTbDS/E= +go.opentelemetry.io/collector/featuregate v1.69.0 h1:RapEMYjBfvt9IUP3ypQ9Y7/g21nwRzvtKO3dSOCCL6E= +go.opentelemetry.io/collector/featuregate v1.69.0/go.mod h1:dRYifiJa2vQ6LWpPwHny4mL82mnGWsWEVeVWw+DhYJw= +go.opentelemetry.io/collector/internal/componentalias v0.163.0 h1:VkZUBC9KtnInkP9cs3RjOVQWEe6ThbX1u92FItOc8/U= +go.opentelemetry.io/collector/internal/componentalias v0.163.0/go.mod h1:6jHVr/SaT4hBiOMV2vHqK4jvonUki33IQ5Q6fX/NRAc= +go.opentelemetry.io/collector/internal/sharedcomponent v0.163.0 h1:/+mURJyT01hBkveLdwGw5Tfp4jvwK9bmG0GdZR1e7GA= +go.opentelemetry.io/collector/internal/sharedcomponent v0.163.0/go.mod h1:WVp2LS0rKtXraGNTocky/yVgzpaBnrr5UVFaYYvOE0Q= +go.opentelemetry.io/collector/internal/statusutil v0.163.0 h1:IV1fo04jU5zWiFxPoSVEvEyjyRO8rrsUoY1ED1iQgto= +go.opentelemetry.io/collector/internal/statusutil v0.163.0/go.mod h1:ZXwHY6Q3YyH5fgkK7VDk9+UEumPIcMbqlsAObX8VOT0= +go.opentelemetry.io/collector/internal/telemetry v0.163.0 h1:d94W1rXLPJ7gtbL/bSPnuPyr1IEK/C6yVjF1k3EPc2k= +go.opentelemetry.io/collector/internal/telemetry v0.163.0/go.mod h1:2HfaPqG31qSrrfQTaLt8gTIjzTeU8kXqDwv2gHyskys= +go.opentelemetry.io/collector/internal/testutil v0.163.0 h1:JR65ZHtuOP1E5cJZKl4ub/UFHQ7xo3KN/Iryhtc0P5E= +go.opentelemetry.io/collector/internal/testutil v0.163.0/go.mod h1:RUvWdMZGEBicc0C3teDcnTrifUX8HB4AJ4l8pMONURU= +go.opentelemetry.io/collector/pdata v1.69.0 h1:u0a2n1kOS/OGwIEQMOKCnl+bbmFG5wGO/OfFXajICpA= +go.opentelemetry.io/collector/pdata v1.69.0/go.mod h1:5U4aIRhQKQgo3DifeWgVdWBhSBbYV2BQ6Nz4bJc7dfE= +go.opentelemetry.io/collector/pdata/pprofile v0.163.0 h1:+JzjYkCizAR00yyCV6HFPm7SREi9R/TOnaeWQyExmt0= +go.opentelemetry.io/collector/pdata/pprofile v0.163.0/go.mod h1:aXV1wYxl0VXf8NfrrfBaqYoOqkkIgQJ2N/KdgtBxLl8= +go.opentelemetry.io/collector/pdata/testdata v0.163.0 h1:Rv/uY6N2cNhLSnldAFGsSpAyf+xvQGx27x837q6+kwk= +go.opentelemetry.io/collector/pdata/testdata v0.163.0/go.mod h1:ERljDOnKN4633p/Vpbr3XhNHm1f6lrwD6vuSJSRi+04= +go.opentelemetry.io/collector/pdata/xpdata v0.163.0 h1:4CvvCCNkGSbhrmwUtpn1JcFsaOFDQn8BearBpD8skbc= +go.opentelemetry.io/collector/pdata/xpdata v0.163.0/go.mod h1:acCF/biwBrWt/6r3t9y/lRJ5tMvUOdQv411UPbyptVo= +go.opentelemetry.io/collector/pipeline v1.69.0 h1:zUimUSVCh+ects/w9gOdUZaIHTKd4dFvmSbzzuVTpA4= +go.opentelemetry.io/collector/pipeline v1.69.0/go.mod h1:4S7iD/7hGDNXg4yPi+5es5WTvwo/Uie2OoHio79xokI= +go.opentelemetry.io/collector/pipeline/xpipeline v0.163.0 h1:j1M4z68B5PtxgvV9+AhbUG+CJAMBROntSDkaCLbzsV4= +go.opentelemetry.io/collector/pipeline/xpipeline v0.163.0/go.mod h1:aG5gJf1JwOG2UXSc9ltNpYPqDkUtOh2hFE8VtPtWANM= +go.opentelemetry.io/collector/receiver v1.69.0 h1:XRK6OXrvlmNZXJu5Nvs+ihFlzglKEuI8eUCyXoBnHF4= +go.opentelemetry.io/collector/receiver v1.69.0/go.mod h1:d8QaZMEnzz1ju65PSwrgPpadcyZrIwVkqfnrpHZ4I70= +go.opentelemetry.io/collector/receiver/otlpreceiver v0.163.0 h1:12mAw4p5gvGCCmh38BgnT66MdYHjFuiMxfN1jp9nPSk= +go.opentelemetry.io/collector/receiver/otlpreceiver v0.163.0/go.mod h1:8xYquCyB5IV9JsD8Xha2rOWSp72Nxe79naiPrq14sWc= +go.opentelemetry.io/collector/receiver/receiverhelper v0.163.0 h1:uni+wobCouwb1I6YKZdKlUSubN9X2kiFOpCM/3R7XxU= +go.opentelemetry.io/collector/receiver/receiverhelper v0.163.0/go.mod h1:Tz8gNto3OIzOgWmCz4jSCGw2MhRwLy29GTRb3s+d0Gc= +go.opentelemetry.io/collector/receiver/receivertest v0.163.0 h1:KhM4t571zn+9O6puAjboCknKyXu/FFdlsaWWXiPwybc= +go.opentelemetry.io/collector/receiver/receivertest v0.163.0/go.mod h1:You1UKh9N51AAwbcHT3dwp6M3+Zyla4Zoj+zNM6v9YM= +go.opentelemetry.io/collector/receiver/xreceiver v0.163.0 h1:CRqBGzszjz9yrU5NbF4Ea6rll9gekaKuwccktv7jXjA= +go.opentelemetry.io/collector/receiver/xreceiver v0.163.0/go.mod h1:gxvXr+KbhQNdGkcDU8sXh5Ir95X4aywttph7tBAPzb0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 h1:oECp5f+hN7nkwjU/8BxQ/q23bGPb8FIrD839owX222E= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0/go.mod h1:DqEFwLumhzMBDQv9PcWbyoDxHI/4lAk6CM4nJBH39sc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= +go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= +go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= +go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= +go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= +go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= +go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/metric/x v0.69.0 h1:DjRLr15H83v+hCW7JA9NoJvOkYTtmq5YoDRbe9deYpM= +go.opentelemetry.io/otel/metric/x v0.69.0/go.mod h1:uVvsMPMFFyj/HUQfrUnH3JjnOQ1dwFDorgFLRBasM0k= +go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= +go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= +go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= +go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= +go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= +go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= +go.opentelemetry.io/proto/slim/otlp v1.11.1 h1:qf2OA3kQcIDXh3q7b6kc0IzRe8BFXlVIlH5uaOACrVQ= +go.opentelemetry.io/proto/slim/otlp v1.11.1/go.mod h1:15pWfj6/rYjQzyrC3ZGQoZ0rNgLqI/Lk+jFgYgV4UIE= +go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.4.1 h1:pgPIbi29ViNY/gE//bHlHuLyqHE+PDhhw7jAhItWIaM= +go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.4.1/go.mod h1:MSbd82YGFGySHE+GsR2J7+RvMf7mgnknr7IFjsxfoTs= +go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.4.1 h1:pso/53PWXaoGeW+/ww9WHKxoWTbTJBSmUH1VsiEmelU= +go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.4.1/go.mod h1:axuN0N02xZ3l+3zFqWXXRBTC6766vGQ0PbeCh46kuXA= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= +golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw= +golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= +google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/hack/collector-fixture/main.go b/hack/collector-fixture/main.go new file mode 100644 index 0000000..ef6973c --- /dev/null +++ b/hack/collector-fixture/main.go @@ -0,0 +1,95 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "time" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/config/configgrpc" + "go.opentelemetry.io/collector/config/configoptional" + "go.opentelemetry.io/collector/config/configtls" + "go.opentelemetry.io/collector/exporter" + "go.opentelemetry.io/collector/exporter/otlpexporter" + "go.opentelemetry.io/collector/exporter/xexporter" + "go.opentelemetry.io/collector/receiver" + "go.opentelemetry.io/collector/receiver/otlpreceiver" + "go.opentelemetry.io/collector/receiver/xreceiver" +) + +type collectorOptions struct { + Listen, Endpoint, QueueSizer string +} + +func main() { + listen := flag.String("listen", "", "Collector HTTP listener") + endpoint := flag.String("export", "", "Telltale gRPC endpoint") + queueSizer := flag.String("queue-sizer", "requests", "Sending queue unit: requests or items") + flag.Parse() + if err := run(collectorOptions{Listen: *listen, Endpoint: *endpoint, QueueSizer: *queueSizer}); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(options collectorOptions) (result error) { + if options.Listen == "" || options.Endpoint == "" { + return fmt.Errorf("listen and export endpoints are required") + } + if options.QueueSizer != "requests" && options.QueueSizer != "items" { + return fmt.Errorf("queue-sizer must be requests or items, received %q", options.QueueSizer) + } + ctx := context.Background() + host := componenttest.NewNopHost() + telemetry := componenttest.NewNopTelemetrySettings() + exportFactory := otlpexporter.NewFactory() + exportConfig := exportFactory.CreateDefaultConfig().(*otlpexporter.Config) + exportConfig.ClientConfig.Endpoint = options.Endpoint + exportConfig.ClientConfig.TLS = configtls.ClientConfig{Insecure: true} + queue := exportConfig.QueueConfig.GetOrInsertDefault() + queue.NumConsumers = 1 + if err := queue.Sizer.UnmarshalText([]byte(options.QueueSizer)); err != nil { + return fmt.Errorf("configure Collector queue sizing: %w", err) + } + exported, err := exportFactory.(xexporter.Factory).CreateProfiles(ctx, exporter.Settings{ + ID: component.MustNewID("otlp"), TelemetrySettings: telemetry, + }, exportConfig) + if err != nil { + return fmt.Errorf("create Collector exporter: %w", err) + } + if err = exported.Start(ctx, host); err != nil { + return fmt.Errorf("start Collector exporter: %w", err) + } + defer func() { + shutdown, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + result = errors.Join(result, exported.Shutdown(shutdown)) + }() + receiveFactory := otlpreceiver.NewFactory() + receiveConfig := receiveFactory.CreateDefaultConfig().(*otlpreceiver.Config) + receiveConfig.Protocols.GRPC = configoptional.None[configgrpc.ServerConfig]() + httpConfig := receiveConfig.Protocols.HTTP.GetOrInsertDefault() + httpConfig.ServerConfig.NetAddr.Endpoint = options.Listen + received, err := receiveFactory.(xreceiver.Factory).CreateProfiles(ctx, receiver.Settings{ + ID: component.MustNewID("otlp"), TelemetrySettings: telemetry, + }, receiveConfig, exported) + if err != nil { + return fmt.Errorf("create Collector receiver: %w", err) + } + if err = received.Start(ctx, host); err != nil { + return fmt.Errorf("start Collector receiver: %w", err) + } + fmt.Println("ready") + _, readErr := io.Copy(io.Discard, os.Stdin) + shutdown, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + if err := received.Shutdown(shutdown); err != nil { + return err + } + return readErr +} diff --git a/hack/crd/main.go b/hack/crd/main.go index 83b7d36..1290431 100644 --- a/hack/crd/main.go +++ b/hack/crd/main.go @@ -1,6 +1,6 @@ -// Command crd writes the TraceDefinition CustomResourceDefinition reflected -// from pkg/definition to definition.CRDFile, run from the repository root by -// `task generate`. +// Command crd writes the CustomResourceDefinitions reflected from +// pkg/definition (TraceDefinition) and pkg/deploy (Deployer) to crds/, run +// from the repository root by `task generate`. package main import ( @@ -9,6 +9,7 @@ import ( "path/filepath" "github.com/flanksource/telltale/pkg/definition" + "github.com/flanksource/telltale/pkg/deploy" ) func main() { @@ -19,12 +20,17 @@ func main() { } func write() error { - data, err := definition.CRD() - if err != nil { - return err + for file, generate := range map[string]func() ([]byte, error){definition.CRDFile: definition.CRD, deploy.CRDFile: deploy.CRD} { + data, err := generate() + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(file), 0o755); err != nil { + return err + } + if err := os.WriteFile(file, data, 0o644); err != nil { + return err + } } - if err := os.MkdirAll(filepath.Dir(definition.CRDFile), 0o755); err != nil { - return err - } - return os.WriteFile(definition.CRDFile, data, 0o644) + return nil } diff --git a/hack/demo/DemoWorkload.java b/hack/demo/DemoWorkload.java new file mode 100644 index 0000000..9846951 --- /dev/null +++ b/hack/demo/DemoWorkload.java @@ -0,0 +1,22 @@ +import java.util.concurrent.locks.LockSupport; + +public class DemoWorkload { + private static volatile long sink; + private static final Object lock = new Object(); + + public static void main(String[] args) throws Exception { + Runnable[] jobs = { + () -> { while (true) { for (int i = 0; i < 100_000; i++) sink += i; LockSupport.parkNanos(1_000_000); } }, + () -> { while (true) { byte[] allocation = new byte[1024 * 1024]; sink = allocation.length; LockSupport.parkNanos(10_000_000); } }, + () -> { while (true) { synchronized (lock) { LockSupport.parkNanos(30_000_000); } } }, + () -> { while (true) { synchronized (lock) { sink++; } LockSupport.parkNanos(10_000_000); } } + }; + for (Runnable job : jobs) { + Thread thread = new Thread(job); + thread.setDaemon(true); + thread.start(); + } + System.out.println("Demo workload ready"); + System.in.read(); + } +} diff --git a/hack/demo/README.md b/hack/demo/README.md new file mode 100644 index 0000000..a9aff64 --- /dev/null +++ b/hack/demo/README.md @@ -0,0 +1,69 @@ +# Record every trace kind + +Run this from the repository root to create one ordinary persisted Telltale run with fresh recordings for every kind returned by `traces.Registrations()`. The program exits unsuccessfully if any kind is missing, has no useful records, loses records, fails to stop, or fails cleanup. It never loads recorded fixtures. + +## Prerequisites + +- Run `make setup` and use the repository's local Go workspace. +- Put `git`, `java`, `javac`, `jcmd`, and `ps` on `PATH`. Use a JDK that supports Flight Recorder and allows attaching to a child JVM. +- Configure `connection://mssql` for **mssql.lab:31433**, with permission to create/drop a database, create/drop Extended Events sessions, inspect plan-cache DMVs, and read `system_health` deadlock events. This demo uses the lab server, not Docker. It creates only a uniquely named `telltale_demo_*` database and XE session and removes them after the captures drain. +- Authenticate a Captain API model with known pricing. Supply the model explicitly. One fresh, short request is made per recording, with a 128-token output limit, no tools or caching, and a maximum approved cost of USD 0.05. No LLM request is made by tests or `--verify-run`. + +To save the existing lab Connection resource in the shared connection store, run this once: + +```sh +go run ./hack/demo --save-connection-from "$HOME/.config/sqlglot-go/connections/lab.yaml" +``` + +The helper validates the endpoint, normalizes the database to `master`, saves `mssql.yaml` with mode `0600`, and refuses to overwrite an existing shared connection. Credentials stay in that connection file and memory; trace params contain `connection://mssql`. + +## Record and inspect + +```sh +go run ./hack/demo --model api:claude-haiku-5-5 +``` + +Use any authenticated Captain API model whose bounded request fits the cap. `--data-dir` defaults to `.tmp/demo/data`; `--timeout` defaults to `2m`. Progress goes to stderr and the final JSON report goes to stdout. The report includes the run ID, store directory, native record counts, final session summaries, and errors. It is also saved as `demo-report.json` in the run directory. The workload's SQLite database, Java source/classes, and JVM log remain in a sibling `work-*` directory for inspection. + +Recheck a saved recording without another paid request: + +```sh +go run ./hack/demo --verify-run +``` + +After `make build`, view the saved records using the local binary: + +```sh +.bin/telltale trace replay --data-dir .tmp/demo/data --format ndjson +.bin/telltale trace replay --data-dir .tmp/demo/data --format ansi +.bin/telltale trace replay --data-dir .tmp/demo/data --tui +.bin/telltale trace replay --data-dir .tmp/demo/data --ui +``` + +## Recorded work + +| Kinds | Real workload | +| --- | --- | +| `http`, `sql`, `logs`, `spans` | Instrumented localhost orders requests reading a freshly seeded SQLite database | +| `exec`, `terminal` | A Clicky execution and a separate PTY execution of `git --version`, including output, resize, and measured exit status | +| `stacks`, `goprof` | Periodic native stack and heap readings of the demo process while it holds allocations | +| `process`, `process/metrics`, `process/facets` | Native process identities, numeric observations, and resource events during the workloads | +| `jvm_jfr` | A child JVM allocating, computing, parking, and contending while Flight Recorder samples it | +| `prometheus`, `query` | VictoriaMetrics scraping the demo's request counter and a windowed PromQL query reading those samples | +| `sql_xevent`, `sql_plancache`, `sql_deadlock` | Lab database statements, execution deltas, and two transactions updating rows in opposite order to produce a deadlock | +| `sql_top` | The lab query's one-second `WAITFOR`, seen running by a 200ms poll | +| `llm` | One observed Captain API request and its actual response/token usage | + +The run is stopped and drained before the JVM, local HTTP server, and lab database are removed. Errors and partial traces are retained. A new catalog kind requires a new real workload and evidence predicate; it cannot silently pass. + +## Clicky UI event playground + +Export representative events through the same Clicky pretty printers used by CLI follow and replay, without another LLM call or external SQL workload: + +```sh +CLICKY_FORCE_INTERACTIVE=1 go run ./hack/demo --verify-run --export-playground ../clicky-ui/apps/playground/src/pages/telltale/_trace-events.json +``` + +The sibling Clicky UI playground page is `?page=telltale%2Ftrace-events`. Styled log rows are the primary view; Details, Native view, and Stored record tabs expose the full presentation and original payload. HTTP, SQL, XEvents, and execution rows reuse the existing shared formatting code. All registered kinds and named result tables are included, with variants such as JFR CPU/allocation/lock/park, LLM request/response, terminal output/resize/snapshot/exit, and SQL Extended Events. + +When the source recording predates the current process schema, the exporter records a three-second local process capture with an observed file descriptor. These process, metrics, and facet examples identify their own source run. Other examples retain their original recording, payloads, and sequence numbers. Counts describe the complete source streams; opening the playground requires no running capture services. diff --git a/hack/demo/coverage.go b/hack/demo/coverage.go new file mode 100644 index 0000000..0f62918 --- /dev/null +++ b/hack/demo/coverage.go @@ -0,0 +1,220 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/pkg/traces/read" +) + +type evidence struct { + Kind string `json:"kind"` + Count int `json:"count"` + Meaningful bool `json:"meaningful"` +} + +func validateCoverage(kinds []string, rows map[string][]query.Row) ([]evidence, error) { + var result []evidence + var failures []error + for _, kind := range kinds { + item := evidence{Kind: kind, Count: len(rows[kind])} + if kind == "prometheus" { + item.Meaningful = slices.ContainsFunc(rows["query"], func(row query.Row) bool { return meaningful("query", row) }) + } + for _, row := range rows[kind] { + item.Meaningful = item.Meaningful || meaningful(kind, row) + if err := captureLoss(map[string]any(row)); err != nil { + failures = append(failures, fmt.Errorf("%s: %w", kind, err)) + } + } + if kind == "terminal" { + item.Meaningful = item.Meaningful && slices.ContainsFunc(rows[kind], func(row query.Row) bool { return row["kind"] == "exit" && number(row["exitCode"]) == 0 }) + } + if kind == "llm" && (len(rows[kind]) != 2 || !slices.ContainsFunc(rows[kind], func(row query.Row) bool { return row["phase"] == "request" })) { + failures = append(failures, fmt.Errorf("llm: expected exactly one request and response")) + } + if !item.Meaningful { + failures = append(failures, fmt.Errorf("%s: %d rows, no successful workload evidence", kind, item.Count)) + } + result = append(result, item) + } + return result, errors.Join(failures...) +} + +func meaningful(kind string, row query.Row) bool { + switch kind { + case "exec": + return text(row["name"]) != "" && number(row["exitCode"]) == 0 && text(row["error"]) == "" + case "terminal": + return row["kind"] == "o" && text(row["data"]) != "" + case "goprof", "stacks": + return text(row["frames"]) != "" && text(row["frames"]) != "[]" + case "jvm_jfr": + return strings.Contains(text(row["stack"]), "DemoWorkload") && number(row["value"]) > 0 + case "sql_plancache": + return number(row["executions"]) > 0 + case "sql_top": + return number(row["sessionId"]) > 0 && strings.Contains(text(row["sql"]), "WAITFOR") && number(row["elapsedMs"]) > 0 + case "sql_deadlock": + return number(row["victims"]) > 0 && strings.Contains(text(row["xml"]), "deadlock") + case "sql_xevent", "sql": + return text(row["sql"]) != "" && text(row["error"]) == "" + case "llm": + usage, ok := row["usage"].(map[string]any) + return ok && row["phase"] == "response" && text(row["error"]) == "" && text(row["response"]) != "" && + number(usage["inputTokens"]) > 0 && number(usage["outputTokens"]) > 0 && number(row["cost"]) >= 0 && number(row["cost"]) <= llmBudget + case "http": + return number(row["status"]) == 200 && text(row["error"]) == "" + case "logs": + return text(row["message"]) != "" + case "spans": + return row["status"] == "ok" && text(row["operation"]) != "" + case "query": + values, ok := row["row"].(map[string]any) + return ok && row["provider"] == "prometheus" && values["__name__"] == "demo_requests_total" && number(values["value"]) > 0 + case "process": + return number(row["pid"]) > 0 && text(row["command"]) != "" && row["tags"] != nil + default: + return false + } +} + +func text(value any) string { + if value == nil { + return "" + } + if s, ok := value.(string); ok { + return s + } + data, err := json.Marshal(value) + if err != nil { + panic(fmt.Errorf("encode recorded value: %w", err)) + } + return string(data) +} + +func number(value any) float64 { + value64, err := strconv.ParseFloat(fmt.Sprint(value), 64) + if err != nil { + return math.NaN() + } + return value64 +} + +func captureLoss(values map[string]any) error { + for key, value := range values { + switch key { + case "dropped", "unencodable": + if number(value) != 0 { + return fmt.Errorf("%s records: %v", key, value) + } + case "truncated": + if value != nil && text(value) != "false" && text(value) != "[]" && text(value) != "" { + return fmt.Errorf("truncated records: %v", value) + } + } + if nested, ok := value.(map[string]any); ok { + if err := captureLoss(nested); err != nil { + return err + } + } + } + return nil +} + +type report struct { + RunID string `json:"runId"` + Directory string `json:"directory"` + Coverage []evidence `json:"coverage"` + Errors []string `json:"errors"` + Sessions []sessionEvidence `json:"sessions"` + WorkloadError string `json:"workloadError,omitempty"` +} + +type verification struct { + DataDir, RunID string + Failure error + Sessions []sessionEvidence +} + +func verifyStored(ctx context.Context, v verification) error { + dataDir, runID := v.DataDir, v.RunID + info, err := telltale.Get(telltale.GetOptions{DataDir: dataDir, RunID: runID}) + if err != nil { + return err + } + r := report{RunID: runID, Directory: info.Dir, Errors: []string{}, Sessions: v.Sessions} + if v.Sessions == nil { + previous, err := os.ReadFile(filepath.Join(info.Dir, "demo-report.json")) + if err != nil { + return fmt.Errorf("read original demo evidence: %w", err) + } + var original report + if err := json.Unmarshal(previous, &original); err != nil { + return err + } + r.WorkloadError, r.Sessions = original.WorkloadError, original.Sessions + } + if v.Failure != nil { + r.WorkloadError = v.Failure.Error() + } + if r.WorkloadError != "" { + r.Errors = append(r.Errors, r.WorkloadError) + } + if err := validateSessions(r.Sessions); err != nil { + r.Errors = append(r.Errors, err.Error()) + } + rows := map[string][]query.Row{} + var kinds []string + for _, registration := range demoRegistrations() { + kinds = append(kinds, registration.Name) + if registration.Name == "prometheus" { + continue + } + selected, err := read.Resolve(registration.Name, read.Scope{DataDir: dataDir, Run: runID}) + if err != nil { + r.Errors = append(r.Errors, err.Error()) + continue + } + for row, readErr := range read.Rows(ctx, selected.Refs, read.Query{All: true}) { + if readErr != nil { + r.Errors = append(r.Errors, readErr.Error()) + break + } + rows[registration.Name] = append(rows[registration.Name], row) + } + if err := selected.Close(); err != nil { + r.Errors = append(r.Errors, err.Error()) + } + } + r.Coverage, err = validateCoverage(kinds, rows) + if err != nil { + r.Errors = append(r.Errors, err.Error()) + } + if info.Error != "" { + r.Errors = append(r.Errors, info.Error) + } + r.Errors = append(r.Errors, info.CheckpointErrors...) + data, err := json.MarshalIndent(r, "", " ") + if err != nil { + return err + } + if err := os.WriteFile(filepath.Join(info.Dir, "demo-report.json"), append(data, '\n'), 0o600); err != nil { + return err + } + fmt.Println(string(data)) + if len(r.Errors) != 0 { + return fmt.Errorf("demo run %s failed verification; see %s/demo-report.json", runID, info.Dir) + } + return nil +} diff --git a/hack/demo/demo_test.go b/hack/demo/demo_test.go new file mode 100644 index 0000000..8e61ba0 --- /dev/null +++ b/hack/demo/demo_test.go @@ -0,0 +1,72 @@ +package main + +import ( + "encoding/json" + "testing" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/telltale/pkg/traces" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestDemo(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Real demo trace") } + +var _ = Describe("capture coverage", func() { + It("rejects missing kinds and empty captures", func() { + _, err := validateCoverage([]string{"exec", "terminal"}, map[string][]query.Row{"exec": {{"exitCode": 0, "name": "git"}}}) + Expect(err).To(MatchError(ContainSubstring("terminal"))) + }) + It("requires a stored metric sample as evidence for the Prometheus capture", func() { + _, err := validateCoverage([]string{"prometheus"}, map[string][]query.Row{}) + Expect(err).To(MatchError(ContainSubstring("prometheus"))) + }) + It("accepts genuine successful captures and reports their complete coverage", func() { + result, err := validateCoverage([]string{"exec", "terminal"}, map[string][]query.Row{ + "exec": {{"exitCode": 0, "name": "git"}}, + "terminal": {{"kind": "o", "data": "git version"}, {"kind": "exit", "exitCode": 0}}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal([]evidence{{Kind: "exec", Count: 1, Meaningful: true}, {Kind: "terminal", Count: 2, Meaningful: true}})) + }) + It("rejects a terminal without a measured exit status", func() { + _, err := validateCoverage([]string{"terminal"}, map[string][]query.Row{"terminal": {{"kind": "o", "data": "git version"}, {"kind": "exit"}}}) + Expect(err).To(HaveOccurred()) + }) + It("rejects a capture with dropped records", func() { + _, err := validateCoverage([]string{"exec"}, map[string][]query.Row{"exec": {{"name": "git", "exitCode": 0, "dropped": 1}}}) + Expect(err).To(HaveOccurred()) + }) + It("recognizes native JFR stacks and windowed Prometheus query rows", func() { + coverage, err := validateCoverage([]string{"jvm_jfr", "prometheus", "query"}, map[string][]query.Row{ + "jvm_jfr": {{"stack": "DemoWorkload.compute", "value": 2}}, + "query": {{"provider": "prometheus", "row": map[string]any{"__name__": "demo_requests_total", "value": 3}}}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(coverage).To(ContainElement(evidence{Kind: "prometheus", Meaningful: true})) + }) + It("validates parameters for every registered kind before starting workloads", func() { + specs, err := demoSpecs("http://127.0.0.1:43210", "telltale_demo_test", 12345) + Expect(err).NotTo(HaveOccurred()) + kinds, err := traces.Kinds() + Expect(err).NotTo(HaveOccurred()) + var names []string + for _, spec := range specs { + names = append(names, spec.Kind) + Expect(traces.Validate(kinds, spec.Kind, spec.Params)).To(Succeed(), spec.Kind) + } + var recorded []string + for _, registration := range demoRegistrations() { + recorded = append(recorded, registration.Name) + } + Expect(names).To(Equal(recorded)) + Expect(traces.KindNames(kinds)).To(ContainElements(recorded)) + }) + It("rejects loss reported inside a completed session summary", func() { + Expect(validateSessions([]sessionEvidence{{Kind: "exec", State: query.SessionCompleted, Summary: json.RawMessage(`{"source":{"dropped":1}}`)}})).To(MatchError(ContainSubstring("dropped"))) + }) + It("requires a single LLM request with measured response usage", func() { + _, err := validateCoverage([]string{"llm"}, map[string][]query.Row{"llm": {{"phase": "request"}, {"phase": "response", "response": "example reply"}}}) + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/hack/demo/export.go b/hack/demo/export.go new file mode 100644 index 0000000..c8afc70 --- /dev/null +++ b/hack/demo/export.go @@ -0,0 +1,195 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "time" + + "github.com/flanksource/clicky" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/pkg/traces/read" +) + +type playgroundEvent struct { + Variant string `json:"variant"` + Seq int64 `json:"seq"` + ANSI string `json:"ansi"` + DetailANSI string `json:"detailAnsi"` + Record query.Row `json:"record"` + Stream string `json:"stream"` + Timestamp any `json:"timestamp"` +} + +type playgroundKind struct { + Name string `json:"name"` + Title string `json:"title"` + Description string `json:"description"` + Count int `json:"count"` + Events []playgroundEvent `json:"events"` + SourceRunID string `json:"sourceRunId"` + rootPID int +} + +type playgroundRecording struct { + RunID string `json:"runId"` + StartedAt time.Time `json:"startedAt"` + EndedAt time.Time `json:"endedAt"` + Kinds []playgroundKind `json:"kinds"` + Processes read.ProcessSnapshot `json:"processes"` +} + +func exportPlayground(ctx context.Context, o options) (err error) { + info, err := telltale.Get(telltale.GetOptions{DataDir: o.dataDir, RunID: o.verifyRun}) + if err != nil { + return err + } + if info.Status != telltale.StatusSucceeded { + return fmt.Errorf("export needs a succeeded run, got %s", info.Status) + } + result := playgroundRecording{RunID: info.RunID, StartedAt: info.StartedAt, EndedAt: info.EndedAt} + sources, closeSources, err := playgroundSources(ctx, o, info) + if err != nil { + return err + } + defer func() { err = errors.Join(err, closeSources()) }() + process := sources["process"] + refs, err := read.RunRefs(process.info, process.runtime) + if err != nil { + return err + } + result.Processes, err = read.Processes(ctx, refs) + if err != nil { + return err + } + for _, registration := range demoRegistrations() { + source := sources["recording"] + if registration.Name == "process" { + source = sources["process"] + } + items, err := source.samples(ctx, registration) + if err != nil { + return err + } + result.Kinds = append(result.Kinds, items...) + } + data, err := json.MarshalIndent(result, "", " ") + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(o.exportPlayground), 0o755); err != nil { + return err + } + if err := os.WriteFile(o.exportPlayground, append(data, '\n'), 0o600); err != nil { + return err + } + fmt.Fprintf(os.Stderr, "Exported %d recorded kinds to %s\n", len(result.Kinds), o.exportPlayground) + return nil +} + +func playgroundSamples(ctx context.Context, refs []read.Ref, item playgroundKind) (playgroundKind, error) { + byVariant := map[string]read.Record{} + q := read.FeedQuery{Query: read.Query{Limit: 500}} + for { + page, err := read.FeedPage(ctx, refs, q) + if err != nil { + return item, err + } + for _, record := range page.Rows { + item.Count++ + if item.Name == "process" && number(record.Record["pid"]) != float64(item.rootPID) { + continue + } + if item.Name == "process_metrics" && !strings.HasPrefix(text(record.Record["entity"]), fmt.Sprintf("%d@", item.rootPID)) { + continue + } + if item.Name == "process_facets" && !strings.HasPrefix(text(record.Record["process"]), fmt.Sprintf("%d@", item.rootPID)) { + continue + } + key := exportSampleKey(item.Name, record.Record) + if key == "" { + continue + } + previous, exists := byVariant[key] + if !exists || number(record.Record["value"]) > number(previous.Record["value"]) { + byVariant[key] = record + } + } + if page.NextCursor == "" { + break + } + q.Cursor = page.NextCursor + } + if len(byVariant) == 0 { + return item, fmt.Errorf("%s: no meaningful recorded playground sample", item.Name) + } + for variant, record := range byVariant { + ansi, err := clicky.Format(record.Pretty(), clicky.FormatOptions{Format: "pretty"}) + if err != nil { + return item, err + } + detail, err := clicky.Format(record.PrettyFull(), clicky.FormatOptions{Format: "pretty"}) + if err != nil { + return item, err + } + item.Events = append(item.Events, playgroundEvent{Variant: variant, Seq: record.Seq, ANSI: ansi, DetailANSI: detail, Record: record.Record, Stream: record.Stream, Timestamp: record.Timestamp}) + } + slices.SortFunc(item.Events, func(a, b playgroundEvent) int { + if a.Seq < b.Seq { + return -1 + } + if a.Seq > b.Seq { + return 1 + } + return 0 + }) + return item, nil +} + +func exportSampleKey(kind string, row query.Row) string { + switch kind { + case "exec": + if filepath.Base(text(row["name"])) == "git" && meaningful(kind, row) { + return "record" + } + case "terminal": + return text(row["kind"]) + case "llm": + return text(row["phase"]) + case "jvm_jfr": + if meaningful(kind, row) { + return text(row["event"]) + } + case "sql_xevent": + if row["name"] == "error_reported" { + if number(row["errorNumber"]) == 1205 { + return "error_reported" + } + } else if strings.Contains(text(row["sql"]), "orders") { + return text(row["name"]) + } + case "stacks": + if meaningful(kind, row) { + return text(row["state"]) + } + case "process_metrics": + if number(row["rssBytes"]) > 0 && row["cpuPercent"] != nil { + return "record" + } + case "process_facets": + if row["resource"] != nil { + return text(row["facet"]) + "/" + text(row["operation"]) + } + default: + if meaningful(kind, row) { + return "record" + } + } + return "" +} diff --git a/hack/demo/export_sources.go b/hack/demo/export_sources.go new file mode 100644 index 0000000..fa55c01 --- /dev/null +++ b/hack/demo/export_sources.go @@ -0,0 +1,112 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "os" + "slices" + "time" + + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/pkg/traces" + "github.com/flanksource/telltale/pkg/traces/read" + "github.com/flanksource/telltale/pkg/tracing" +) + +type playgroundSource struct { + info telltale.RunInfo + runtime *tracing.Runtime +} + +func playgroundSources(ctx context.Context, o options, info telltale.RunInfo) (map[string]playgroundSource, func() error, error) { + runtime, closeRecording, err := telltale.Reopen(o.dataDir, info.RunID) + if err != nil { + return nil, nil, err + } + recording := playgroundSource{info: info, runtime: runtime} + for _, capture := range info.Captures { + if capture.Kind == "process" { + return map[string]playgroundSource{"recording": recording, "process": recording}, closeRecording, nil + } + } + if info.Process != nil && info.Process.Tree.Kind == "process" { + return map[string]playgroundSource{"recording": recording, "process": recording}, closeRecording, nil + } + process, err := recordPlaygroundProcess(ctx, o.dataDir) + if err != nil { + return nil, nil, errors.Join(err, closeRecording()) + } + processRuntime, closeProcess, err := telltale.Reopen(o.dataDir, process.RunID) + if err != nil { + return nil, nil, errors.Join(err, closeRecording()) + } + return map[string]playgroundSource{"recording": recording, "process": {info: process, runtime: processRuntime}}, func() error { return errors.Join(closeRecording(), closeProcess()) }, nil +} + +func recordPlaygroundProcess(ctx context.Context, dataDir string) (info telltale.RunInfo, err error) { + file, err := os.CreateTemp(dataDir, "playground-observed-") + if err != nil { + return info, err + } + defer func() { err = errors.Join(err, file.Close()) }() + run, err := telltale.Start(ctx, telltale.StartOptions{DataDir: dataDir, PID: os.Getpid(), Duration: 3 * time.Second, + Traces: []telltale.TraceSpec{{Kind: "process", Params: json.RawMessage(`{"target":{"seam":"process"},"interval":"500ms","facets":["files"]}`)}}}) + if err != nil { + return info, err + } + defer func() { err = errors.Join(err, run.Stop()) }() + if _, err := run.Wait(); err != nil { + return info, err + } + info = run.Info() + if info.Status != telltale.StatusSucceeded { + return info, fmt.Errorf("local process capture %s: %s", info.RunID, info.Status) + } + fmt.Fprintln(os.Stderr, "Recorded current process examples in", info.RunID) + return info, nil +} + +func (s playgroundSource) samples(ctx context.Context, registration traces.Registration) ([]playgroundKind, error) { + primary := []tracing.TraceRef{} + for _, capture := range s.info.Captures { + if capture.Kind == registration.Name { + primary = append(primary, capture.Ref) + } + } + if s.info.Process != nil { + for _, ref := range []tracing.TraceRef{s.info.Process.Tree, s.info.Process.Stacks} { + if ref.Kind == registration.Name { + primary = append(primary, ref) + } + } + } + byKind, seen := map[string][]read.Ref{}, map[string]bool{} + for _, ref := range primary { + for _, result := range s.runtime.Kinds.CaptureRefs(ref) { + if seen[result.Stream] { + continue + } + seen[result.Stream] = true + byKind[result.Kind] = append(byKind[result.Kind], read.Ref{RunID: s.info.RunID, Runtime: s.runtime, TraceRef: result}) + } + } + if len(byKind) == 0 { + return nil, fmt.Errorf("run %s has no recorded %s capture", s.info.RunID, registration.Name) + } + items := []playgroundKind{} + for _, name := range slices.Sorted(maps.Keys(byKind)) { + plugin, ok := s.runtime.Kinds.Get(name) + if !ok { + return nil, fmt.Errorf("no registered presentation for %s", name) + } + item, err := playgroundSamples(ctx, byKind[name], playgroundKind{Name: name, Title: plugin.Title(), Description: registration.Summary, SourceRunID: s.info.RunID, rootPID: s.info.PID}) + if err != nil { + return nil, err + } + items = append(items, item) + } + return items, nil +} diff --git a/hack/demo/export_test.go b/hack/demo/export_test.go new file mode 100644 index 0000000..339d340 --- /dev/null +++ b/hack/demo/export_test.go @@ -0,0 +1,22 @@ +package main + +import ( + "github.com/flanksource/commons-db/query" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("playground sample selection", func() { + DescribeTable("keeps meaningful native event variants", + func(kind string, row query.Row, key string) { Expect(exportSampleKey(kind, row)).To(Equal(key)) }, + Entry("JFR CPU stack", "jvm_jfr", query.Row{"event": "cpu", "stack": "DemoWorkload.compute", "value": 2}, "cpu"), + Entry("JFR runtime-only sample", "jvm_jfr", query.Row{"event": "cpu", "stack": "[no stack]", "value": 2}, ""), + Entry("LLM request", "llm", query.Row{"phase": "request"}, "request"), + Entry("PTY resize", "terminal", query.Row{"kind": "r"}, "r"), + Entry("deadlock error", "sql_xevent", query.Row{"name": "error_reported", "errorNumber": 1205}, "error_reported"), + Entry("database context notice", "sql_xevent", query.Row{"name": "error_reported", "errorNumber": 5701}, ""), + Entry("process metrics", "process_metrics", query.Row{"rssBytes": 4096, "cpuPercent": 1.5}, "record"), + Entry("process descriptor", "process_facets", query.Row{"facet": "files", "operation": "open", "resource": map[string]any{"name": "playground-observed-recording"}}, "files/open"), + Entry("existing process descriptor", "process_facets", query.Row{"facet": "files", "operation": "open", "resource": map[string]any{"name": "/work/orders.db"}}, "files/open"), + ) +}) diff --git a/hack/demo/generate.go b/hack/demo/generate.go new file mode 100644 index 0000000..b302469 --- /dev/null +++ b/hack/demo/generate.go @@ -0,0 +1,137 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "time" + + "github.com/flanksource/captain/pkg/ai" + clickyexec "github.com/flanksource/clicky/exec" + "github.com/flanksource/telltale" +) + +func generate(ctx context.Context, o options) (err error) { + var workDir string + defer func() { + if err != nil && workDir != "" { + data, encodeErr := json.Marshal(map[string]string{"error": err.Error(), "dataDir": o.dataDir, "workDir": workDir}) + if encodeErr != nil { + err = errors.Join(err, encodeErr) + return + } + err = errors.Join(err, os.WriteFile(filepath.Join(workDir, "demo-failure.json"), append(data, '\n'), 0o600)) + fmt.Fprintln(os.Stderr, "Partial demo artifacts:", workDir) + } + }() + for _, binary := range []string{"git", "java", "javac", "jcmd", "ps"} { + if _, err := exec.LookPath(binary); err != nil { + return fmt.Errorf("demo prerequisite: %w", err) + } + } + provider, err := prepareLLM(o.model) + if err != nil { + return err + } + lab, err := openLab(ctx) + if err != nil { + return err + } + defer func() { + if lab != nil { + err = errors.Join(err, lab.close()) + } + }() + if err := os.MkdirAll(filepath.Dir(o.dataDir), 0o700); err != nil { + return err + } + dir, err := os.MkdirTemp(filepath.Dir(o.dataDir), "work-") + if err != nil { + return err + } + workDir = dir + local, err := startLocal(ctx, dir) + if err != nil { + return err + } + defer func() { + if local != nil { + err = errors.Join(err, local.close()) + } + }() + jvm, err := startJava(ctx, dir) + if err != nil { + return err + } + defer func() { + if jvm != nil { + err = errors.Join(err, jvm.close()) + } + }() + if err := lab.create(ctx); err != nil { + return err + } + specs, err := demoSpecs(local.url, lab.name, jvm.cmd.Process.Pid) + if err != nil { + return err + } + run, err := telltale.Start(ctx, telltale.StartOptions{DataDir: o.dataDir, PID: os.Getpid(), Every: 3 * time.Second, Traces: specs, Connections: lab.connections}) + if err != nil { + return err + } + fmt.Fprintln(os.Stderr, "Recording demo", run.ID(), "into", run.Dir()) + workErr := exercise(ctx, run, local, lab, provider) + stopErr := run.Stop() + _, waitErr := run.Wait() + err = errors.Join(workErr, stopErr, waitErr, jvm.close(), local.close(), lab.close()) + jvm, local, lab = nil, nil, nil + verificationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + return verifyStored(verificationCtx, verification{DataDir: o.dataDir, RunID: run.ID(), Failure: err, Sessions: captureSessions(run.Runtime().Sessions.List())}) +} + +func exercise(ctx context.Context, run *telltale.Run, local *localWorkload, lab *labWorkload, provider ai.Provider) error { + for _, capture := range run.Info().Captures { + if capture.Kind == "terminal" { + if err := recordTerminal(ctx, capture.Ref.Stream); err != nil { + return err + } + } + } + if result := clickyexec.NewExec("git", "--version").WithContext(ctx).Run().Result(); result.Error != nil { + return result.Error + } + if err := lab.deadlock(ctx); err != nil { + return err + } + var held [][]byte + ticker := time.NewTicker(250 * time.Millisecond) + defer ticker.Stop() + until := time.NewTimer(20 * time.Second) + defer until.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-until.C: + if err := recordLLM(ctx, provider); err != nil { + return err + } + runtime.KeepAlive(held) + return nil + case <-ticker.C: + held = append(held, make([]byte, 64<<10)) + if err := local.request(ctx); err != nil { + return err + } + if err := lab.query(ctx); err != nil { + return err + } + } + } +} diff --git a/hack/demo/llm.go b/hack/demo/llm.go new file mode 100644 index 0000000..23b4d24 --- /dev/null +++ b/hack/demo/llm.go @@ -0,0 +1,56 @@ +package main + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/flanksource/captain/pkg/ai" + "github.com/flanksource/captain/pkg/ai/observe" + "github.com/flanksource/captain/pkg/ai/pricing" + _ "github.com/flanksource/captain/pkg/ai/provider" + "github.com/flanksource/captain/pkg/api" +) + +const llmBudget = 0.05 +const outputTokens = 128 +const demoPrompt = "Reply with one short sentence confirming that the acme checkout demonstration is ready." + +func prepareLLM(model string) (ai.Provider, error) { + provider, err := ai.NewProvider(ai.Config{Model: api.Model{Name: model}, Budget: api.Budget{Cost: llmBudget, MaxTokens: outputTokens}, NoCache: true}) + if err != nil { + return nil, err + } + if provider.GetRuntime().Mode != ai.ModeAPI { + return nil, fmt.Errorf("demo requires an API runtime with a bounded output token limit") + } + pricing.EnsureLoaded(pricing.LoadOptions{}) + price, err := pricing.CalculateCost(provider.GetModel(), 1024, outputTokens, outputTokens, 0, 0) + if err != nil { + return nil, fmt.Errorf("demo needs known model pricing: %w", err) + } + if price.TotalCost > llmBudget { + return nil, fmt.Errorf("maximum demo request price $%.4f exceeds $%.2f", price.TotalCost, llmBudget) + } + return observe.Wrap(provider), nil +} + +func recordLLM(ctx context.Context, provider ai.Provider) error { + ctx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + response, err := provider.Execute(ctx, ai.Request{Prompt: api.Prompt{User: demoPrompt}, Budget: api.Budget{Cost: llmBudget, MaxTokens: outputTokens, MaxTurns: 1}, Model: api.Model{Name: provider.GetModel(), NoCache: true}}) + if err != nil { + return err + } + if response == nil || strings.TrimSpace(response.Text) == "" { + return fmt.Errorf("LLM returned no text") + } + if response.Usage.TotalTokens() == 0 { + return fmt.Errorf("LLM returned no measured token usage") + } + if response.CostUSD > llmBudget { + return fmt.Errorf("LLM response cost $%.4f exceeded $%.2f", response.CostUSD, llmBudget) + } + return nil +} diff --git a/hack/demo/main.go b/hack/demo/main.go new file mode 100644 index 0000000..d0c5183 --- /dev/null +++ b/hack/demo/main.go @@ -0,0 +1,61 @@ +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "os/signal" + "syscall" + "time" + + "gorm.io/gorm/logger" +) + +type options struct { + dataDir, model, saveConnection, verifyRun, exportPlayground string + timeout time.Duration +} + +func main() { + logger.Default = logger.New(log.New(os.Stderr, "", log.LstdFlags), logger.Config{LogLevel: logger.Warn, SlowThreshold: 200 * time.Millisecond}) + var o options + flag.StringVar(&o.dataDir, "data-dir", ".tmp/demo/data", "Demo trace data directory") + flag.StringVar(&o.model, "model", "", "Captain API model for one request costing at most $0.05") + flag.StringVar(&o.saveConnection, "save-connection-from", "", "Save an existing Connection resource as shared connection://mssql, then exit") + flag.StringVar(&o.verifyRun, "verify-run", "", "Verify an existing demo run without making another LLM call") + flag.StringVar(&o.exportPlayground, "export-playground", "", "Export recorded ANSI events and native rows for the Clicky UI playground (requires --verify-run)") + flag.DurationVar(&o.timeout, "timeout", 2*time.Minute, "Overall recording deadline") + flag.Parse() + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := command(ctx, o); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func command(ctx context.Context, o options) error { + if o.exportPlayground != "" { + if o.verifyRun == "" { + return fmt.Errorf("--export-playground requires --verify-run") + } + return exportPlayground(ctx, o) + } + if o.saveConnection != "" { + return saveConnection(o.saveConnection) + } + if o.verifyRun != "" { + return verifyStored(ctx, verification{DataDir: o.dataDir, RunID: o.verifyRun}) + } + if o.model == "" { + return fmt.Errorf("--model is required; use an authenticated Captain API model") + } + if o.timeout <= 0 { + return fmt.Errorf("--timeout must be positive") + } + ctx, cancel := context.WithTimeout(ctx, o.timeout) + defer cancel() + return generate(ctx, o) +} diff --git a/hack/demo/mssql.go b/hack/demo/mssql.go new file mode 100644 index 0000000..eb19b86 --- /dev/null +++ b/hack/demo/mssql.go @@ -0,0 +1,159 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "time" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/task" + "github.com/flanksource/commons-db/connection/store" + dbcontext "github.com/flanksource/commons-db/context" + commonsctx "github.com/flanksource/commons/context" + mssql "github.com/microsoft/go-mssqldb" + "github.com/microsoft/go-mssqldb/msdsn" +) + +type labWorkload struct { + admin, db *sql.DB + name string + connections *store.Store + created bool +} + +func openLab(ctx context.Context) (*labWorkload, error) { + connections, err := store.Open("telltale") + if err != nil { + return nil, err + } + model, err := connections.ResolveConnection(dbcontext.NewContext(ctx), "connection://mssql") + if err != nil { + return nil, err + } + if model == nil { + return nil, fmt.Errorf("connection://mssql is not configured") + } + config, err := msdsn.Parse(model.URL) + if err != nil { + return nil, fmt.Errorf("invalid SQL Server DSN in connection://mssql") + } + if config.Host != "mssql.lab" || config.Port != 31433 { + return nil, fmt.Errorf("connection://mssql must target mssql.lab:31433") + } + config.Database = "master" + admin, err := sql.Open("sqlserver", config.URL().String()) + if err != nil { + return nil, fmt.Errorf("open lab SQL Server connection") + } + if err := admin.PingContext(ctx); err != nil { + return nil, errors.Join(err, admin.Close()) + } + var canCreate, canObserve int + if err := admin.QueryRowContext(ctx, "SELECT HAS_PERMS_BY_NAME(NULL, NULL, 'CREATE ANY DATABASE'), HAS_PERMS_BY_NAME(NULL, NULL, 'ALTER ANY EVENT SESSION')").Scan(&canCreate, &canObserve); err != nil { + return nil, errors.Join(err, admin.Close()) + } + if canCreate != 1 || canObserve != 1 { + return nil, errors.Join(fmt.Errorf("mssql needs CREATE ANY DATABASE and ALTER ANY EVENT SESSION"), admin.Close()) + } + return &labWorkload{admin: admin, connections: connections, name: fmt.Sprintf("telltale_demo_%d", time.Now().UnixNano())}, nil +} + +func (w *labWorkload) create(ctx context.Context) error { + if _, err := w.admin.ExecContext(ctx, "CREATE DATABASE ["+w.name+"]"); err != nil { + return err + } + w.created = true + // Credentials remain in memory; the trace only names connection://mssql. + connection, err := w.connections.ResolveConnection(dbcontext.NewContext(ctx), "connection://mssql") + if err != nil { + return err + } + config, err := msdsn.Parse(connection.URL) + if err != nil { + return fmt.Errorf("invalid SQL Server DSN") + } + config.Database, config.AppName = w.name, "telltale-demo" + w.db, err = sql.Open("sqlserver", config.URL().String()) + if err != nil { + return fmt.Errorf("open demo database") + } + _, err = w.db.ExecContext(ctx, "CREATE TABLE dbo.orders (id int PRIMARY KEY, value int); INSERT INTO dbo.orders VALUES (1, 0), (2, 0)") + return err +} + +func (w *labWorkload) query(ctx context.Context) error { + var total int + // The delay keeps the request running across sql_top polls. + return w.db.QueryRowContext(ctx, "WAITFOR DELAY '00:00:01'; SELECT SUM(value) FROM dbo.orders /* telltale_demo */").Scan(&total) +} + +func (w *labWorkload) deadlock(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + first, err := w.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = first.Rollback() }() + second, err := w.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = second.Rollback() }() + if _, err := first.ExecContext(ctx, "UPDATE dbo.orders SET value=value+1 WHERE id=1"); err != nil { + return err + } + if _, err := second.ExecContext(ctx, "UPDATE dbo.orders SET value=value+1 WHERE id=2"); err != nil { + return err + } + var handles []task.TypedTask[error] + for index, tx := range []*sql.Tx{first, second} { + id, transaction := 2-index, tx + handles = append(handles, clicky.StartTask("Induce demo deadlock", func(_ commonsctx.Context, _ *task.Task) (error, error) { + _, err := transaction.ExecContext(ctx, "UPDATE dbo.orders SET value=value+1 WHERE id=@p1", id) + rollback := transaction.Rollback() + if errors.Is(rollback, sql.ErrTxDone) { + rollback = nil + } + return err, rollback + }, task.WithContext(ctx), task.WithCancellationDrain())) + } + victims := 0 + for _, handle := range handles { + result, err := handle.GetResult() + if err != nil { + return err + } + if result != nil { + var serverError mssql.Error + if !errors.As(result, &serverError) || serverError.Number != 1205 { + return result + } + victims++ + } + } + if victims != 1 { + return fmt.Errorf("expected one deadlock victim, got %d", victims) + } + return nil +} + +func (w *labWorkload) close() error { + var err error + if w.db != nil { + err = w.db.Close() + } + if w.created { + if !strings.HasPrefix(w.name, "telltale_demo_") { + return errors.Join(err, fmt.Errorf("refusing to drop non-demo database"), w.admin.Close()) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, dropErr := w.admin.ExecContext(ctx, "DROP DATABASE ["+w.name+"]") + err = errors.Join(err, dropErr) + } + return errors.Join(err, w.admin.Close()) +} diff --git a/hack/demo/sessions.go b/hack/demo/sessions.go new file mode 100644 index 0000000..7d03ad5 --- /dev/null +++ b/hack/demo/sessions.go @@ -0,0 +1,48 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/flanksource/commons-db/query" +) + +type sessionEvidence struct { + Kind string `json:"kind"` + State query.SessionState `json:"state"` + Error string `json:"error,omitempty"` + Warning string `json:"warning,omitempty"` + Summary json.RawMessage `json:"summary"` +} + +func captureSessions(infos []query.SessionInfo) []sessionEvidence { + result := make([]sessionEvidence, 0, len(infos)) + for _, info := range infos { + result = append(result, sessionEvidence{Kind: strings.TrimPrefix(info.Profile, "traces/"), State: info.State, Error: info.Error, Warning: info.Warning, Summary: info.Summary}) + } + return result +} + +func validateSessions(sessions []sessionEvidence) error { + if len(sessions) == 0 { + return fmt.Errorf("no final session evidence recorded") + } + var failures []error + for _, session := range sessions { + if session.State != query.SessionStopped && session.State != query.SessionCompleted { + failures = append(failures, fmt.Errorf("%s session ended %s: %s", session.Kind, session.State, session.Error)) + } + if session.Warning != "" || session.Error != "" { + failures = append(failures, fmt.Errorf("%s session: %s %s", session.Kind, session.Error, session.Warning)) + } + var summary map[string]any + if err := json.Unmarshal(session.Summary, &summary); err != nil { + failures = append(failures, fmt.Errorf("%s session summary: %w", session.Kind, err)) + } else if err := captureLoss(summary); err != nil { + failures = append(failures, fmt.Errorf("%s session: %w", session.Kind, err)) + } + } + return errors.Join(failures...) +} diff --git a/hack/demo/setup.go b/hack/demo/setup.go new file mode 100644 index 0000000..6574b23 --- /dev/null +++ b/hack/demo/setup.go @@ -0,0 +1,57 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/flanksource/commons-db/connection/crd" + "github.com/flanksource/commons-db/connection/store" + "github.com/flanksource/commons-db/types" + "github.com/microsoft/go-mssqldb/msdsn" + "sigs.k8s.io/yaml" +) + +func saveConnection(source string) error { + data, err := os.ReadFile(source) + if err != nil { + return err + } + var resource crd.Connection + if err := yaml.UnmarshalStrict(data, &resource); err != nil { + return fmt.Errorf("decode connection resource: invalid YAML") + } + model, err := crd.ToModel(&resource) + if err != nil { + return fmt.Errorf("decode connection resource: invalid connection") + } + config, err := msdsn.Parse(model.URL) + if err != nil { + return fmt.Errorf("parse SQL Server connection: invalid DSN") + } + if config.Host != "mssql.lab" || config.Port != 31433 { + return fmt.Errorf("expected mssql.lab:31433") + } + config.Database = "master" + resource.Name, resource.Namespace = "mssql", "" + resource.Spec = crd.ConnectionSpec{Type: "sql_server", URL: types.EnvVar{ValueStatic: config.URL().String()}} + connections, err := store.Open("telltale") + if err != nil { + return err + } + global, err := store.ConfigDir(store.GlobalApp) + if err != nil { + return err + } + path := filepath.Join(global, "connections", "mssql.yaml") + if _, err := os.Stat(path); err == nil { + return fmt.Errorf("connection://mssql already exists; refusing to overwrite %s", path) + } else if !os.IsNotExist(err) { + return err + } + if err := connections.Save(store.LayerGlobal, &resource); err != nil { + return err + } + fmt.Println("Saved connection://mssql at", path) + return nil +} diff --git a/hack/demo/setup_test.go b/hack/demo/setup_test.go new file mode 100644 index 0000000..0662442 --- /dev/null +++ b/hack/demo/setup_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "os" + "path/filepath" + + "github.com/flanksource/commons-db/connection/crd" + "github.com/flanksource/commons-db/types" + "github.com/microsoft/go-mssqldb/msdsn" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/yaml" +) + +var _ = Describe("shared lab connection", func() { + It("normalizes the lab DSN, saves privately, and refuses to overwrite it", func() { + dir := GinkgoT().TempDir() + GinkgoT().Setenv("XDG_CONFIG_HOME", dir) + resource := crd.Connection{} + resource.Name = "lab" + resource.Spec = crd.ConnectionSpec{Type: "sql_server", URL: types.EnvVar{ValueStatic: "sqlserver://mssql.lab:31433?database=example"}} + data, err := yaml.Marshal(resource) + Expect(err).NotTo(HaveOccurred()) + source := filepath.Join(dir, "lab.yaml") + Expect(os.WriteFile(source, data, 0o600)).To(Succeed()) + Expect(saveConnection(source)).To(Succeed()) + path := filepath.Join(dir, "flanksource", "connections", "mssql.yaml") + info, err := os.Stat(path) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + stored, err := os.ReadFile(path) + Expect(err).NotTo(HaveOccurred()) + Expect(yaml.UnmarshalStrict(stored, &resource)).To(Succeed()) + config, err := msdsn.Parse(resource.Spec.URL.ValueStatic) + Expect(err).NotTo(HaveOccurred()) + Expect(config.Database).To(Equal("master")) + Expect(resource.Name).To(Equal("mssql")) + Expect(saveConnection(source)).To(MatchError(ContainSubstring("refusing to overwrite"))) + again, err := os.ReadFile(path) + Expect(err).NotTo(HaveOccurred()) + Expect(again).To(Equal(stored)) + }) +}) diff --git a/hack/demo/specs.go b/hack/demo/specs.go new file mode 100644 index 0000000..9d2ca25 --- /dev/null +++ b/hack/demo/specs.go @@ -0,0 +1,62 @@ +package main + +import ( + "encoding/json" + "fmt" + "slices" + + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/pkg/traces" +) + +// demoRegistrations are the kinds the demo records: every registered kind but +// the seam presets (Registration.Base), which capture through their base +// kind's code on a seam the demo has no workload for (its proxies need a +// command pointed at them and, for pg_proxy, a Postgres upstream). +func demoRegistrations() []traces.Registration { + return slices.DeleteFunc(traces.Registrations(), func(registration traces.Registration) bool { return registration.Base != "" }) +} + +func demoSpecs(url, database string, javaPID int) ([]telltale.TraceSpec, error) { + self := map[string]any{"seam": "self"} + params := map[string]map[string]any{ + "process": {"target": map[string]any{"seam": "process"}, "interval": "500ms", "facets": []string{"files", "sockets", "connections"}}, + "stacks": {"target": self}, + "goprof": {"target": self}, + "http": {"target": self, "features": []string{"demo"}}, + "sql": {"target": self, "connections": []string{"demo"}}, + "exec": {"target": self, "tailBytes": 4096}, + "logs": {"target": self, "minLevel": "info"}, + "spans": {"target": self}, + "terminal": {"target": self, "width": 100, "height": 24, "command": []string{"git", "--version"}}, + "jvm_jfr": {"target": map[string]any{"seam": "process", "pid": javaPID}, "engine": "jfr", "settings": "profile", "threads": true}, + "prometheus": {"target": map[string]any{"seam": "host"}, "endpoints": []string{url + "/metrics"}, "interval": "1s"}, + "query": {"target": map[string]any{"seam": "host"}, "untilStopped": true, "provider": map[string]any{"type": "prometheus", "connection": "connection://telltale-prometheus", "query": "demo_requests_total", "prometheus": map[string]any{"range": map[string]any{"step": "1s"}}}}, + "sql_xevent": {"connection": "connection://mssql", "databases": []string{database}, "sessionName": database, "poll": "200ms"}, + "sql_plancache": {"connection": "connection://mssql", "databases": []string{database}, "poll": "1s", "top": 20}, + "sql_deadlock": {"connection": "connection://mssql", "databases": []string{database}, "poll": "1s"}, + "sql_top": {"connection": "connection://mssql", "databases": []string{database}, "poll": "200ms"}, + "llm": {"target": self, "includePrompts": true, "includeResponses": true}, + } + registrations := demoRegistrations() + if len(params) != len(registrations) { + return nil, fmt.Errorf("demo workload catalog has %d kinds; registry has %d", len(params), len(registrations)) + } + var specs []telltale.TraceSpec + for _, registration := range registrations { + p, exists := params[registration.Name] + if !exists { + return nil, fmt.Errorf("no real demo workload for registered kind %s", registration.Name) + } + raw, err := json.Marshal(p) + if err != nil { + return nil, err + } + spec := telltale.TraceSpec{Kind: registration.Name, Params: raw} + if registration.Name == "goprof" { + spec.Checkpoint = json.RawMessage(`{"profile":"heap","source":"gops","reason":"demo"}`) + } + specs = append(specs, spec) + } + return specs, nil +} diff --git a/hack/demo/workloads.go b/hack/demo/workloads.go new file mode 100644 index 0000000..a450063 --- /dev/null +++ b/hack/demo/workloads.go @@ -0,0 +1,171 @@ +package main + +import ( + "context" + "database/sql" + _ "embed" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "sync/atomic" + "syscall" + "time" + + "github.com/creack/pty" + "github.com/flanksource/clicky" + clickyexec "github.com/flanksource/clicky/exec" + "github.com/flanksource/clicky/task" + "github.com/flanksource/commons-db/observability" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/query/providers/sqlite" + commonsctx "github.com/flanksource/commons/context" + "github.com/flanksource/telltale/pkg/tracing/terminal" + sdk "github.com/flanksource/telltale/sdk" +) + +//go:embed DemoWorkload.java +var javaSource []byte + +type localWorkload struct { + url string + db *sql.DB + server *http.Server + serve task.TypedTask[struct{}] + requests atomic.Int64 +} + +func startLocal(ctx context.Context, dir string) (*localWorkload, error) { + db, err := sql.Open("sqlite", filepath.Join(dir, "orders.sqlite")) + if err != nil { + return nil, err + } + if _, err := db.ExecContext(ctx, "CREATE TABLE orders (id INTEGER PRIMARY KEY, name TEXT); INSERT INTO orders VALUES (1, 'acme checkout'), (2, 'acme shipment')"); err != nil { + return nil, errors.Join(err, db.Close()) + } + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, errors.Join(err, db.Close()) + } + w := &localWorkload{db: db, url: "http://" + listener.Addr().String()} + mux := http.NewServeMux() + mux.HandleFunc("GET /orders", w.orders) + mux.HandleFunc("GET /metrics", func(out http.ResponseWriter, _ *http.Request) { + out.Header().Set("Content-Type", "text/plain; version=0.0.4") + if _, err := fmt.Fprintf(out, "# TYPE demo_requests_total counter\ndemo_requests_total %d\n", w.requests.Load()); err != nil { + slog.Error("write metrics", "error", err) + } + }) + w.server = &http.Server{Handler: mux, ReadHeaderTimeout: 5 * time.Second} + w.serve = clicky.StartTask("Demo HTTP server", func(_ commonsctx.Context, _ *task.Task) (struct{}, error) { + err := w.server.Serve(listener) + if errors.Is(err, http.ErrServerClosed) { + err = nil + } + return struct{}{}, err + }, task.WithContext(ctx), task.WithCancellationDrain()) + return w, nil +} + +func (w *localWorkload) orders(out http.ResponseWriter, request *http.Request) { + ctx, end := sdk.StartSpan(request.Context(), "demo.orders", "service.name", "telltale-demo") + result, err := sqlite.ReadSQLPage(ctx, w.db, "sqlite", sqlite.SQLPageRequest{Connection: "demo", Query: "SELECT id, name FROM orders", Page: query.PageRequest{Limit: 10}}) + end(err) + if err != nil { + http.Error(out, "read demo orders", http.StatusInternalServerError) + return + } + w.requests.Add(1) + slog.New(sdk.SlogHandler(slog.NewTextHandler(io.Discard, nil))).InfoContext(ctx, "demo orders read", "rows", len(result.Rows)) + out.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(out).Encode(result.Rows); err != nil { + slog.Error("write orders", "error", err) + } +} + +func (w *localWorkload) request(ctx context.Context) error { + client := &http.Client{Transport: observability.ApplyHTTPObservability(ctx, "demo", nil, nil), Timeout: 5 * time.Second} + req, err := http.NewRequestWithContext(ctx, http.MethodGet, w.url+"/orders", nil) + if err != nil { + return err + } + response, err := client.Do(req) + if err != nil { + return err + } + _, readErr := io.Copy(io.Discard, response.Body) + if response.StatusCode != http.StatusOK { + readErr = errors.Join(readErr, fmt.Errorf("orders returned %d", response.StatusCode)) + } + return errors.Join(readErr, response.Body.Close()) +} + +func (w *localWorkload) close() error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + err := w.server.Shutdown(ctx) + if err != nil { + err = errors.Join(err, w.server.Close()) + } + _, serveErr := w.serve.GetResult() + return errors.Join(err, serveErr, w.db.Close()) +} + +type javaWorkload struct { + cmd *exec.Cmd + input io.WriteCloser + output *os.File +} + +func startJava(ctx context.Context, dir string) (*javaWorkload, error) { + source := filepath.Join(dir, "DemoWorkload.java") + if err := os.WriteFile(source, javaSource, 0o600); err != nil { + return nil, err + } + if result := clickyexec.NewExec("javac", "-d", dir, source).WithContext(ctx).Run().Result(); result.Error != nil { + return nil, fmt.Errorf("compile demo JVM: %w", result.Error) + } + output, err := os.Create(filepath.Join(dir, "jvm.log")) + if err != nil { + return nil, err + } + cmd := exec.CommandContext(ctx, "java", "-Xmx64m", "-cp", dir, "DemoWorkload") + cmd.Stdout, cmd.Stderr = output, output + input, err := cmd.StdinPipe() + if err != nil { + return nil, errors.Join(err, output.Close()) + } + if err := cmd.Start(); err != nil { + return nil, errors.Join(err, input.Close(), output.Close()) + } + return &javaWorkload{cmd: cmd, input: input, output: output}, nil +} + +func (w *javaWorkload) close() error { + return errors.Join(w.input.Close(), w.cmd.Wait(), w.output.Close()) +} + +func recordTerminal(ctx context.Context, stream string) (err error) { + writer, err := terminal.Writer(stream) + if err != nil { + return err + } + defer func() { err = errors.Join(err, writer.Close()) }() + cmd := exec.CommandContext(ctx, "git", "--version") + file, err := pty.StartWithSize(cmd, &pty.Winsize{Rows: 24, Cols: 100}) + if err != nil { + return err + } + _, copyErr := io.Copy(writer, file) + if errors.Is(copyErr, syscall.EIO) { + copyErr = nil + } + waitErr := cmd.Wait() + return errors.Join(copyErr, waitErr, terminal.Resize(stream, 120, 30), terminal.SetExitCode(stream, cmd.ProcessState.ExitCode()), file.Close()) +} diff --git a/hack/docs-catalog/main.go b/hack/docs-catalog/main.go new file mode 100644 index 0000000..d85b50c --- /dev/null +++ b/hack/docs-catalog/main.go @@ -0,0 +1,82 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "os" + + "github.com/flanksource/telltale/internal/cli" + "github.com/flanksource/telltale/pkg/traces" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/spf13/pflag" +) + +type kind struct { + tracing.KindInfo + Flags []argument `json:"flags"` +} + +type argument struct { + Name string `json:"name"` + Type string `json:"type"` + Description string `json:"description"` + Default string `json:"default"` + Required bool `json:"required"` +} + +func main() { + check := flag.Bool("check", false, "check that the documentation catalog is current") + flag.Parse() + kinds, err := traces.Kinds() + if err != nil { + fatal(err) + } + list, err := kinds.List() + if err != nil { + fatal(err) + } + root := cli.NewRoot("dev") + entries := make([]kind, 0, len(list)) + for _, info := range list { + cmd, _, err := root.Find([]string{"run", info.Name}) + if err != nil || cmd.Name() != info.Name { + fatal(fmt.Errorf("find command for trace kind %s: %v", info.Name, err)) + } + entry := kind{KindInfo: info, Flags: []argument{}} + cmd.Flags().VisitAll(func(f *pflag.Flag) { + if !f.Hidden && f.Name != "help" { + entry.Flags = append(entry.Flags, argument{Name: f.Name, Type: f.Value.Type(), Description: f.Usage, Default: f.DefValue, Required: len(f.Annotations["cobra_annotation_bash_completion_one_required_flag"]) > 0}) + } + }) + entries = append(entries, entry) + } + data, err := json.MarshalIndent(entries, "", " ") + if err != nil { + fatal(err) + } + data = append(data, '\n') + const path = "docs/_data/kinds.json" + if *check { + current, err := os.ReadFile(path) + if err != nil { + fatal(err) + } + if !bytes.Equal(current, data) { + fatal(fmt.Errorf("%s is stale; run go run ./hack/docs-catalog", path)) + } + return + } + if err := os.MkdirAll("docs/_data", 0o755); err != nil { + fatal(err) + } + if err := os.WriteFile(path, data, 0o644); err != nil { + fatal(err) + } +} + +func fatal(err error) { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) +} diff --git a/hack/jms-profile-fixture/main.go b/hack/jms-profile-fixture/main.go new file mode 100644 index 0000000..290a5de --- /dev/null +++ b/hack/jms-profile-fixture/main.go @@ -0,0 +1,29 @@ +// Emit a deterministic-shape JMS handler profile for local viewer smoke checks. +package main + +import ( + "fmt" + "github.com/flanksource/telltale/pkg/otlp/profiles" + "github.com/google/uuid" + "os" + "time" +) + +func main() { + if len(os.Args) != 2 { + panic("usage: go run ./hack/jms-profile-fixture ") + } + now := time.Now() + snapshot := profiles.Snapshot{Kind: "jms_top", CaptureID: "handler-fixture", Source: "fixture-jvm", ProfileID: uuid.NewString(), Sequence: 1, Complete: true, Observed: 2, Limit: 50, StartedAt: now, CompletedAt: now, Resource: map[string]any{"service.name": "jms-viewer-fixture"}, Rows: []profiles.Row{ + {OperationID: "orders-handler", ElapsedNS: int64(2 * time.Second), SampledAt: now, Attributes: map[string]any{"destination": "orders", "handler": "OrderListener.accept", "state": "busy", "activeCount": 2}}, + {OperationID: "replies-handler", SampledAt: now, Attributes: map[string]any{"destination": "replies", "handler": "ReplyListener.accept", "state": "idle", "activeCount": 0}}, + }} + data, err := profiles.Encode([]profiles.Snapshot{snapshot}) + if err != nil { + panic(err) + } + if err := os.WriteFile(os.Args[1], data, 0600); err != nil { + panic(err) + } + fmt.Println(os.Args[1]) +} diff --git a/hack/jvm-demo/checks.go b/hack/jvm-demo/checks.go new file mode 100644 index 0000000..6df3fd5 --- /dev/null +++ b/hack/jvm-demo/checks.go @@ -0,0 +1,140 @@ +package main + +import ( + "encoding/base64" + "fmt" + "strings" + "time" + + "github.com/flanksource/telltale/pkg/otlp/profiles" + "go.opentelemetry.io/collector/pdata/pprofile" +) + +type topFrame struct { + ID string `json:"id"` + Source string `json:"source"` + Tick int64 `json:"tick"` + CompletedAt time.Time `json:"completedAt"` + Summary map[string]any `json:"summary"` +} + +type topSnapshot struct { + Frame topFrame `json:"frame"` + Rows []map[string]any `json:"rows"` + Count int `json:"count"` +} + +func decodeSnapshot(top topSnapshot) (profiles.Snapshot, error) { + native, ok := top.Frame.Summary["otlp"].(string) + if !ok || native == "" { + return profiles.Snapshot{}, fmt.Errorf("top frame %s has no native Profiles evidence", top.Frame.ID) + } + data, err := base64.StdEncoding.DecodeString(native) + if err != nil { + return profiles.Snapshot{}, fmt.Errorf("decode native Profiles base64: %w", err) + } + snapshots, err := profiles.Decode(data, profiles.Options{}) + if err != nil { + return profiles.Snapshot{}, fmt.Errorf("decode native Profiles: %w", err) + } + if len(snapshots) != 1 { + return profiles.Snapshot{}, fmt.Errorf("expected one native snapshot, received %d", len(snapshots)) + } + snapshot := snapshots[0] + if snapshot.Sequence != top.Frame.Tick || snapshot.ProfileID != top.Frame.ID { + return profiles.Snapshot{}, fmt.Errorf("projected frame identity differs from native Profiles") + } + return snapshot, nil +} + +func checkActive(kind string, top topSnapshot) error { + snapshot, err := decodeSnapshot(top) + if err != nil { + return err + } + if snapshot.Kind != kind || !snapshot.Complete { + return fmt.Errorf("expected complete %s snapshot, received %s complete=%t", kind, snapshot.Kind, snapshot.Complete) + } + if len(top.Rows) != len(snapshot.Rows) || top.Count != len(snapshot.Rows) { + return fmt.Errorf("native and projected operation counts differ: native=%d rows=%d count=%d", len(snapshot.Rows), len(top.Rows), top.Count) + } + field, values := "direction", []string{"incoming", "outgoing"} + if kind == "sql_top" { + field, values = "api", []string{"jdbc", "r2dbc"} + } + for _, value := range values { + if !hasOperation(snapshot.Rows, field, value) { + return fmt.Errorf("%s has no active demo %s=%s operation", kind, field, value) + } + } + return nil +} + +func hasOperation(rows []profiles.Row, field, value string) bool { + for _, row := range rows { + if row.Attributes[field] != value || row.OperationID == "" || row.ElapsedNS <= 0 { + continue + } + if field == "api" && strings.Contains(strings.ToUpper(fmt.Sprint(row.Attributes["sql"])), "DEMO_HOLD") { + return true + } + if field == "direction" && strings.Contains(fmt.Sprint(row.Attributes["url"]), "/demo/") { + if value == "outgoing" || strings.Contains(fmt.Sprint(row.Attributes["route"]), "/demo/{") { + return true + } + } + } + return false +} + +func checkUpdated(previous, current topSnapshot) error { + before, err := decodeSnapshot(previous) + if err != nil { + return err + } + after, err := decodeSnapshot(current) + if err != nil { + return err + } + if before.Kind != after.Kind || before.CaptureID != after.CaptureID || before.Source != after.Source { + return fmt.Errorf("snapshot capture identity changed while observing held operations") + } + if after.Sequence <= before.Sequence { + return fmt.Errorf("snapshot sequence has not advanced: previous=%d current=%d", before.Sequence, after.Sequence) + } + currentRows := make(map[string]profiles.Row, len(after.Rows)) + for _, row := range after.Rows { + currentRows[row.OperationID] = row + } + for _, row := range before.Rows { + updated, ok := currentRows[row.OperationID] + if !ok { + return fmt.Errorf("held operation %s disappeared", row.OperationID) + } + if updated.ElapsedNS <= row.ElapsedNS { + return fmt.Errorf("operation %s elapsed did not increase: previous=%d current=%d", row.OperationID, row.ElapsedNS, updated.ElapsedNS) + } + } + return nil +} + +func checkCleared(top topSnapshot) error { + snapshot, err := decodeSnapshot(top) + if err != nil { + return err + } + if len(snapshot.Rows) != 0 || snapshot.Observed != 0 || !snapshot.Complete { + return fmt.Errorf("native snapshot is not a complete empty clear: retained=%d observed=%d complete=%t", len(snapshot.Rows), snapshot.Observed, snapshot.Complete) + } + if len(top.Rows) != 0 || top.Count != 0 { + return fmt.Errorf("empty snapshot contains projected operation rows: rows=%d count=%d", len(top.Rows), top.Count) + } + native, err := (&pprofile.ProtoUnmarshaler{}).UnmarshalProfiles(snapshot.Native) + if err != nil { + return fmt.Errorf("read native clear sample: %w", err) + } + if native.SampleCount() != 1 { + return fmt.Errorf("schema-v2 clear requires one control sample, received %d", native.SampleCount()) + } + return nil +} diff --git a/hack/jvm-demo/checks_test.go b/hack/jvm-demo/checks_test.go new file mode 100644 index 0000000..1da25a7 --- /dev/null +++ b/hack/jvm-demo/checks_test.go @@ -0,0 +1,116 @@ +package main + +import ( + "encoding/base64" + "testing" + "time" + + "github.com/flanksource/telltale/pkg/otlp/profiles" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestDemoChecks(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "JVM demo evidence") +} + +func snapshotFixture(kind string, sequence int64, rows []profiles.Row) topSnapshot { + GinkgoHelper() + at := time.Date(2026, time.November, 15, 12, 0, 0, 0, time.UTC) + snapshot := profiles.Snapshot{Kind: kind, CaptureID: "demo-capture", Source: "demo-jvm", + ProfileID: "97fa6305-a036-4d96-8789-37c1bbe378f3", Sequence: sequence, + StartedAt: at, CompletedAt: at.Add(time.Second), Complete: true, Limit: 50, Observed: len(rows), Rows: rows} + for i := range snapshot.Rows { + snapshot.Rows[i].SampledAt = snapshot.CompletedAt + } + native, err := profiles.Encode([]profiles.Snapshot{snapshot}) + Expect(err).NotTo(HaveOccurred()) + projected := make([]map[string]any, len(rows)) + for i, row := range rows { + projected[i] = map[string]any{"operationId": row.OperationID} + } + return topSnapshot{Frame: topFrame{ID: snapshot.ProfileID, Tick: sequence, + Summary: map[string]any{"otlp": base64.StdEncoding.EncodeToString(native)}}, Rows: projected, Count: len(rows)} +} + +func httpRows(elapsed int64) []profiles.Row { + return []profiles.Row{ + {OperationID: "http-client", ElapsedNS: elapsed, Attributes: map[string]any{"direction": "outgoing", "url": "http://127.0.0.1/demo/hold", "method": "GET"}}, + {OperationID: "http-server", ElapsedNS: elapsed, Attributes: map[string]any{"direction": "incoming", "url": "http://127.0.0.1/demo/hold", "route": "/demo/{id}", "handler": "DemoController#demo", "method": "GET"}}, + } +} + +var _ = Describe("active operation evidence", func() { + It("requires both incoming and outgoing HTTP operations", func() { + Expect(checkActive("http_top", snapshotFixture("http_top", 1, httpRows(1000)))).To(Succeed()) + Expect(checkActive("http_top", snapshotFixture("http_top", 1, httpRows(1000)[:1]))).To(MatchError(ContainSubstring("incoming"))) + }) + + It("requires JDBC and R2DBC operations for the demo query", func() { + rows := []profiles.Row{ + {OperationID: "jdbc-query", ElapsedNS: 1000, Attributes: map[string]any{"api": "jdbc", "sql": "SELECT DEMO_HOLD('jdbc')"}}, + {OperationID: "r2dbc-query", ElapsedNS: 1000, Attributes: map[string]any{"api": "r2dbc", "sql": "SELECT DEMO_HOLD('r2dbc')"}}, + } + Expect(checkActive("sql_top", snapshotFixture("sql_top", 1, rows))).To(Succeed()) + Expect(checkActive("sql_top", snapshotFixture("sql_top", 1, rows[:1]))).To(MatchError(ContainSubstring("r2dbc"))) + }) + + It("rejects changing operation identities and non-increasing elapsed values", func() { + previous := snapshotFixture("http_top", 1, httpRows(1000)) + Expect(checkUpdated(previous, snapshotFixture("http_top", 2, httpRows(2000)))).To(Succeed()) + Expect(checkUpdated(previous, snapshotFixture("http_top", 1, httpRows(2000)))).To(MatchError(ContainSubstring("sequence"))) + Expect(checkUpdated(previous, snapshotFixture("http_top", 2, httpRows(1000)))).To(MatchError(ContainSubstring("elapsed"))) + changed := httpRows(2000) + changed[0].OperationID = "replacement-client" + Expect(checkUpdated(previous, snapshotFixture("http_top", 2, changed))).To(MatchError(ContainSubstring("http-client"))) + }) + + It("requires a native empty control snapshot and zero projected rows", func() { + empty := snapshotFixture("http_top", 3, nil) + Expect(checkCleared(empty)).To(Succeed()) + Expect(checkCleared(snapshotFixture("http_top", 3, httpRows(3000)))).To(MatchError(ContainSubstring("empty"))) + empty.Rows = []map[string]any{{"operationId": "phantom"}} + Expect(checkCleared(empty)).To(MatchError(ContainSubstring("projected"))) + }) + + It("rejects missing native Profiles evidence", func() { + _, err := decodeSnapshot(topSnapshot{}) + Expect(err).To(MatchError(ContainSubstring("native"))) + }) +}) + +func completedFixture() completedEvidence { + return completedEvidence{ + HTTP: []httpRecord{ + {Direction: "incoming", URL: "http://127.0.0.1/demo/hold", Route: "/demo/{id}", Handler: "DemoController#demo", Status: 200}, + {Direction: "outgoing", URL: "http://127.0.0.1/demo/hold", Status: 200}, + }, + SQL: []sqlRecord{{API: "jdbc", SQL: "SELECT DEMO_HOLD('jdbc')"}, {API: "r2dbc", SQL: "SELECT DEMO_HOLD('r2dbc')"}}, + JMS: []jmsRecord{ + {Direction: "outgoing", Destination: "demo.orders", MessageID: "message-7", Outcome: "success"}, + {Direction: "incoming", Destination: "demo.orders", MessageID: "message-7", Framework: "spring-jms", Outcome: "success"}, + }, + Mirrored: []spanRecord{{TraceID: "trace-7", SpanID: "span-7", Operation: "jvm-demo-cycle", DurationMs: 1000}}, + Received: []spanRecord{{TraceID: "trace-7", SpanID: "span-7", Operation: "jvm-demo-cycle", DurationMs: 1000}}, + } +} + +var _ = Describe("completed protocol evidence", func() { + It("accepts successful Spring protocol observations with preserved exporter delivery", func() { + Expect(checkCompleted(completedFixture())).To(Succeed()) + }) + + DescribeTable("rejects missing or changed protocol evidence", func(change func(*completedEvidence), message string) { + evidence := completedFixture() + change(&evidence) + Expect(checkCompleted(evidence)).To(MatchError(ContainSubstring(message))) + }, + Entry("missing outgoing HTTP", func(e *completedEvidence) { e.HTTP = e.HTTP[:1] }, "outgoing"), + Entry("unmatched Spring route", func(e *completedEvidence) { e.HTTP[0].Route = "" }, "route"), + Entry("missing reactive SQL", func(e *completedEvidence) { e.SQL = e.SQL[:1] }, "r2dbc"), + Entry("unmatched JMS message", func(e *completedEvidence) { e.JMS[1].MessageID = "different-message" }, "message"), + Entry("lost exporter span", func(e *completedEvidence) { e.Received = nil }, "exporter"), + Entry("changed exporter span", func(e *completedEvidence) { e.Received[0].DurationMs++ }, "changed"), + ) +}) diff --git a/hack/jvm-demo/client.go b/hack/jvm-demo/client.go new file mode 100644 index 0000000..cdd43bf --- /dev/null +++ b/hack/jvm-demo/client.go @@ -0,0 +1,121 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +type apiClient struct { + opts options + http *http.Client +} + +type captureRequest struct{ kind, run string } + +type jsonRequest struct { + path string + output any +} + +type recordsRequest struct { + capture captureRequest + output any +} + +func newClient(opts options) (apiClient, error) { + if opts.artifacts == "" || opts.agent == "" { + return apiClient{}, fmt.Errorf("artifacts directory and attached agent ID are required") + } + for _, endpoint := range []string{opts.server, opts.workload} { + parsed, err := url.Parse(endpoint) + if err != nil || parsed.Scheme != "http" || parsed.Host == "" || parsed.User != nil { + return apiClient{}, fmt.Errorf("invalid loopback HTTP endpoint %q", endpoint) + } + if parsed.Hostname() != "127.0.0.1" && parsed.Hostname() != "localhost" && parsed.Hostname() != "::1" { + return apiClient{}, fmt.Errorf("demo endpoint must be loopback: %s", endpoint) + } + } + switch opts.phase { + case "active", "updated", "cleared": + if opts.httpTop == "" || opts.sqlTop == "" { + return apiClient{}, fmt.Errorf("capture run IDs for HTTP and SQL Top are required") + } + case "completed": + if opts.http == "" || opts.sql == "" || opts.jms == "" || opts.spans == "" { + return apiClient{}, fmt.Errorf("capture run IDs for HTTP, SQL, JMS and mirrored spans are required") + } + default: + return apiClient{}, fmt.Errorf("unknown demo phase %q", opts.phase) + } + return apiClient{opts: opts, http: &http.Client{Timeout: 5 * time.Second}}, nil +} + +func recordsPath(capture captureRequest) string { + return "/api/traces/" + url.PathEscape(capture.run) + "/traces/" + url.PathEscape(capture.kind) + "/records" +} + +func (c apiClient) get(ctx context.Context, input jsonRequest) (err error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(c.opts.server, "/")+input.path, nil) + if err != nil { + return err + } + response, err := c.http.Do(request) + if err != nil { + return fmt.Errorf("GET %s: %w", request.URL, err) + } + defer func() { err = errors.Join(err, response.Body.Close()) }() + if response.StatusCode != http.StatusOK { + body, err := io.ReadAll(io.LimitReader(response.Body, 4096)) + if err != nil { + return fmt.Errorf("read HTTP %d response: %w", response.StatusCode, err) + } + return fmt.Errorf("GET %s: HTTP %d: %s", request.URL, response.StatusCode, body) + } + if err := json.NewDecoder(io.LimitReader(response.Body, 32<<20)).Decode(input.output); err != nil { + return fmt.Errorf("decode GET %s: %w", request.URL, err) + } + return nil +} + +func (c apiClient) records(ctx context.Context, input recordsRequest) error { + var page struct { + Rows json.RawMessage `json:"rows"` + NextCursor string `json:"nextCursor"` + } + if err := c.get(ctx, jsonRequest{path: recordsPath(input.capture) + "?all=true", output: &page}); err != nil { + return err + } + if page.NextCursor != "" { + return fmt.Errorf("demo records unexpectedly paginated despite all=true") + } + if err := json.Unmarshal(page.Rows, input.output); err != nil { + return fmt.Errorf("decode %s records: %w", input.capture.kind, err) + } + return nil +} + +func (c apiClient) workloadState(ctx context.Context) error { + var state struct { + Ready bool `json:"ready"` + Failure *string `json:"failure"` + } + workload := c + workload.opts.server = c.opts.workload + if err := workload.get(ctx, jsonRequest{path: "/control/state", output: &state}); err != nil { + return err + } + if state.Failure != nil { + return fmt.Errorf("demo workload failed: %s", *state.Failure) + } + if !state.Ready { + return fmt.Errorf("demo workload is not ready") + } + return nil +} diff --git a/hack/jvm-demo/completed.go b/hack/jvm-demo/completed.go new file mode 100644 index 0000000..624e3ef --- /dev/null +++ b/hack/jvm-demo/completed.go @@ -0,0 +1,159 @@ +package main + +import ( + "fmt" + "strings" + "time" +) + +type httpRecord struct { + Direction string `json:"direction"` + RequestID string `json:"requestId"` + URL string `json:"url"` + Route string `json:"route"` + Handler string `json:"handler"` + Status int `json:"status"` + Error string `json:"error"` +} + +type sqlRecord struct { + API string `json:"api"` + SQL string `json:"sql"` + Error string `json:"error"` +} + +type jmsRecord struct { + Direction string `json:"direction"` + Destination string `json:"destination"` + MessageID string `json:"messageId"` + Framework string `json:"framework"` + Outcome string `json:"outcome"` + Error string `json:"error"` +} + +type spanRecord struct { + TraceID string `json:"traceId"` + SpanID string `json:"spanId"` + ParentID string `json:"parentId"` + Operation string `json:"operation"` + StartedAt time.Time `json:"startedAt"` + DurationMs float64 `json:"durationMs"` + Status string `json:"status"` +} + +type completedEvidence struct { + HTTP []httpRecord `json:"http"` + SQL []sqlRecord `json:"sql"` + JMS []jmsRecord `json:"jms"` + Mirrored []spanRecord `json:"mirrored"` + Received []spanRecord `json:"received"` +} + +func checkCompleted(evidence completedEvidence) error { + if err := checkHTTP(evidence.HTTP); err != nil { + return err + } + if err := checkSQL(evidence.SQL); err != nil { + return err + } + if err := checkJMS(evidence.JMS); err != nil { + return err + } + return checkMirror(evidence) +} + +func checkHTTP(rows []httpRecord) error { + directions, matchedRoute := map[string]bool{}, false + for _, row := range rows { + if !strings.Contains(row.URL, "/demo/") { + continue + } + if row.Error != "" || row.Status != 200 { + return fmt.Errorf("demo HTTP %s failed: status=%d error=%s", row.URL, row.Status, row.Error) + } + directions[row.Direction] = true + if row.Direction == "incoming" && strings.Contains(row.Route, "/demo/{") && row.Handler != "" { + matchedRoute = true + } + } + for _, direction := range []string{"incoming", "outgoing"} { + if !directions[direction] { + return fmt.Errorf("no completed %s demo HTTP operation", direction) + } + } + if !matchedRoute { + return fmt.Errorf("incoming HTTP has no matched Spring route and handler") + } + return nil +} + +func checkSQL(rows []sqlRecord) error { + apis := map[string]bool{} + for _, row := range rows { + if !strings.Contains(strings.ToUpper(row.SQL), "DEMO_HOLD") { + continue + } + if row.Error != "" { + return fmt.Errorf("demo %s SQL failed: %s", row.API, row.Error) + } + apis[row.API] = true + } + for _, api := range []string{"jdbc", "r2dbc"} { + if !apis[api] { + return fmt.Errorf("no completed %s demo SQL operation", api) + } + } + return nil +} + +func checkJMS(rows []jmsRecord) error { + produced, consumed := map[string]bool{}, map[string]bool{} + for _, row := range rows { + if row.Destination != "demo.orders" { + continue + } + if row.Error != "" || row.Outcome == "error" { + return fmt.Errorf("demo JMS %s failed: %s", row.Direction, row.Error) + } + if row.Outcome != "success" || row.MessageID == "" { + continue + } + if row.Direction == "outgoing" { + produced[row.MessageID] = true + } else if row.Direction == "incoming" && row.Framework == "spring-jms" { + consumed[row.MessageID] = true + } + } + for id := range produced { + if consumed[id] { + return nil + } + } + return fmt.Errorf("no demo JMS message links outgoing production to incoming Spring processing") +} + +func checkMirror(evidence completedEvidence) error { + received := make(map[string]spanRecord, len(evidence.Received)) + for _, span := range evidence.Received { + received[span.TraceID+"/"+span.SpanID] = span + } + cycle := false + for _, span := range evidence.Mirrored { + key := span.TraceID + "/" + span.SpanID + if span.TraceID == "" || span.SpanID == "" { + return fmt.Errorf("mirrored span has no trace/span identity") + } + original, found := received[key] + if !found { + return fmt.Errorf("original exporter has not delivered mirrored span %s (%s)", key, span.Operation) + } + if original != span { + return fmt.Errorf("original exporter span %s changed during mirroring", key) + } + cycle = cycle || span.Operation == "jvm-demo-cycle" + } + if !cycle { + return fmt.Errorf("no mirrored jvm-demo-cycle span from the existing exporter") + } + return nil +} diff --git a/hack/jvm-demo/java/DemoCycle.java b/hack/jvm-demo/java/DemoCycle.java new file mode 100644 index 0000000..955a927 --- /dev/null +++ b/hack/jvm-demo/java/DemoCycle.java @@ -0,0 +1,83 @@ +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import io.opentelemetry.api.trace.Span; + +final class DemoCycle { + private final CountDownLatch release = new CountDownLatch(1); + private final CountDownLatch entered = new CountDownLatch(5); + private final CountDownLatch messageFinished = new CountDownLatch(1); + private final AtomicInteger httpEntered = new AtomicInteger(); + private final AtomicBoolean jdbcEntered = new AtomicBoolean(); + private final AtomicBoolean r2dbcEntered = new AtomicBoolean(); + private final AtomicBoolean jmsEntered = new AtomicBoolean(); + final String id; + final Span span; + CompletableFuture work; + + DemoCycle(String id, boolean held, Span span) { + this.id = id; + this.span = span; + if (!held) release.countDown(); + } + + void http() throws InterruptedException { + httpEntered.incrementAndGet(); + entered.countDown(); + awaitRelease(); + } + + int sql(String api) throws InterruptedException { + AtomicBoolean marker; + if ("jdbc".equals(api)) marker = jdbcEntered; + else if ("r2dbc".equals(api)) marker = r2dbcEntered; + else throw new IllegalArgumentException("Unknown demo SQL API: " + api); + if (!marker.compareAndSet(false, true)) throw new IllegalStateException("SQL API entered twice: " + api); + entered.countDown(); + awaitRelease(); + return 7; + } + + void message() throws InterruptedException { + if (!jmsEntered.compareAndSet(false, true)) throw new IllegalStateException("Demo message entered twice"); + entered.countDown(); + awaitRelease(); + messageFinished.countDown(); + } + + void awaitEntered() throws InterruptedException { + if (!entered.await(20, TimeUnit.SECONDS)) throw new IllegalStateException("Demo operations did not enter: " + state()); + } + + void release() { + release.countDown(); + } + + void awaitFinished() throws Exception { + work.get(20, TimeUnit.SECONDS); + if (!messageFinished.await(20, TimeUnit.SECONDS)) throw new IllegalStateException("Demo JMS processing did not complete"); + span.end(); + } + + Map state() { + Map values = new LinkedHashMap<>(); + values.put("cycle", id); + values.put("holding", release.getCount() != 0); + values.put("httpEntered", httpEntered.get()); + values.put("jdbcEntered", jdbcEntered.get()); + values.put("r2dbcEntered", r2dbcEntered.get()); + values.put("jmsEntered", jmsEntered.get()); + return values; + } + + private void awaitRelease() throws InterruptedException { + if (!release.await(Duration.ofMinutes(2).toMillis(), TimeUnit.MILLISECONDS)) { + throw new IllegalStateException("Demo cycle was not released before its deadline: " + id); + } + } +} diff --git a/hack/jvm-demo/java/DemoOperations.java b/hack/jvm-demo/java/DemoOperations.java new file mode 100644 index 0000000..b49e973 --- /dev/null +++ b/hack/jvm-demo/java/DemoOperations.java @@ -0,0 +1,259 @@ +import io.netty.channel.ChannelOption; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.r2dbc.spi.ConnectionFactories; +import io.r2dbc.spi.ConnectionFactory; +import java.nio.file.Path; +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.jms.Message; +import org.apache.activemq.artemis.api.core.TransportConfiguration; +import org.apache.activemq.artemis.core.config.impl.ConfigurationImpl; +import org.apache.activemq.artemis.core.remoting.impl.invm.InVMAcceptorFactory; +import org.apache.activemq.artemis.core.server.embedded.EmbeddedActiveMQ; +import org.apache.activemq.artemis.jms.client.ActiveMQConnectionFactory; +import org.h2.jdbcx.JdbcDataSource; +import org.springframework.http.client.SimpleClientHttpRequestFactory; +import org.springframework.http.client.reactive.ReactorClientHttpConnector; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jms.core.JmsTemplate; +import org.springframework.jms.listener.DefaultMessageListenerContainer; +import org.springframework.jms.listener.SessionAwareMessageListener; +import org.springframework.r2dbc.core.DatabaseClient; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.reactive.function.client.WebClient; +import reactor.netty.http.client.HttpClient; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +final class DemoOperations implements AutoCloseable { + private static final Duration TIMEOUT = Duration.ofSeconds(30); + private final AtomicReference failure = new AtomicReference<>(); + private final AtomicBoolean closed = new AtomicBoolean(); + private final AtomicLong sequences = new AtomicLong(); + private final AtomicLong completed = new AtomicLong(); + private final ExecutorService workers = Executors.newCachedThreadPool(); + private final ScheduledExecutorService traffic = Executors.newSingleThreadScheduledExecutor(); + private final EmbeddedActiveMQ broker = new EmbeddedActiveMQ(); + private final ActiveMQConnectionFactory factory = new ActiveMQConnectionFactory("vm://0"); + private final DefaultMessageListenerContainer listener = new DefaultMessageListenerContainer(); + private final JmsTemplate jms; + private final JdbcTemplate jdbc; + private final DatabaseClient r2dbc; + private final RestTemplate rest; + private final WebClient web; + private volatile DemoCycle active; + private volatile DemoCycle last; + private volatile boolean ready; + private volatile boolean trafficEnabled; + private String baseUrl; + + DemoOperations(Path directory) throws Exception { + try { + JdbcDataSource dataSource = new JdbcDataSource(); + dataSource.setURL("jdbc:h2:mem:demo_jdbc;DB_CLOSE_DELAY=-1"); + jdbc = new JdbcTemplate(dataSource); + jdbc.execute("CREATE ALIAS DEMO_HOLD FOR 'Workload.hold'"); + ConnectionFactory connections = ConnectionFactories.get("r2dbc:h2:mem:///demo_r2dbc;DB_CLOSE_DELAY=-1"); + Mono.usingWhen(Mono.from(connections.create()), + connection -> Flux.from(connection.createStatement("CREATE ALIAS DEMO_HOLD FOR 'Workload.hold'").execute()) + .flatMap(result -> result.getRowsUpdated()).then(), + connection -> Mono.from(connection.close())).block(TIMEOUT); + r2dbc = DatabaseClient.create(connections); + SimpleClientHttpRequestFactory requests = new SimpleClientHttpRequestFactory(); + requests.setConnectTimeout(5000); + requests.setReadTimeout(120000); + rest = new RestTemplate(requests); + HttpClient client = HttpClient.create().option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 5000).responseTimeout(Duration.ofMinutes(2)); + web = WebClient.builder().clientConnector(new ReactorClientHttpConnector(client)).build(); + broker.setConfiguration(new ConfigurationImpl().setPersistenceEnabled(false).setSecurityEnabled(false) + .setBindingsDirectory(directory.resolve("artemis/bindings").toString()) + .setJournalDirectory(directory.resolve("artemis/journal").toString()) + .setPagingDirectory(directory.resolve("artemis/paging").toString()) + .setLargeMessagesDirectory(directory.resolve("artemis/large-messages").toString()) + .addAcceptorConfiguration(new TransportConfiguration(InVMAcceptorFactory.class.getName()))); + broker.start(); + jms = new JmsTemplate(factory); + listener.setConnectionFactory(factory); + listener.setDestinationName("demo.orders"); + listener.setSessionTransacted(true); + listener.setErrorHandler(this::recordFailure); + listener.setMessageListener((SessionAwareMessageListener) (message, session) -> { + DemoCycle cycle = requireCycle(message.getStringProperty("cycle")); + try { cycle.message(); } + catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Demo JMS processing interrupted", interrupted); + } + }); + listener.afterPropertiesSet(); + listener.start(); + } catch (Exception | Error error) { + try { close(); } catch (Exception cleanup) { error.addSuppressed(cleanup); } + throw error; + } + } + + void start(String url) throws Exception { + baseUrl = url; + DemoCycle warm = startCycle(false); + warm.awaitEntered(); + finish(warm); + ready = true; + traffic.scheduleWithFixedDelay(this::runTraffic, 1, 2, TimeUnit.SECONDS); + } + + synchronized Map begin() throws Exception { + checkHealthy(); + if (trafficEnabled) throw new IllegalStateException("Stop interactive traffic before holding a deterministic cycle"); + DemoCycle cycle = startCycle(true); + cycle.awaitEntered(); + return state(); + } + + synchronized Map release() throws Exception { + checkHealthy(); + if (active == null) throw new IllegalStateException("There is no active demo cycle to release"); + finish(active); + return state(); + } + + Map traffic(boolean enabled) throws Exception { + synchronized (this) { + checkHealthy(); + if (enabled && active != null) throw new IllegalStateException("Release the held demo cycle before starting interactive traffic"); + trafficEnabled = enabled; + } + if (!enabled) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (active != null) { + checkHealthy(); + if (System.nanoTime() > deadline) throw new IllegalStateException("Interactive demo traffic did not stop"); + Thread.sleep(10); + } + } + return state(); + } + + void holdHttp(String id) throws InterruptedException { + int separator = id.indexOf('-'); + if (separator < 0) throw new IllegalArgumentException("Demo request requires a client and cycle ID: " + id); + requireCycle(id.substring(separator + 1)).http(); + } + + int holdSql(String api) throws InterruptedException { + DemoCycle cycle = active; + if (cycle == null) throw new IllegalStateException("SQL started outside a demo cycle"); + return cycle.sql(api); + } + + boolean healthy() { + return ready && failure.get() == null; + } + + Map state() { + DemoCycle cycle = active; + if (cycle == null) cycle = last; + Map values = cycle == null ? new LinkedHashMap<>() : cycle.state(); + values.put("ready", ready); + values.put("trafficEnabled", trafficEnabled); + values.put("completedCycles", completed.get()); + values.put("failure", failure.get() == null ? null : failure.get().toString()); + return values; + } + + private DemoCycle startCycle(boolean held) { + if (active != null) throw new IllegalStateException("Demo cycle already active: " + active.id); + DemoCycle cycle = new DemoCycle(Long.toString(sequences.incrementAndGet()), held, + GlobalOpenTelemetry.getTracer("jvm-demo").spanBuilder("jvm-demo-cycle").startSpan()); + active = cycle; + try (Scope ignored = cycle.span.makeCurrent()) { + cycle.work = CompletableFuture.allOf( + task(() -> expect(rest.getForObject(baseUrl + "/demo/rest-" + cycle.id, String.class), "demo-rest-" + cycle.id)), + task(() -> expect(web.get().uri(baseUrl + "/demo/web-" + cycle.id).retrieve().bodyToMono(String.class).block(Duration.ofMinutes(2)), "demo-web-" + cycle.id)), + task(() -> expect(jdbc.queryForObject("SELECT DEMO_HOLD('jdbc') AS demo_value", Integer.class), 7)), + task(() -> expect(r2dbc.sql("SELECT DEMO_HOLD('r2dbc') AS demo_value").map((row, metadata) -> row.get("DEMO_VALUE", Integer.class)).one().block(Duration.ofMinutes(2)), 7)), + task(() -> jms.send("demo.orders", session -> { + javax.jms.TextMessage message = session.createTextMessage("demo-order-" + cycle.id); + message.setStringProperty("cycle", cycle.id); + return message; + }))); + } + return cycle; + } + + private CompletableFuture task(Runnable operation) { + return CompletableFuture.runAsync(Context.current().wrap(() -> { + try { operation.run(); } + catch (RuntimeException error) { recordFailure(error); throw error; } + }), workers); + } + + private void finish(DemoCycle cycle) throws Exception { + cycle.release(); + cycle.awaitFinished(); + completed.incrementAndGet(); + last = cycle; + active = null; + } + + private DemoCycle requireCycle(String id) { + DemoCycle cycle = active; + if (cycle == null || !cycle.id.equals(id)) throw new IllegalStateException("Unknown active demo cycle: " + id); + return cycle; + } + + private void runTraffic() { + try { + DemoCycle cycle; + synchronized (this) { + if (!trafficEnabled) return; + checkHealthy(); + cycle = startCycle(true); + } + cycle.awaitEntered(); + Thread.sleep(3000); + synchronized (this) { finish(cycle); } + } catch (Exception error) { + recordFailure(error); + throw new CompletionException("Interactive demo traffic failed", error); + } + } + + private void checkHealthy() { + if (!healthy()) throw new IllegalStateException("Demo workload is not healthy", failure.get()); + } + + private void recordFailure(Throwable error) { + failure.compareAndSet(null, error); + DemoCycle cycle = active; + if (cycle != null) cycle.release(); + } + + private static void expect(Object actual, Object expected) { + if (!expected.equals(actual)) throw new IllegalStateException("Expected " + expected + ", received " + actual); + } + + @Override public void close() throws Exception { + if (!closed.compareAndSet(false, true)) return; + ready = false; + trafficEnabled = false; + DemoCycle cycle = active; + if (cycle != null) cycle.release(); + traffic.shutdownNow(); + workers.shutdownNow(); + DemoResources.close(listener::stop, listener::destroy, factory::close, broker::stop, + () -> reactor.netty.http.HttpResources.disposeLoopsAndConnectionsLater().block(TIMEOUT)); + } +} diff --git a/hack/jvm-demo/java/DemoResources.java b/hack/jvm-demo/java/DemoResources.java new file mode 100644 index 0000000..e151fa8 --- /dev/null +++ b/hack/jvm-demo/java/DemoResources.java @@ -0,0 +1,15 @@ +final class DemoResources { + private DemoResources() {} + + static void close(AutoCloseable... resources) throws Exception { + Exception failure = null; + for (AutoCloseable resource : resources) { + try { resource.close(); } + catch (Exception error) { + if (failure == null) failure = error; + else failure.addSuppressed(error); + } + } + if (failure != null) throw failure; + } +} diff --git a/hack/jvm-demo/java/DemoServer.java b/hack/jvm-demo/java/DemoServer.java new file mode 100644 index 0000000..c7eb7f6 --- /dev/null +++ b/hack/jvm-demo/java/DemoServer.java @@ -0,0 +1,98 @@ +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.catalina.Context; +import org.apache.catalina.Wrapper; +import org.apache.catalina.startup.Tomcat; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.context.support.AnnotationConfigWebApplicationContext; +import org.springframework.web.server.ResponseStatusException; +import org.springframework.web.servlet.DispatcherServlet; +import org.springframework.web.servlet.config.annotation.EnableWebMvc; + +final class DemoServer implements AutoCloseable { + private final Tomcat tomcat = new Tomcat(); + private final AnnotationConfigWebApplicationContext spring = new AnnotationConfigWebApplicationContext(); + private final AtomicBoolean closed = new AtomicBoolean(); + + DemoServer(int port, Path directory, DemoOperations operations) throws Exception { + try { + Path root = Files.createDirectories(directory.resolve("tomcat")).toAbsolutePath(); + tomcat.setBaseDir(root.toString()); + tomcat.setPort(port); + tomcat.getConnector().setProperty("address", "127.0.0.1"); + Context context = tomcat.addContext("", root.toString()); + context.setParentClassLoader(Workload.class.getClassLoader()); + spring.setServletContext(context.getServletContext()); + spring.register(MvcConfiguration.class); + spring.addBeanFactoryPostProcessor(factory -> factory.registerSingleton("demoController", new Controller(operations))); + spring.refresh(); + Wrapper servlet = Tomcat.addServlet(context, "spring", new DispatcherServlet(spring)); + servlet.setAsyncSupported(true); + servlet.setLoadOnStartup(1); + context.addServletMappingDecoded("/", "spring"); + tomcat.start(); + } catch (Exception | Error error) { + try { close(); } catch (Exception cleanup) { error.addSuppressed(cleanup); } + throw error; + } + } + + String baseUrl() { + return "http://127.0.0.1:" + tomcat.getConnector().getLocalPort(); + } + + @Override public void close() throws Exception { + if (!closed.compareAndSet(false, true)) return; + DemoResources.close(tomcat::stop, tomcat::destroy, spring::close); + } + + @Configuration + @EnableWebMvc + public static class MvcConfiguration {} + + @RestController + public static final class Controller { + private final DemoOperations operations; + + Controller(DemoOperations operations) { + this.operations = operations; + } + + @GetMapping("/health") public Map health() { + if (!operations.healthy()) throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "Demo is not ready: " + operations.state()); + return operations.state(); + } + + @GetMapping("/demo/{id}") public ResponseEntity demo(@PathVariable("id") String id) throws Exception { + operations.holdHttp(id); + return ResponseEntity.ok().contentType(MediaType.TEXT_PLAIN) + .header("X-Content-Type-Options", "nosniff").body("demo-" + id); + } + + @GetMapping("/control/state") public Map state() { + return operations.state(); + } + + @PostMapping("/control/hold") public Map hold() throws Exception { + return operations.begin(); + } + + @PostMapping("/control/release") public Map release() throws Exception { + return operations.release(); + } + + @PostMapping("/control/traffic") public Map traffic(@RequestParam("enabled") boolean enabled) throws Exception { + return operations.traffic(enabled); + } + } +} diff --git a/hack/jvm-demo/java/Workload.java b/hack/jvm-demo/java/Workload.java new file mode 100644 index 0000000..f14193e --- /dev/null +++ b/hack/jvm-demo/java/Workload.java @@ -0,0 +1,53 @@ +import java.nio.file.Paths; +import java.nio.file.Path; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +public final class Workload { + private static volatile DemoOperations operations; + + private Workload() {} + + public static double compute(long value) { + return Math.sqrt(value + 1); + } + + public static int hold(String api) throws InterruptedException { + if (operations == null) throw new IllegalStateException("Demo operations are not initialized"); + return operations.holdSql(api); + } + + public static void main(String[] args) throws Exception { + if (args.length != 2) throw new IllegalArgumentException("Usage: Workload "); + int port = Integer.parseInt(args[0]); + if (port < 0 || port > 65535) throw new IllegalArgumentException("Invalid demo HTTP port: " + port); + Path artifacts = Paths.get(args[1]); + try (DemoOperations runtime = new DemoOperations(artifacts)) { + operations = runtime; + try (DemoServer server = new DemoServer(port, artifacts, runtime)) { + run(runtime, server); + } + } + } + + private static void run(DemoOperations runtime, DemoServer server) throws Exception { + ScheduledExecutorService compute = Executors.newSingleThreadScheduledExecutor(); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + compute.shutdownNow(); + try { DemoResources.close(runtime, server); } + catch (Exception failure) { throw new IllegalStateException("Could not stop JVM demo", failure); } + }, "demo-shutdown")); + try { + runtime.start(server.baseUrl()); + compute.scheduleWithFixedDelay(() -> { + for (int value = 0; value < 32; value++) compute(value); + }, 0, 10, TimeUnit.MILLISECONDS); + System.out.println("FIXTURE READY " + server.baseUrl()); + new CountDownLatch(1).await(); + } finally { + compute.shutdownNow(); + } + } +} diff --git a/hack/jvm-demo/main.go b/hack/jvm-demo/main.go new file mode 100644 index 0000000..7cc7e48 --- /dev/null +++ b/hack/jvm-demo/main.go @@ -0,0 +1,183 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "path/filepath" + "time" +) + +type options struct { + server, workload, agent, artifacts, phase string + httpTop, sqlTop, http, sql, jms, spans string +} + +type topObservation struct { + kind string + previous, current topSnapshot +} + +func main() { + var opts options + flag.StringVar(&opts.server, "server", "", "Loopback Telltale server URL") + flag.StringVar(&opts.workload, "workload", "", "Loopback workload URL") + flag.StringVar(&opts.agent, "agent", "", "Attached demo agent ID") + flag.StringVar(&opts.artifacts, "artifacts", "", "Directory for native snapshot and protocol evidence") + flag.StringVar(&opts.phase, "phase", "", "active, updated, cleared or completed") + flag.StringVar(&opts.httpTop, "http-top", "", "HTTP Top capture run ID") + flag.StringVar(&opts.sqlTop, "sql-top", "", "SQL Top capture run ID") + flag.StringVar(&opts.http, "http", "", "HTTP capture run ID") + flag.StringVar(&opts.sql, "sql", "", "SQL capture run ID") + flag.StringVar(&opts.jms, "jms", "", "JMS capture run ID") + flag.StringVar(&opts.spans, "spans", "", "Mirrored spans capture run ID") + flag.Parse() + if err := run(opts); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(opts options) error { + client, err := newClient(opts) + if err != nil { + return err + } + if err := os.MkdirAll(opts.artifacts, 0o700); err != nil { + return fmt.Errorf("create evidence directory: %w", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if err := client.workloadState(ctx); err != nil { + return err + } + if opts.phase == "completed" { + err = client.completed(ctx) + } else { + err = client.top(ctx) + } + if err != nil { + return fmt.Errorf("demo %s evidence: %w", opts.phase, err) + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"passed": true, "phase": opts.phase, "agent": opts.agent, "artifacts": opts.artifacts}) +} + +func (c apiClient) top(ctx context.Context) error { + for _, capture := range []captureRequest{{kind: "http_top", run: c.opts.httpTop}, {kind: "sql_top", run: c.opts.sqlTop}} { + observation := topObservation{kind: capture.kind} + if c.opts.phase != "active" { + previousOptions := c.opts + previousOptions.phase = "active" + if c.opts.phase == "cleared" { + previousOptions.phase = "updated" + } + if err := readEvidence(evidenceFile(previousOptions, capture.kind), &observation.previous); err != nil { + return err + } + } + err := await(ctx, func() error { + if err := c.get(ctx, jsonRequest{path: recordsPath(capture) + "?view=top&all=true", output: &observation.current}); err != nil { + return err + } + return c.checkTop(observation) + }) + if err != nil { + if observation.current.Frame.ID != "" { + path := filepath.Join(c.opts.artifacts, c.opts.phase+"-"+capture.kind+"-failed.json") + err = errors.Join(err, writeEvidence(path, observation.current)) + } + return fmt.Errorf("%s: %w", capture.kind, err) + } + if err := writeEvidence(evidenceFile(c.opts, capture.kind), observation.current); err != nil { + return err + } + } + return nil +} + +func (c apiClient) checkTop(observation topObservation) error { + if c.opts.phase == "active" { + return checkActive(observation.kind, observation.current) + } + if c.opts.phase == "cleared" { + if observation.current.Frame.Tick <= observation.previous.Frame.Tick { + return fmt.Errorf("clear snapshot sequence %d must follow held snapshot %d", observation.current.Frame.Tick, observation.previous.Frame.Tick) + } + return checkCleared(observation.current) + } + if err := checkActive(observation.kind, observation.current); err != nil { + return err + } + return checkUpdated(observation.previous, observation.current) +} + +func (c apiClient) completed(ctx context.Context) error { + var evidence completedEvidence + err := await(ctx, func() error { + requests := []struct { + capture captureRequest + output any + }{ + {captureRequest{kind: "http", run: c.opts.http}, &evidence.HTTP}, + {captureRequest{kind: "sql", run: c.opts.sql}, &evidence.SQL}, + {captureRequest{kind: "jms", run: c.opts.jms}, &evidence.JMS}, + {captureRequest{kind: "spans", run: c.opts.spans}, &evidence.Mirrored}, + {captureRequest{kind: "spans", run: "receiver"}, &evidence.Received}, + } + for _, request := range requests { + if err := c.records(ctx, recordsRequest{capture: request.capture, output: request.output}); err != nil { + return err + } + } + return checkCompleted(evidence) + }) + if err != nil { + return errors.Join(err, writeEvidence(filepath.Join(c.opts.artifacts, "completed-failed.json"), evidence)) + } + return writeEvidence(filepath.Join(c.opts.artifacts, "completed.json"), evidence) +} + +func evidenceFile(opts options, kind string) string { + return filepath.Join(opts.artifacts, opts.phase+"-"+kind+".json") +} + +func writeEvidence(path string, value any) error { + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return fmt.Errorf("encode evidence %s: %w", path, err) + } + if err := os.WriteFile(path, append(data, '\n'), 0o600); err != nil { + return fmt.Errorf("write evidence %s: %w", path, err) + } + return nil +} + +func readEvidence(path string, output any) error { + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("read previous phase evidence %s: %w", path, err) + } + if err := json.Unmarshal(data, output); err != nil { + return fmt.Errorf("decode previous phase evidence %s: %w", path, err) + } + return nil +} + +func await(ctx context.Context, check func() error) error { + tick := time.NewTicker(200 * time.Millisecond) + defer tick.Stop() + for { + err := check() + if err == nil { + return nil + } + select { + case <-ctx.Done(): + return fmt.Errorf("%w; last observation: %w", ctx.Err(), err) + case <-tick.C: + } + } +} diff --git a/hack/jvm-top-demo/main.go b/hack/jvm-top-demo/main.go new file mode 100644 index 0000000..bb0c3d0 --- /dev/null +++ b/hack/jvm-top-demo/main.go @@ -0,0 +1,95 @@ +// Run after make build: go run ./hack/jvm-top-demo +package main + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strings" + "syscall" + + "github.com/flanksource/commons/logger" +) + +func main() { + if err := run(); err != nil { + logger.Errorf("%v", err) + os.Exit(1) + } +} +func run() (err error) { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + if err := os.MkdirAll(".tmp", 0700); err != nil { + return err + } + dir, err := os.MkdirTemp(".tmp", "jvm-top-demo-") + if err != nil { + return err + } + defer func() { err = errors.Join(err, os.RemoveAll(dir)) }() + dir, err = filepath.Abs(dir) + if err != nil { + return err + } + env := append(os.Environ(), "XDG_CONFIG_HOME="+filepath.Join(dir, "config")) + tokenCommand := exec.CommandContext(ctx, ".bin/telltale", "agents", "token", "create", "demo") + tokenCommand.Env = env + token, err := tokenCommand.Output() + if err != nil { + return err + } + server := exec.CommandContext(ctx, ".bin/telltale", "serve", "--port", "0", "--data-dir", dir) + server.Env = env + server.Stderr = os.Stderr + stdout, err := server.StdoutPipe() + if err != nil { + return err + } + if err = server.Start(); err != nil { + return err + } + defer func() { cancel(); _ = server.Wait() }() + scanner := bufio.NewScanner(stdout) + address := "" + for scanner.Scan() { + if value, ok := strings.CutPrefix(scanner.Text(), "Telltale API: "); ok { + address = strings.TrimSpace(value) + break + } + } + if address == "" { + return fmt.Errorf("server did not report its address: %v", scanner.Err()) + } + javac := exec.CommandContext(ctx, "javac", "-d", dir, "pkg/agentinstall/testdata/Workload.java") + if data, err := javac.CombinedOutput(); err != nil { + return fmt.Errorf("javac: %w: %s", err, data) + } + config, err := json.Marshal(map[string]any{"url": address, "token": strings.TrimSpace(string(token)), "service": "jvm-top-demo", "allowMutation": false}) + if err != nil { + return err + } + path := filepath.Join(dir, "agent.json") + if err = os.WriteFile(path, config, 0600); err != nil { + return err + } + jar, err := filepath.Abs("pkg/agentinstall/java/assets/telltale-agent.jar") + if err != nil { + return err + } + java := exec.CommandContext(ctx, "java", "-javaagent:"+jar+"="+path, "-cp", dir, "Workload") + java.Stderr = os.Stderr + if err = java.Start(); err != nil { + return err + } + defer func() { cancel(); _ = java.Wait() }() + fmt.Printf("Demo hub: %s\nTerminal: .bin/telltale jvm top --server %s --agent jvm-top-demo\n", address, address) + <-ctx.Done() + return nil +} diff --git a/hack/process-header/README.md b/hack/process-header/README.md new file mode 100644 index 0000000..8050a58 --- /dev/null +++ b/hack/process-header/README.md @@ -0,0 +1,15 @@ +# Process header demo + +Run a local workload with changing CPU/RAM, repeated HTTP requests, and a child process that exits after two seconds: + +```sh +.bin/telltale trace run --data-dir .tmp/process-header/data --tui -- go run ./hack/process-header --duration 30s +``` + +The process rows update above the event stream. Press `e` to show exited processes, Alt+↑/↓ to scroll process rows, and Space to pause events while process measurements continue updating. The command prints its saved run ID; open it afterward with: + +```sh +.bin/telltale trace replay --data-dir .tmp/process-header/data --ui +``` + +The web viewer uses the same process measurements and Go pretty printers. Show exited reveals the child’s last sample. All network requests remain on localhost, and no credentials or external services are needed. diff --git a/hack/process-header/main.go b/hack/process-header/main.go new file mode 100644 index 0000000..931226c --- /dev/null +++ b/hack/process-header/main.go @@ -0,0 +1,105 @@ +// Run through telltale trace run to observe changing CPU/RAM, HTTP events, and a child exiting. +package main + +import ( + "crypto/sha256" + "errors" + "flag" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "os/exec" + "runtime" + "time" +) + +func main() { + duration := flag.Duration("duration", 30*time.Second, "Duration of the local workload") + child := flag.Bool("child", false, "Run the short-lived child workload") + flag.Parse() + if *child { + time.Sleep(2 * time.Second) + return + } + if err := run(*duration); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(duration time.Duration) (err error) { + if duration <= 0 { + return fmt.Errorf("duration must be positive") + } + proxy, err := url.Parse(os.Getenv("HTTP_PROXY")) + if err != nil || proxy.Scheme == "" || proxy.Host == "" { + return fmt.Errorf("run this workload through telltale trace run with HTTP proxy capture") + } + if proxy.Hostname() != "127.0.0.1" && proxy.Hostname() != "localhost" && proxy.Hostname() != "::1" { + return fmt.Errorf("the demo requires Telltale's localhost HTTP proxy") + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("recorded local workload")) })) + defer server.Close() + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = http.ProxyURL(proxy) + defer transport.CloseIdleConnections() + client := &http.Client{Transport: transport, Timeout: 5 * time.Second} + executable, err := os.Executable() + if err != nil { + return err + } + child := exec.Command(executable, "--child") + if err := child.Start(); err != nil { + return err + } + defer func() { + if child != nil { + err = errors.Join(err, child.Wait()) + } + }() + started := time.Now() + var held [][]byte + for time.Since(started) < duration { + if child != nil && time.Since(started) > 3*time.Second { + if err := child.Wait(); err != nil { + return err + } + child = nil + } + block := make([]byte, 2<<20) + for i := range block { + block[i] = byte(i) + } + held = append(held, block) + busyUntil := time.Now().Add(80 * time.Millisecond) + for time.Now().Before(busyUntil) { + sum := sha256.Sum256(block) + runtime.KeepAlive(sum) + } + response, err := client.Get(server.URL + "/orders") + if err != nil { + return err + } + _, readErr := io.Copy(io.Discard, response.Body) + closeErr := response.Body.Close() + if readErr != nil { + return readErr + } + if closeErr != nil { + return closeErr + } + if response.StatusCode != http.StatusOK { + return fmt.Errorf("local request: status %d", response.StatusCode) + } + if len(held) >= 32 { + held = nil + runtime.GC() + } + time.Sleep(400 * time.Millisecond) + } + runtime.KeepAlive(held) + return nil +} diff --git a/hack/signing-connection/main.go b/hack/signing-connection/main.go new file mode 100644 index 0000000..c5fc816 --- /dev/null +++ b/hack/signing-connection/main.go @@ -0,0 +1,34 @@ +// Command signing-connection writes a connection whose signingKey secret is a +// new RSA key, for a Deployer's spec.auth.issuer. The key never leaves the +// file: agents get only its public JWKS. `telltale jvm deploy` creates the +// default issuer (connection://telltale/issuer) the same way on first use. +// +// go run ./hack/signing-connection +package main + +import ( + "fmt" + "os" + + "github.com/flanksource/telltale/pkg/deploy" +) + +func main() { + if err := run(os.Args[1:]); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(args []string) error { + if len(args) != 2 { + return fmt.Errorf("usage: signing-connection ") + } + name, dir := args[0], args[1] + path, err := deploy.WriteIssuer(dir, "", name) + if err != nil { + return err + } + fmt.Printf("wrote %s: issuer connection://%s\n", path, name) + return nil +} diff --git a/hack/top-fixture/main.go b/hack/top-fixture/main.go new file mode 100644 index 0000000..a24d69b --- /dev/null +++ b/hack/top-fixture/main.go @@ -0,0 +1,84 @@ +// A local PostgreSQL lock wait recording for manual Top UI and TUI checks. +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "time" + + "github.com/flanksource/commons-db/connection/crd" + "github.com/flanksource/commons-db/dbtest" + "github.com/flanksource/commons-db/types" + "github.com/flanksource/telltale" +) + +func main() { + if err := record(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func record() (err error) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + db, closeDB, err := dbtest.Open(dbtest.Options{Name: "sample_top_demo", DataDir: ".tmp/top-ux-review/postgres"}) + if err != nil { + return err + } + defer func() { err = errors.Join(err, closeDB()) }() + pool := db.SQL() + if _, err := pool.ExecContext(ctx, "CREATE TABLE orders (id integer PRIMARY KEY, paid boolean); INSERT INTO orders VALUES (1, false)"); err != nil { + return err + } + holder, err := pool.Conn(ctx) + if err != nil { + return err + } + defer func() { err = errors.Join(err, holder.Close()) }() + if _, err := holder.ExecContext(ctx, "BEGIN; UPDATE orders SET paid = true WHERE id = 1"); err != nil { + return err + } + defer func() { + _, rollback := holder.ExecContext(context.Background(), "ROLLBACK") + err = errors.Join(err, rollback) + }() + waiter, err := pool.Conn(ctx) + if err != nil { + return err + } + defer func() { err = errors.Join(err, waiter.Close()) }() + waitCtx, stopWait := context.WithCancel(ctx) + defer stopWait() + waited := make(chan error, 1) + go func() { + _, err := waiter.ExecContext(waitCtx, "UPDATE orders SET paid = true WHERE id = 1") + waited <- err + }() + defer func() { + select { + case waitErr := <-waited: + err = errors.Join(err, fmt.Errorf("waiter completed before cancellation: %v", waitErr)) + default: + stopWait() + if waitErr := <-waited; waitErr == nil { + err = errors.Join(err, errors.New("waiter did not report cancellation")) + } + } + }() + connection := crd.Connection{} + connection.Name = "top-demo" + connection.Spec = crd.ConnectionSpec{Type: "postgres", URL: types.EnvVar{ValueStatic: db.DSN()}} + run, err := telltale.Start(ctx, telltale.StartOptions{DataDir: ".tmp/top-ux-review/data", Duration: 3 * time.Second, RunConnections: []crd.Connection{connection}, Traces: []telltale.TraceSpec{{Kind: "sql_top", Params: json.RawMessage(`{"connection":"connection://top-demo","poll":"1s","top":50}`)}}}) + if err != nil { + return err + } + result, err := run.Wait() + if err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"runId": run.ID(), "dir": run.Dir(), "captures": result.Captures}) +} diff --git a/internal/cli/agent_capture_test.go b/internal/cli/agent_capture_test.go index afd77a5..49bbdeb 100644 --- a/internal/cli/agent_capture_test.go +++ b/internal/cli/agent_capture_test.go @@ -106,6 +106,8 @@ var _ = Describe("CLI agent captures", func() { close(stop) <-done Expect(runErr).NotTo(HaveOccurred(), stderr) + Expect(stderr).To(ContainSubstring(expected), "live records must reach stderr without --follow") + Expect(stderr).NotTo(ContainSubstring(secret)) result := manifest(dataDir) Expect(result.Captures).To(HaveLen(1)) Expect(result.Captures[0].Kind).To(Equal(kind)) diff --git a/internal/cli/agent_target.go b/internal/cli/agent_target.go new file mode 100644 index 0000000..07ced74 --- /dev/null +++ b/internal/cli/agent_target.go @@ -0,0 +1,48 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/traces" + "github.com/flanksource/telltale/pkg/tracing/target" +) + +// withAgentTarget is a capture's params with target.agent set to the id of +// the agent it runs on, when it runs on one: --agent may be a short name, or +// absent when only one agent offers the kind. A selector or a pid is left to +// the run. +func withAgentTarget(ctx context.Context, registration traces.Registration, raw json.RawMessage) (json.RawMessage, error) { + params := map[string]any{} + if len(raw) > 0 { + if err := json.Unmarshal(raw, ¶ms); err != nil { + return nil, fmt.Errorf("trace %s params: %w", registration.Name, err) + } + } + selection, _ := params["target"].(map[string]any) + if selection == nil { + selection = map[string]any{} + } + for _, other := range []string{"selector", "pid"} { + if _, set := selection[other]; set { + return raw, nil + } + } + seam, _ := selection["seam"].(string) + query, _ := selection["agent"].(string) + if seam == "" && query == "" && registration.DefaultSeam != target.Agent { + return raw, nil + } + if seam != "" && seam != string(target.Agent) { + return raw, nil + } + agent, err := agentclient.Default.Lookup(ctx, query, agentclient.Want{Probe: registration.Name}) + if err != nil { + return nil, err + } + selection["agent"] = agent.Hello.AgentID + params["target"] = selection + return json.Marshal(params) +} diff --git a/internal/cli/agents.go b/internal/cli/agents.go index 036c26d..d777aa6 100644 --- a/internal/cli/agents.go +++ b/internal/cli/agents.go @@ -13,7 +13,6 @@ import ( "github.com/flanksource/telltale/pkg/agentclient" "github.com/flanksource/telltale/pkg/credentials/apitoken" "github.com/flanksource/telltale/pkg/datadir" - "github.com/flanksource/telltale/sdk/protocol" "github.com/spf13/cobra" ) @@ -21,29 +20,17 @@ func newAgentsCommand() *cobra.Command { agents := &cobra.Command{Use: "agents", Short: "Manage agents and their credentials"} var serverURL string list := &cobra.Command{ - Use: "list", Short: "List discovered and connected agents", Args: cobra.NoArgs, + Use: "list", Short: "List discovered, connected and deployed agents", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { infos, err := agentInfos(cmd.Context(), serverURL) if err != nil { return err } - groups := make(map[string][]protocol.Hello) - for _, info := range infos { - key := "other" - if namespace := info.Resource["k8s.namespace.name"]; namespace != "" { - key = "namespace/" + namespace - } else if provider := info.Resource["cloud.provider"]; provider != "" { - key = "cloud/" + provider + "/" + info.Resource["cloud.region"] - } else if host := info.Resource["host.name"]; host != "" { - key = "host/" + host - } - groups[key] = append(groups[key], info) - } - return json.NewEncoder(cmd.OutOrStdout()).Encode(groups) + return printRecords(cmd, agentclient.GroupInventory(infos)) }, } get := &cobra.Command{ - Use: "get ", Short: "Show a discovered or connected agent", Args: cobra.ExactArgs(1), + Use: "get ", Short: "Show a discovered, connected or deployed agent", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { infos, err := agentInfos(cmd.Context(), serverURL) if err != nil { @@ -57,8 +44,8 @@ func newAgentsCommand() *cobra.Command { return fmt.Errorf("agent %q not found", args[0]) }, } - list.Flags().StringVar(&serverURL, "server", "", "Telltale server URL for connected agents") - get.Flags().StringVar(&serverURL, "server", "", "Telltale server URL for connected agents") + list.Flags().StringVar(&serverURL, "server", "", "Telltale server URL for agent inventory") + get.Flags().StringVar(&serverURL, "server", "", "Telltale server URL for agent inventory") tokens := &cobra.Command{Use: "token", Short: "Manage hub credentials"} var scope string create := &cobra.Command{ @@ -78,11 +65,13 @@ func newAgentsCommand() *cobra.Command { } create.Flags().StringVar(&scope, "scope", string(apitoken.ScopeAgent), "Token scope: agent or api") tokens.AddCommand(create) - agents.AddCommand(list, get, tokens, newAgentInstallCommand()) + install := &cobra.Command{Use: "install", Short: "Install a runtime agent"} + install.AddCommand(newJavaDeployCommand("java", "Attach the Java agent to one PID, or deploy it to a Deployer's pods (same command as jvm deploy)", "agents/install/java")) + agents.AddCommand(list, get, tokens, install) return agents } -func agentInfos(ctx context.Context, serverURL string) (infos []protocol.Hello, err error) { +func agentInfos(ctx context.Context, serverURL string) (infos []agentclient.Info, err error) { if serverURL != "" { request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(serverURL, "/")+"/api/agents", nil) if err != nil { @@ -102,14 +91,12 @@ func agentInfos(ctx context.Context, serverURL string) (infos []protocol.Hello, if err := json.NewDecoder(io.LimitReader(response.Body, 16<<20)).Decode(&infos); err != nil { return nil, fmt.Errorf("decode agents: %w", err) } + for _, info := range infos { + if info.AgentID == "" || info.Group == "" || (!info.Connected && info.Known == nil) || (info.Connected && info.Hello == nil) { + return nil, fmt.Errorf("server returned incomplete agent inventory metadata for %q", info.AgentID) + } + } return infos, nil } - if err := agentclient.Default.Discover(ctx, ""); err != nil { - return nil, err - } - infos = make([]protocol.Hello, 0) - for _, agent := range agentclient.Default.Match(nil) { - infos = append(infos, agent.Hello) - } - return infos, nil + return agentclient.Default.Inventory(ctx) } diff --git a/internal/cli/agents_install.go b/internal/cli/agents_install.go deleted file mode 100644 index af4555e..0000000 --- a/internal/cli/agents_install.go +++ /dev/null @@ -1,36 +0,0 @@ -package cli - -import ( - "encoding/json" - "github.com/flanksource/telltale/pkg/agentinstall" - "github.com/spf13/cobra" -) - -func newAgentInstallCommand() *cobra.Command { - install := &cobra.Command{Use: "install", Short: "Install a runtime agent"} - var o agentinstall.Options - java := &cobra.Command{Use: "java", Short: "Attach the Java agent to a local, Kubernetes or SSH JVM", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { - result, err := (agentinstall.Installer{Runtime: agentinstall.Java{Home: o.JavaHome}}).Install(cmd.Context(), o) - if err != nil { - return err - } - return json.NewEncoder(cmd.OutOrStdout()).Encode(result) - }} - f := java.Flags() - f.StringVar(&o.Destination.Kind, "destination", "local", "Destination: local, kubernetes or ssh") - f.IntVar(&o.PID, "pid", 0, "Target JVM PID") - f.StringVar(&o.Directory, "directory", "", "Absolute destination directory for versioned agent artifacts") - f.StringVar(&o.JavaHome, "java-home", "", "Target-compatible JDK home, including Attach API") - f.StringVar(&o.Destination.Host, "host", "", "SSH host from the native SSH configuration") - f.StringVar(&o.Destination.SSHConfig, "ssh-config", "", "Native SSH configuration file") - f.StringVar(&o.Destination.Context, "context", "", "Kubernetes context") - f.StringVar(&o.Destination.Namespace, "namespace", "", "Kubernetes namespace") - f.StringVar(&o.Destination.Pod, "pod", "", "Kubernetes pod") - f.StringVar(&o.Destination.Container, "container", "", "Kubernetes container") - f.StringVar(&o.HubURL, "hub-url", "", "Hub URL reachable from the target JVM") - f.StringVar(&o.TokenFile, "token-file", "", "Agent-scope hub token file; contents are not printed") - f.StringVar(&o.Service, "service", "java", "Service identity advertised by the agent") - f.BoolVar(&o.DryRun, "dry-run", false, "Preview the installation without changing the target") - install.AddCommand(java) - return install -} diff --git a/internal/cli/agents_inventory_internal_test.go b/internal/cli/agents_inventory_internal_test.go new file mode 100644 index 0000000..83d8c15 --- /dev/null +++ b/internal/cli/agents_inventory_internal_test.go @@ -0,0 +1,61 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "path/filepath" + + "github.com/flanksource/telltale/pkg/agentclient" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/spf13/cobra" +) + +var _ = Describe("agent inventory commands", func() { + const target = "deployer/orders/shop/orders-0" + var command *cobra.Command + BeforeEach(func() { + original := agentclient.Default + DeferCleanup(func() { agentclient.Default = original }) + agentclient.Default = agentclient.NewRegistry() + agentclient.Default.Dir = filepath.Join(GinkgoT().TempDir(), "no-local-agents") + command = NewRoot("test") + agentclient.Default.SetLister("deployer/", func(context.Context) ([]agentclient.Known, error) { + return []agentclient.Known{{ID: target, Runtime: "java", Names: []string{"orders"}, + Resource: map[string]string{"k8s.namespace.name": "shop"}, State: "deployed", Endpoint: "http://orders-0:7070"}}, nil + }) + }) + + It("groups an unconnected deployed agent by namespace", func() { + output := &bytes.Buffer{} + command.SetOut(output) + command.SetArgs([]string{"agents", "list", "--format", "json"}) + Expect(command.Execute()).To(Succeed()) + var groups map[string][]agentclient.Info + Expect(json.Unmarshal(output.Bytes(), &groups)).To(Succeed()) + Expect(groups).To(HaveKey("namespace/shop")) + Expect(groups["namespace/shop"]).To(ConsistOf(And(HaveField("AgentID", target), HaveField("Connected", false), HaveField("Known.State", "deployed")))) + }) + + It("shows deployment details for an exact known agent id", func() { + output := &bytes.Buffer{} + command.SetOut(output) + command.SetArgs([]string{"agents", "get", target}) + Expect(command.Execute()).To(Succeed()) + var info agentclient.Info + Expect(json.Unmarshal(output.Bytes(), &info)).To(Succeed()) + Expect(info.AgentID).To(Equal(target)) + Expect(info.Known.Endpoint).To(Equal("http://orders-0:7070")) + }) + + It("pretty prints namespace groups and deployment and connection columns by default", func() { + output := &bytes.Buffer{} + command.SetOut(output) + command.SetArgs([]string{"agents", "list", "--format", "pretty"}) + Expect(command.Execute()).To(Succeed()) + Expect(output.String()).To(And(ContainSubstring("namespace/shop"), ContainSubstring("Agent"), + ContainSubstring("Connection"), ContainSubstring("Not connected"), ContainSubstring("deployed"))) + Expect(json.Valid(output.Bytes())).To(BeFalse()) + }) +}) diff --git a/internal/cli/agents_test.go b/internal/cli/agents_test.go index 2f13001..11930a1 100644 --- a/internal/cli/agents_test.go +++ b/internal/cli/agents_test.go @@ -13,7 +13,6 @@ import ( "github.com/flanksource/telltale/pkg/agentclient" "github.com/flanksource/telltale/pkg/credentials/apitoken" "github.com/flanksource/telltale/pkg/datadir" - "github.com/flanksource/telltale/sdk/protocol" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -47,17 +46,19 @@ var _ = Describe("agent token command", func() { GinkgoT().Setenv("TELLTALE_API_TOKEN", "example-api-token") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { Expect(r.Header.Get("Authorization")).To(Equal("Bearer example-api-token")) - _, _ = w.Write([]byte(`[{"agentId":"remote-agent","resource":{"service.name":"example"}}]`)) + _, _ = w.Write([]byte(`[{"agentId":"remote-agent","service":"example","connected":true,"group":"other","resource":{"service.name":"example"}}]`)) })) DeferCleanup(server.Close) output := &bytes.Buffer{} root := cli.NewRoot("test") root.SetOut(output) - root.SetArgs([]string{"agents", "list", "--server", server.URL}) + root.SetArgs([]string{"agents", "list", "--server", server.URL, "--format", "json"}) Expect(root.Execute()).To(Succeed()) - var groups map[string][]protocol.Hello + var groups map[string][]agentclient.Info Expect(json.Unmarshal(output.Bytes(), &groups)).To(Succeed()) Expect(groups["other"]).To(HaveLen(1)) Expect(groups["other"][0].AgentID).To(Equal("remote-agent")) + Expect(groups["other"][0].Connected).To(BeTrue()) + Expect(groups["other"][0].Resource).To(HaveKeyWithValue("service.name", "example")) }) }) diff --git a/internal/cli/connection_completion.go b/internal/cli/connection_completion.go new file mode 100644 index 0000000..2550e49 --- /dev/null +++ b/internal/cli/connection_completion.go @@ -0,0 +1,53 @@ +package cli + +import ( + "encoding/json" + "github.com/flanksource/telltale" + "github.com/spf13/cobra" + "net/url" + "slices" + "strings" +) + +func bindConnectionCompletions(cmd *cobra.Command, params paramFlags) error { + for _, param := range params.bound { + if param.path[len(param.path)-1] != "connection" { + continue + } + raw, err := json.Marshal(param.schema.Extensions["x-clicky-lookup"]) + if err != nil { + return err + } + var lookup struct { + URL string `json:"url"` + } + if err := json.Unmarshal(raw, &lookup); err != nil { + return err + } + endpoint, err := url.Parse(lookup.URL) + if err != nil { + return err + } + types := strings.Split(endpoint.Query().Get("types"), ",") + if err := cmd.RegisterFlagCompletionFunc(param.flag.Name, func(_ *cobra.Command, _ []string, prefix string) ([]string, cobra.ShellCompDirective) { + connections, err := telltale.OpenConnections() + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + infos, err := telltale.ListConnections(connections, "") + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + choices := []string{} + for _, info := range infos { + if strings.HasPrefix(info.Reference, prefix) && (types[0] == "" || slices.Contains(types, info.Type)) { + choices = append(choices, info.Reference+"\t"+info.Name+" ("+info.Type+")") + } + } + return choices, cobra.ShellCompDirectiveNoFileComp + }); err != nil { + return err + } + } + return nil +} diff --git a/internal/cli/explain.go b/internal/cli/explain.go new file mode 100644 index 0000000..46cb1f9 --- /dev/null +++ b/internal/cli/explain.go @@ -0,0 +1,95 @@ +package cli + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" + "github.com/flanksource/telltale/pkg/traces" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/target" + "github.com/spf13/cobra" +) + +type kindExplanation struct { + tracing.KindInfo + Summary string `json:"summary"` + Description string `json:"description"` + Seams []target.Seam `json:"seams"` + DefaultSeam target.Seam `json:"defaultSeam"` +} + +func (e kindExplanation) Pretty() api.Text { + t := clicky.Text(e.Name, "font-bold").Append(" — " + e.Summary).NewLine() + if e.Description != "" { + t = t.Append(e.Description).NewLine() + } + seams := make([]string, 0, len(e.Seams)) + for _, seam := range e.Seams { + seams = append(seams, string(seam)) + } + t = t.Append("Targets: " + strings.Join(seams, ", ") + " (default: " + string(e.DefaultSeam) + ")").NewLine() + data, err := json.MarshalIndent(e.Params, "", " ") + if err != nil { + panic(fmt.Errorf("marshal %s parameter schema: %w", e.Name, err)) + } + t = t.Append("Parameters", "font-bold").NewLine().Add(clicky.CodeBlock("application/json", string(data))).NewLine() + t = t.Append("Record schema", "font-bold").NewLine() + for _, column := range e.Columns { + t = t.Append(fmt.Sprintf("%s %s %s", column.Name, column.Type, column.Label)).NewLine() + } + return t +} + +func newExplainCommand() *cobra.Command { + return &cobra.Command{ + Use: "explain [kind]", Short: "Describe trace kinds and their schemas", Args: cobra.MaximumNArgs(1), + ValidArgsFunction: func(_ *cobra.Command, _ []string, prefix string) ([]string, cobra.ShellCompDirective) { + var names []string + for _, registration := range traces.Registrations() { + if strings.HasPrefix(registration.Name, prefix) { + names = append(names, registration.Name) + } + } + return names, cobra.ShellCompDirectiveNoFileComp + }, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + var list []struct { + Name string `json:"name"` + Summary string `json:"summary"` + } + for _, registration := range traces.Registrations() { + list = append(list, struct { + Name string `json:"name"` + Summary string `json:"summary"` + }{registration.Name, registration.Summary}) + } + return printRecords(cmd, list) + } + kinds, err := traces.Kinds() + if err != nil { + return err + } + if _, err := traces.Lookup(kinds, args[0]); err != nil { + return err + } + registration, err := traces.Registered(args[0]) + if err != nil { + return err + } + list, err := kinds.List() + if err != nil { + return err + } + for _, info := range list { + if info.Name == args[0] { + return printRecords(cmd, kindExplanation{info, registration.Summary, registration.Description, registration.Seams, registration.DefaultSeam}) + } + } + return fmt.Errorf("trace kind %q is missing from catalog", args[0]) + }, + } +} diff --git a/internal/cli/explain_test.go b/internal/cli/explain_test.go new file mode 100644 index 0000000..a9763f1 --- /dev/null +++ b/internal/cli/explain_test.go @@ -0,0 +1,52 @@ +package cli_test + +import ( + "bytes" + "encoding/json" + + "github.com/flanksource/telltale/internal/cli" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("explain", func() { + It("lists registered kinds and rejects excess arguments", func() { + root := cli.NewRoot("test") + out := &bytes.Buffer{} + root.SetOut(out) + root.SetArgs([]string{"explain", "--json"}) + Expect(root.Execute()).To(Succeed()) + var kinds []map[string]any + Expect(json.Unmarshal(out.Bytes(), &kinds)).To(Succeed()) + Expect(kinds).To(ContainElement(HaveKeyWithValue("name", "http"))) + root = cli.NewRoot("test") + root.SetArgs([]string{"explain", "http", "sql"}) + Expect(root.Execute()).To(HaveOccurred()) + }) + It("returns the HTTP parameter schema and record columns as JSON", func() { + root := cli.NewRoot("test") + out := &bytes.Buffer{} + root.SetOut(out) + root.SetArgs([]string{"explain", "http", "--json"}) + Expect(root.Execute()).To(Succeed()) + var result map[string]any + Expect(json.Unmarshal(out.Bytes(), &result)).To(Succeed()) + Expect(result).To(HaveKey("params")) + Expect(result).To(HaveKey("columns")) + Expect(result["name"]).To(Equal("http")) + params := result["params"].(map[string]any)["properties"].(map[string]any) + Expect(params["mode"].(map[string]any)["enum"]).To(Equal([]any{"connect", "mitm"})) + }) + It("rejects unknown kinds", func() { + root := cli.NewRoot("test") + root.SetArgs([]string{"explain", "missing"}) + Expect(root.Execute()).To(MatchError(ContainSubstring("unknown trace kind"))) + }) + It("completes registered kind names", func() { + root := cli.NewRoot("test") + cmd, _, err := root.Find([]string{"explain"}) + Expect(err).NotTo(HaveOccurred()) + names, _ := cmd.ValidArgsFunction(cmd, nil, "htt") + Expect(names).To(ContainElement("http")) + }) +}) diff --git a/internal/cli/feed_output_test.go b/internal/cli/feed_output_test.go new file mode 100644 index 0000000..6580c7f --- /dev/null +++ b/internal/cli/feed_output_test.go @@ -0,0 +1,35 @@ +package cli_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + + "github.com/flanksource/clicky" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("live capture stderr", func() { + It("streams styled HTTP records without follow while preserving JSON stdout", func() { + GinkgoT().Setenv("CLICKY_FORCE_INTERACTIVE", "1") + GinkgoT().Setenv("TERM", "xterm-256color") + GinkgoT().Setenv("NO_COLOR", "") + GinkgoT().Setenv("COLOR", "true") + saved := *clicky.Flags + DeferCleanup(func() { *clicky.Flags = saved; clicky.UseFormatter(saved.FormatOptions) }) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("child-output")) + })) + DeferCleanup(upstream.Close) + var stdout, stderr string + var err error + formatted := captureStdout(func() { + stdout, stderr, err = execute("trace", "run", "--data-dir", GinkgoT().TempDir(), "--format", "json", "--", "curl", "--noproxy", "", "-sS", upstream.URL+"/orders") + }) + Expect(err).NotTo(HaveOccurred(), stderr) + Expect(stderr).To(And(ContainSubstring("GET"), ContainSubstring("/orders"), ContainSubstring("\x1b["), ContainSubstring("child-output"))) + Expect(stdout).To(BeEmpty()) + Expect(json.Valid([]byte(formatted))).To(BeTrue(), formatted) + }) +}) diff --git a/internal/cli/feed_printer.go b/internal/cli/feed_printer.go new file mode 100644 index 0000000..dea7b1c --- /dev/null +++ b/internal/cli/feed_printer.go @@ -0,0 +1,78 @@ +package cli + +import ( + "encoding/json" + "fmt" + "io" + "slices" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" + "github.com/flanksource/telltale/pkg/traces/read" + "github.com/flanksource/telltale/pkg/tracing/jvmtrace" +) + +// feedPrinter prints followed or replayed records one line each. A +// jvm_trace stream's method events fold into calls, printed as each outermost +// call exits; ndjson keeps every record as recorded. +type feedPrinter struct { + out io.Writer + ndjson bool + calls map[string]*jvmtrace.Calls + streams []string +} + +func newFeedPrinter(out io.Writer) *feedPrinter { + return &feedPrinter{out: out, ndjson: clicky.Flags.ResolveFormat() == "ndjson", calls: map[string]*jvmtrace.Calls{}} +} + +func (p *feedPrinter) print(record read.Record) error { + if p.ndjson { + return json.NewEncoder(p.out).Encode(record) + } + if record.Kind != "jvm_trace" || record.Result != "" { + return p.line(record.Pretty()) + } + event, err := jvmtrace.EventOf(record.Record) + if err != nil { + return fmt.Errorf("jvm_trace %s #%d: %w", record.Stream, record.Seq, err) + } + calls := p.calls[record.Stream] + if calls == nil { + calls = &jvmtrace.Calls{} + p.calls[record.Stream] = calls + p.streams = append(p.streams, record.Stream) + } + for _, line := range calls.Add(event) { + if err := p.line(line); err != nil { + return err + } + } + return nil +} + +// stats are a line per jvm_trace stream the printer folded, naming the stream +// only when there were several. +func (p *feedPrinter) stats(out io.Writer) error { + for _, stream := range slices.Clone(p.streams) { + name := "jvm_trace" + if len(p.streams) > 1 { + name += " " + stream + } + if err := writeLine(out, api.Text{}.Append(name+" · ", "font-bold").Add(p.calls[stream].Stats().Pretty())); err != nil { + return err + } + } + return nil +} + +func (p *feedPrinter) line(text api.Textable) error { return writeLine(p.out, text) } + +func writeLine(out io.Writer, text api.Textable) error { + output, err := clicky.Format(text, clicky.FormatOptions{Format: "pretty"}) + if err != nil { + return err + } + _, err = fmt.Fprintln(out, output) + return err +} diff --git a/internal/cli/follow.go b/internal/cli/follow.go new file mode 100644 index 0000000..7ff46bb --- /dev/null +++ b/internal/cli/follow.go @@ -0,0 +1,127 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "os" + "time" + + tea "charm.land/bubbletea/v2" + "github.com/flanksource/clicky/rpc" + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/internal/server" + "github.com/flanksource/telltale/internal/shutdown" + "github.com/flanksource/telltale/internal/traceui" + "github.com/flanksource/telltale/pkg/traces" + "github.com/flanksource/telltale/pkg/traces/read" + "github.com/spf13/cobra" + "golang.org/x/term" +) + +func bindFollow(cmd *cobra.Command) { + cmd.Flags().Bool("follow", false, "Print every armed capture's records while recording") + cmd.Flags().Bool("tui", false, "Follow records in an interactive terminal") + cmd.Flags().Bool("ui", false, "Open this run's traces in a local web viewer; keep serving until Ctrl+C") + cmd.Flags().String("sort", "seq", "Record order: seq or timestamp") +} + +func followOptions(cmd *cobra.Command) (bool, bool, bool, string, error) { + follow, err := cmd.Flags().GetBool("follow") + if err != nil { + return false, false, false, "", err + } + tui, err := cmd.Flags().GetBool("tui") + if err != nil { + return false, false, false, "", err + } + ui, err := cmd.Flags().GetBool("ui") + if err != nil { + return false, false, false, "", err + } + sort, err := cmd.Flags().GetString("sort") + if err != nil { + return false, false, false, "", err + } + if sort != "seq" && sort != "timestamp" { + return false, false, false, "", fmt.Errorf("unknown record sort %q", sort) + } + if (tui || ui) && !cmd.Flags().Changed("sort") { + sort = "timestamp" + } + if tui && (!term.IsTerminal(int(os.Stdin.Fd())) || !term.IsTerminal(int(os.Stdout.Fd()))) { + return false, false, false, "", errors.New("--tui requires a terminal") + } + return follow || tui, tui, ui, sort, nil +} + +func openRunViewer(cmd *cobra.Command, options server.ViewerOptions) (*server.Viewer, error) { + viewer, err := server.StartViewer(options) + if err != nil { + return nil, err + } + order, err := cmd.Flags().GetString("sort") + if err != nil { + return nil, err + } + if !cmd.Flags().Changed("sort") { + order = "timestamp" + } + viewer.URL += "&feedSort=" + order + if _, err := fmt.Fprintln(cmd.ErrOrStderr(), viewer.URL); err != nil { + return nil, err + } + // Opening the browser is an output action; readiness is established by the listener. + if err := rpc.OpenBrowser(viewer.URL); err != nil { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return nil, errors.Join(err, viewer.Close(ctx)) + } + return viewer, nil +} + +func waitViewer(ctx context.Context, viewer *server.Viewer) error { + <-ctx.Done() + closeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return viewer.Close(closeCtx) +} + +func showTraceTUI(ctx context.Context, run *telltale.Run, sort string) (_ *telltale.Result, err error) { + runtime, closeStore, err := traces.Reopen(run.Dir()) + if err != nil { + return nil, errors.Join(err, run.Stop()) + } + defer func() { err = errors.Join(err, closeStore()) }() + info := run.Info() + refs, err := read.RunRefs(info, runtime) + if err != nil { + return nil, err + } + feedCtx, cancel := context.WithCancel(context.WithoutCancel(ctx)) + defer cancel() + program := tea.NewProgram(traceui.New(feedCtx, traceui.Options{Refs: refs, Runs: []telltale.RunInfo{info}, Sort: sort}), tea.WithContext(ctx)) + restore, err := shutdown.Terminal(ctx) + if err != nil { + return nil, errors.Join(err, run.Stop()) + } + defer restore() + go func() { + select { + case <-run.Done(): + info := run.Info() + program.Send(traceui.End{Info: &info}) + case <-feedCtx.Done(): + } + }() + final, runErr := program.Run() + restore() + err = errors.Join(runErr, shutdown.Request(ctx)) + if model, ok := final.(traceui.Model); ok { + err = errors.Join(err, model.Err()) + } + stopErr := run.Stop() + result, waitErr := run.Wait() + cancel() + return result, errors.Join(err, stopErr, waitErr) +} diff --git a/internal/cli/help_pages.go b/internal/cli/help_pages.go index 40484ba..af1a82d 100644 --- a/internal/cli/help_pages.go +++ b/internal/cli/help_pages.go @@ -25,6 +25,8 @@ var targets = []struct { "The command is started with the variables the capture exports. They cannot attach to a running process."}, {target.Process, "Kinds that observe a process tree", "They observe telltale's own tree, which holds the command, or the process --pid attaches to."}, + {target.Agent, "Kinds read through an agent", + "They arm a probe in a process that embeds a telltale agent: the process --pid names, or the connected agents --selector matches."}, {target.Self, "Kinds that record telltale itself", "They record telltale's own sources while the run lasts, nothing of the command."}, {target.Host, "Kinds that observe the whole host", @@ -74,7 +76,7 @@ func runHelp(cmd *cobra.Command) helpText { } return h.section("Examples"). command("telltale run http --seam proxy --mode mitm -- curl -sS https://example.com", "decrypt and record a request"). - command("telltale run process_samples --pid 1234 --facets ports --duration 1m", "sample a running process for a minute"). + command("telltale run process --pid 1234 --facets sockets --duration 1m", "sample a running process for a minute"). command("telltale run stacks --every 10s -- ./server", "dump a server's stacks every 10 seconds"). section("See also"). command("telltale run --help", "a kind's params, exports and examples"). diff --git a/internal/cli/interactive_test.go b/internal/cli/interactive_test.go new file mode 100644 index 0000000..e002d3b --- /dev/null +++ b/internal/cli/interactive_test.go @@ -0,0 +1,85 @@ +package cli + +import ( + "context" + "encoding/json" + "github.com/flanksource/clicky/prompt" + "github.com/flanksource/clicky/rpc" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/spf13/cobra" + "os" + "path/filepath" +) + +var _ = Describe("interactive run arguments", func() { + It("installs -i and publishes the required connection chooser in the shared form schema", func() { + root := NewRoot("test") + Expect(root.PersistentFlags().Lookup("interactive")).NotTo(BeNil()) + Expect(root.PersistentFlags().Lookup("interactive").Shorthand).To(Equal("i")) + cmd, _, err := root.Find([]string{"run", "sql_top"}) + Expect(err).NotTo(HaveOccurred()) + op, err := rpc.NewConverter(nil).ConvertCommand(cmd) + Expect(err).NotTo(HaveOccurred()) + Expect(op.Path).To(Equal("/api/v1/run/sql_top")) + raw, err := rpc.OperationInputSchema(*op, rpc.OperationSchemaOptions{}) + Expect(err).NotTo(HaveOccurred()) + var schema map[string]any + Expect(json.Unmarshal(raw, &schema)).To(Succeed()) + Expect(schema["required"]).To(ContainElement("connection")) + properties := schema["properties"].(map[string]any) + Expect(properties).NotTo(HaveKey("flags")) + Expect(properties["connection"]).To(HaveKeyWithValue("x-clicky-lookup", HaveKeyWithValue("url", "/api/v1/connection?types=postgres,sql_server"))) + }) + It("fills the schema before running and preserves supplied command arguments and flags", func() { + manager := prompt.NewManager(prompt.NewMemory(prompt.MemoryConfig{})) + prompt.SetDefault(manager) + DeferCleanup(func() { prompt.SetDefault(nil) }) + root := NewRoot("test") + cmd, _, err := root.Find([]string{"run", "sql_top"}) + Expect(err).NotTo(HaveOccurred()) + root.SetArgs([]string{"run", "sql_top", "-i", "--connection", "connection://saved", "--duration", "1s", "--", "echo", "captured"}) + ctx, cancel := context.WithCancel(prompt.WithScope(context.Background(), prompt.Scope{Owner: "telltale-form"})) + DeferCleanup(cancel) + root.SilenceErrors = true + done := make(chan error, 1) + go func() { done <- root.ExecuteContext(ctx) }() + var snapshot prompt.PromptSnapshot + Eventually(func() int { + pending := manager.List(prompt.Filter{Owner: "telltale-form", Kind: "form", State: "pending"}) + if len(pending) > 0 { + snapshot = pending[0] + } + return len(pending) + }).Should(Equal(1)) + Expect(snapshot.Value).To(HaveKeyWithValue("connection", "connection://saved")) + Expect(snapshot.Value).To(HaveKeyWithValue("args", Equal([]any{"echo", "captured"}))) + Expect(manager.Resolve(snapshot.ID, prompt.Answer{Cancelled: true})).To(Succeed()) + Eventually(done).Should(Receive(MatchError(ContainSubstring("prompt cancelled")))) + Expect(cmd.Flags().Changed("connection")).To(BeTrue()) + }) + It("registers a connection completion for each kind that accepts a connection", func() { + dir := GinkgoT().TempDir() + GinkgoT().Setenv("XDG_CONFIG_HOME", dir) + GinkgoT().Chdir(dir) + folder := filepath.Join(dir, ".connections") + Expect(os.MkdirAll(folder, 0700)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(folder, "database.yaml"), []byte("apiVersion: mission-control.flanksource.com/v1\nkind: Connection\nmetadata: {name: sample-db, namespace: tenant}\nspec: {postgres: {url: {value: 'postgres://localhost/example'}}}\n"), 0600)).To(Succeed()) + root := NewRoot("test") + for _, kind := range []string{"sql_top", "sql_plancache"} { + cmd, _, err := root.Find([]string{"run", kind}) + Expect(err).NotTo(HaveOccurred()) + complete, ok := cmd.GetFlagCompletionFunc("connection") + Expect(ok).To(BeTrue()) + Expect(complete).NotTo(BeNil()) + choices, _ := complete(cmd, nil, "") + if kind == "sql_top" { + Expect(choices).To(ConsistOf("connection://tenant/sample-db\tsample-db (postgres)")) + } else { + Expect(choices).To(BeEmpty()) + } + _, directive := complete(cmd, nil, "connection://does-not-exist") + Expect(directive & cobra.ShellCompDirectiveNoFileComp).NotTo(BeZero()) + } + }) +}) diff --git a/internal/cli/java_deploy.go b/internal/cli/java_deploy.go new file mode 100644 index 0000000..b530fa5 --- /dev/null +++ b/internal/cli/java_deploy.go @@ -0,0 +1,387 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "slices" + "strings" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/rpc" + dbkubernetes "github.com/flanksource/commons-db/kubernetes" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/pkg/agentinstall" + "github.com/flanksource/telltale/pkg/definition" + "github.com/flanksource/telltale/pkg/deploy" +) + +// javaDeployFlags are the inputs of the one command `telltale jvm deploy` and +// `telltale agents install java` both name. Which inputs are given picks how +// the agent gets into the JVM: +// - a Deployer document (--file or --deployer) deploys it; +// - a [kind/]name target or --issuer builds a Deployer from the flags, +// filling what they leave out (withDefaults), and deploys that; +// - otherwise the agent is attached to one --pid at a local, SSH or +// Kubernetes destination, verified locally or through a hub. +type javaDeployFlags struct { + file, deployer string + // target is the positional [kind/]name: sts/cycle, deploy/oipa, pod/cycle-0 or cycle-0. + target string + install agentinstall.Options + // A Deployer built from flags: the selector, the attach listener and auth. + name, issuer, jwksURL string + jwks deploy.JWKSSource + workloadKind, workload string + labels map[string]string + url, bind string + port int +} + +type javaDeployMode int + +const ( + modeAttach javaDeployMode = iota // one PID, local or hub verified + modeDeployerDocument // a Deployer document + modeDeployerFlags // a Deployer built from flags +) + +// The flags only some modes read; any other mode refuses them rather than +// silently ignoring what was asked. +var ( + documentFlags = []string{"file", "deployer"} + attachOnlyFlags = []string{"hub-url", "token-file", "host", "ssh-config"} + builtOnlyFlags = []string{"name", "issuer", "jwks-url", "jwks-mode", "jwks-kind", "jwks-name", "jwks-key", "kind", "workload", "labels", "url", "bind", "port"} + builtSharedFlags = []string{"pid", "directory", "java-home", "service", "allow-mutation", "extensions-dir", "namespace", "pod", "container"} +) + +func newJavaDeployCommand(use, short, route string) *cobra.Command { + var flags javaDeployFlags + cmd := &cobra.Command{ + Use: use + " [target]", + Short: short, + Long: "Gets telltale's Java agent into running JVMs. `telltale jvm deploy` and `telltale agents install java` are the same command.\n\n" + + "Given a target (sts/, deploy/, pod/ or a bare pod name), --issuer, or a Deployer (-f/--deployer), it deploys to the " + + "Kubernetes pods the selector picks: each agent verifies short-lived tokens signed by the issuer against its JWKS, needs no hub, and is " + + "reached later with --agent deployer///. A target or --issuer builds the Deployer from flags and defaults what they " + + "leave out: the issuer to " + deploy.DefaultIssuer + " (created with a new key in the shared Flanksource connections on first use), the " + + "namespace to the kubeconfig context's, the container to the pod's only one, --pid to JVM discovery through /proc, --directory to /tmp/telltale and --service and " + + "--name to the target's name. The deployer is kept with its targets' state and the --kubeconfig/--context it used. --dry-run lists the " + + "pods the selector picks.\n\n" + + "Otherwise it attaches the agent to one --pid at a local, SSH or Kubernetes --destination, verified by local discovery or by a hub " + + "(--hub-url and --token-file). --dry-run previews the artifact and its path without changing the target.", + Example: " telltale agents install java --namespace shop sts/orders\n" + + " telltale jvm deploy orders-0 --dry-run\n" + + " telltale agents install java --pid 1234 --directory ~/.local/share/telltale", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 1 { + flags.target = args[0] + } + mode, err := flags.mode(cmd.Flags()) + if err != nil { + return err + } + switch mode { + case modeAttach: + return runJavaAttach(cmd, flags) + case modeDeployerFlags: + if flags, err = flags.withDefaults(cmd.Flags()); err != nil { + return err + } + } + return runJavaDeployer(cmd, flags, mode) + }, + } + // The optional [target] would otherwise be lifted into the route + // (/api/v1/jvm/{target}/deploy); callers address the flat one. + clicky.AnnotateRoutePath(cmd, route) + f := cmd.Flags() + f.StringVarP(&flags.file, "file", "f", "", "Deployer YAML file (kind: Deployer); local invocations only") + f.StringVar(&flags.deployer, "deployer", "", "Deployer document, YAML or JSON (kind: Deployer)") + cmd.MarkFlagsMutuallyExclusive("file", "deployer") + + f.StringVar(&flags.install.Destination.Kind, "destination", "local", "Destination of a single attach: local, kubernetes or ssh (a deployer is always kubernetes)") + f.StringVar(&flags.install.Destination.Kubeconfig, "kubeconfig", "", "Kubeconfig file (default: KUBECONFIG or ~/.kube/config)") + f.StringVar(&flags.install.Destination.Context, "context", "", "Kubernetes context (default: the kubeconfig's current context)") + f.StringVar(&flags.install.Destination.Namespace, "namespace", "", "Kubernetes namespace") + f.StringVar(&flags.install.Destination.Pod, "pod", "", "Kubernetes pod") + f.StringVar(&flags.install.Destination.Container, "container", "", "Kubernetes container running the JVM") + f.StringVar(&flags.install.Destination.Host, "host", "", "SSH host from the native SSH configuration") + f.StringVar(&flags.install.Destination.SSHConfig, "ssh-config", "", "Native SSH configuration file") + f.IntVar(&flags.install.PID, "pid", 0, "Target JVM PID (Kubernetes: discover the only Java process in /proc when omitted)") + f.StringVar(&flags.install.Directory, "directory", "", "Absolute destination directory for versioned agent artifacts") + f.StringVar(&flags.install.JavaHome, "java-home", "", "Target-compatible JDK home, including Attach API") + f.StringVar(&flags.install.HubURL, "hub-url", "", "Hub URL reachable from the target JVM (single attach)") + f.StringVar(&flags.install.TokenFile, "token-file", "", "Agent-scope hub token file; contents are not printed (single attach)") + f.StringVar(&flags.install.Service, "service", "java", "Service identity advertised by the agent") + f.BoolVar(&flags.install.AllowMutation, "allow-mutation", false, "Enable JVM mutation operations; each call also requires opt-in") + f.StringVar(&flags.install.ExtensionsDir, "extensions-dir", "", "Absolute extension storage directory on the target JVM") + f.BoolVar(&flags.install.DryRun, "dry-run", false, "Deployer: list the pods the selector picks; single attach: preview the artifact. Neither changes anything") + f.BoolVar(&flags.install.Upgrade, "upgrade", false, "Replace an idle attached agent runtime without restarting its JVM") + + f.StringVar(&flags.issuer, "issuer", "", "connection:// reference holding the signingKey secret: builds a Deployer from these flags") + f.StringVar(&flags.name, "name", "", "Name of the built Deployer, part of its target ids (default: --service)") + f.StringVar(&flags.jwksURL, "jwks-url", "", "Where the issuer runtime serves its JWKS, for key rotation without a redeploy") + f.StringVar((*string)(&flags.jwks.Mode), "jwks-mode", "", "read or publish the JWKS in --jwks-kind/--jwks-name (unset: the issuer's own, inline)") + f.StringVar(&flags.jwks.Kind, "jwks-kind", "ConfigMap", "ConfigMap or Secret holding the JWKS") + f.StringVar(&flags.jwks.Name, "jwks-name", "", "Name of the ConfigMap or Secret holding the JWKS") + f.StringVar(&flags.jwks.Key, "jwks-key", "", "Data key holding the JWKS (default jwks.json)") + f.StringVar(&flags.workloadKind, "kind", "", "Workload the built Deployer selects: Deployment, StatefulSet or Pod (--pod implies Pod)") + f.StringVar(&flags.workload, "workload", "", "Name of the one workload to select, instead of --labels") + f.StringToStringVar(&flags.labels, "labels", nil, "Labels of the workloads to select, instead of --workload") + f.StringVar(&flags.url, "url", "", "Listener URL template (default: a port-forward to --port of each pod)") + f.StringVar(&flags.bind, "bind", "127.0.0.1", "Listener address in the container; the default is reachable only through a port-forward") + f.IntVar(&flags.port, "port", 7070, "Listener port in the container") + return cmd +} + +// mode is how the given flags deploy, refusing flags that mode would ignore. +func (f javaDeployFlags) mode(set *pflag.FlagSet) (javaDeployMode, error) { + given := func(names []string) []string { + var out []string + for _, name := range names { + if set.Changed(name) { + out = append(out, "--"+name) + } + } + return out + } + refuse := func(names []string, why string) error { + if extra := given(names); len(extra) > 0 { + return fmt.Errorf("%s: %s", strings.Join(extra, ", "), why) + } + return nil + } + destination := set.Changed("destination") && f.install.Destination.Kind != "kubernetes" + switch { + case len(given(documentFlags)) > 0: + if destination { + return 0, errors.New("a Deployer deploys to Kubernetes; --destination must be kubernetes") + } + if f.target != "" { + return 0, fmt.Errorf("target %s builds a Deployer from flags; the Deployer document already selects its pods", f.target) + } + return modeDeployerDocument, errors.Join(refuse(attachOnlyFlags, "only a single attach reads these, not a Deployer"), + refuse(slices.Concat(builtOnlyFlags, builtSharedFlags), "these build a Deployer from flags; the Deployer document already says them")) + case f.issuer != "" || f.target != "": + if destination { + return 0, errors.New("a Deployer deploys to Kubernetes; --destination must be kubernetes") + } + var selectorErr error + if f.target != "" { + selectorErr = refuse([]string{"pod", "kind", "workload", "labels"}, "target "+f.target+" already selects the pods") + } + return modeDeployerFlags, errors.Join(selectorErr, + refuse(attachOnlyFlags, "only a single attach reads these; a Deployer's agents verify issuer tokens and need no hub")) + default: + return modeAttach, refuse(slices.DeleteFunc(slices.Clone(builtOnlyFlags), func(name string) bool { return name == "issuer" }), + "these build a Deployer, which needs a target or --issuer") + } +} + +// The defaults a Deployer built from flags takes for what they leave out. +const defaultDirectory = "/tmp/telltale" + +// withDefaults fills what a Deployer built from flags leaves out: the +// target's selector, the default issuer, JVM discovery through /proc, +// defaultDirectory, and the target's name as --service. The namespace and +// container default at deploy time, from the kubeconfig and the pods. +func (f javaDeployFlags) withDefaults(set *pflag.FlagSet) (javaDeployFlags, error) { + name := f.install.Destination.Pod + if f.target != "" { + kind, target, err := parseTarget(f.target) + if err != nil { + return f, err + } + name = target + if kind == deploy.WorkloadPod { + f.install.Destination.Pod = target + } else { + f.workloadKind, f.workload = string(kind), target + } + } + if name == "" { + name = f.workload + } + if f.issuer == "" { + f.issuer = deploy.DefaultIssuer + } + if f.install.Directory == "" { + f.install.Directory = defaultDirectory + } + if !set.Changed("service") && name != "" { + f.install.Service = name + } + return f, nil +} + +// parseTarget reads kubectl-style sts/, deploy/ and pod/ +// (or their long and plural forms); a bare name is a pod. +func parseTarget(target string) (deploy.WorkloadKind, string, error) { + prefix, name, typed := strings.Cut(target, "/") + if !typed { + prefix, name = "pod", target + } + if name == "" || strings.Contains(name, "/") { + return "", "", fmt.Errorf("target %q: want [kind/]name, such as sts/orders, deploy/orders or orders-0", target) + } + switch strings.ToLower(prefix) { + case "sts", "statefulset", "statefulsets": + return deploy.WorkloadStatefulSet, name, nil + case "deploy", "deployment", "deployments": + return deploy.WorkloadDeployment, name, nil + case "po", "pod", "pods": + return deploy.WorkloadPod, name, nil + } + return "", "", fmt.Errorf("target %q: kind %q is not sts, deploy or pod", target, prefix) +} + +// builtDeployer is the Deployer the flags describe: one attach listener per +// selected pod, verifying tokens --issuer signed. +func (f javaDeployFlags) builtDeployer() (*deploy.Deployer, error) { + d := f.install.Destination + selector := deploy.Selector{Namespace: d.Namespace, Container: d.Container, Name: f.workload, Labels: f.labels} + if d.Pod != "" { + if f.workload != "" || len(f.labels) > 0 || (f.workloadKind != "" && !strings.EqualFold(f.workloadKind, string(deploy.WorkloadPod))) { + return nil, errors.New("--pod selects one pod; it cannot be combined with --kind, --workload or --labels") + } + selector.Kind, selector.Name = deploy.WorkloadPod, d.Pod + } else { + kind, err := workloadKind(f.workloadKind) + if err != nil { + return nil, err + } + selector.Kind = kind + } + url := f.url + if url == "" { + url = fmt.Sprintf("portforward://{{.Pod.Name}}.{{.Pod.Namespace}}:%d?kind=pod", f.port) + } + container, err := deploy.NewJavaContainerSpec(deploy.AttachSpec{Kind: deploy.KindAttach, URL: url, PID: f.install.PID, Directory: f.install.Directory, + Bind: f.bind, Port: f.port, Service: f.install.Service, JavaHome: f.install.JavaHome, ExtensionsDir: f.install.ExtensionsDir, AllowMutation: f.install.AllowMutation}) + if err != nil { + return nil, err + } + auth := deploy.AuthSpec{Issuer: f.issuer, JWKSURL: f.jwksURL} + if f.jwks.Mode != "" || f.jwks.Name != "" { + jwks := f.jwks + auth.JWKS = &jwks + } + name := f.name + if name == "" { + name = f.install.Service + } + return &deploy.Deployer{TypeMeta: metav1.TypeMeta{APIVersion: definition.APIVersion, Kind: deploy.Kind}, ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: deploy.Spec{Selector: selector, JavaContainer: container, Auth: auth}}, nil +} + +func workloadKind(value string) (deploy.WorkloadKind, error) { + for _, kind := range []deploy.WorkloadKind{deploy.WorkloadDeployment, deploy.WorkloadStatefulSet, deploy.WorkloadPod} { + if strings.EqualFold(value, string(kind)) { + return kind, nil + } + } + return "", fmt.Errorf("--kind %q: want Deployment, StatefulSet or Pod (or --pod)", value) +} + +// loadDeployer is the Deployer a document mode names: the --deployer +// document, or the --file one, which only a local invocation may read. +func (f javaDeployFlags) loadDeployer(ctx context.Context) (*deploy.Deployer, error) { + if f.deployer != "" { + return deploy.Decode([]byte(f.deployer)) + } + if _, remote := rpc.RequestFromContext(ctx); remote { + return nil, errors.New("--file reads a file on the machine running the command; send deployer in the request instead") + } + data, err := os.ReadFile(f.file) + if err != nil { + return nil, err + } + return deploy.Decode(data) +} + +// ensureDefaultIssuer creates deploy.DefaultIssuer in the shared connections +// on first use, saying so on stderr; an existing one keeps its key. +func ensureDefaultIssuer(cmd *cobra.Command) error { + connections, err := telltale.OpenConnections() + if err != nil { + return err + } + created, err := deploy.EnsureIssuer(connections, deploy.DefaultIssuer) + if created { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "created issuer %s with a new signing key in the shared Flanksource connections\n", deploy.DefaultIssuer) + } + return err +} + +func runJavaAttach(cmd *cobra.Command, flags javaDeployFlags) error { + result, err := (agentinstall.Installer{Runtime: agentinstall.Java{}}).Install(cmd.Context(), flags.install) + if err != nil { + return err + } + return json.NewEncoder(cmd.OutOrStdout()).Encode(result) +} + +func runJavaDeployer(cmd *cobra.Command, flags javaDeployFlags, mode javaDeployMode) error { + cluster := deploy.Cluster{Kubeconfig: flags.install.Destination.Kubeconfig, Context: flags.install.Destination.Context} + var deployer *deploy.Deployer + var err error + if mode == modeDeployerDocument { + deployer, err = flags.loadDeployer(cmd.Context()) + } else { + if flags.install.Destination.Namespace == "" { + if flags.install.Destination.Namespace, err = (dbkubernetes.Options{Kubeconfig: cluster.Kubeconfig, Kubecontext: cluster.Context}).Namespace(); err != nil { + return fmt.Errorf("--namespace not given: %w", err) + } + } + if !flags.install.DryRun && flags.issuer == deploy.DefaultIssuer { + if err := ensureDefaultIssuer(cmd); err != nil { + return err + } + } + deployer, err = flags.builtDeployer() + } + if err != nil { + return err + } + ctx, err := deployerContext(cmd.Context()) + if err != nil { + return err + } + if ctx, err = cluster.Bind(ctx); err != nil { + return fmt.Errorf("kubernetes client: %w", err) + } + client, err := ctx.LocalKubernetes() + if err != nil { + return fmt.Errorf("kubernetes client: %w", err) + } + options := deploy.Options{Kubernetes: client, Cluster: cluster, DryRun: flags.install.DryRun, Upgrade: flags.install.Upgrade} + if !options.DryRun { + if options.Issuer, err = deploy.NewIssuer(ctx, deployer.Spec.Auth); err != nil { + return fmt.Errorf("deployer %s: %w", deployer.Name, err) + } + } + status, deployErr := deploy.Deploy(ctx, *deployer, options) + if len(status.Targets) == 0 { + return deployErr + } + if !options.DryRun { + store, err := deployerStore() + if err != nil { + return err + } + cluster.Annotate(deployer) + deployer.Status = status + if err := store.Save(*deployer); err != nil { + return err + } + } + return errors.Join(printRecords(cmd, status.Targets), deployErr) +} diff --git a/internal/cli/java_deploy_internal_test.go b/internal/cli/java_deploy_internal_test.go new file mode 100644 index 0000000..9c5293a --- /dev/null +++ b/internal/cli/java_deploy_internal_test.go @@ -0,0 +1,211 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "slices" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/flanksource/telltale/pkg/agentinstall" + "github.com/flanksource/telltale/pkg/definition" + "github.com/flanksource/telltale/pkg/deploy" +) + +// commandAt is the command root reaches by path. +func commandAtPath(path ...string) *cobra.Command { + cmd, _, err := NewRoot("test").Find(path) + Expect(err).NotTo(HaveOccurred()) + return cmd +} + +func flagNames(cmd *cobra.Command) []string { + var names []string + cmd.Flags().VisitAll(func(flag *pflag.Flag) { names = append(names, flag.Name+"="+flag.DefValue) }) + slices.Sort(names) + return names +} + +// parsedFlags are the javaDeployFlags and flag set args parse into. +func parsedFlags(args ...string) (javaDeployFlags, *pflag.FlagSet) { + cmd := commandAtPath("jvm", "deploy") + Expect(cmd.ParseFlags(args)).To(Succeed()) + var flags javaDeployFlags + // The command's flags bind to its own javaDeployFlags; read them back by name. + get := func(name string) string { return cmd.Flags().Lookup(name).Value.String() } + flags.file, flags.deployer, flags.issuer, flags.name = get("file"), get("deployer"), get("issuer"), get("name") + flags.install.Destination.Kind = get("destination") + return flags, cmd.Flags() +} + +var _ = Describe("jvm deploy and agents install java", func() { + It("are one command with the same flags under both names", func() { + deployCmd, installCmd := commandAtPath("jvm", "deploy"), commandAtPath("agents", "install", "java") + Expect(deployCmd.Name()).To(Equal("deploy")) + Expect(installCmd.Name()).To(Equal("java")) + Expect(flagNames(installCmd)).To(Equal(flagNames(deployCmd))) + Expect(flagNames(deployCmd)).To(ContainElements("hub-url=", "token-file=", "host=", "file=", "deployer=", "issuer=", "jwks-name=", "labels=[]", "kubeconfig=", "context=", "upgrade=false")) + }) + + It("serves both names over RPC with the same inputs", func() { + spec := kindOpenAPI(NewRoot("test")) + inputs := func(path string) []string { + Expect(spec.Paths).To(HaveKey(path)) + var names []string + for name := range spec.Paths[path]["post"].RequestBody.Content["application/json"].Schema.Properties { + names = append(names, name) + } + slices.Sort(names) + return names + } + Expect(inputs("/api/v1/agents/install/java")).To(Equal(inputs("/api/v1/jvm/deploy"))) + Expect(inputs("/api/v1/jvm/deploy")).To(ContainElements("target", "deployer", "issuer", "dry-run", "upgrade")) + }) + + It("builds a Deployer from flags: one pod's attach listener, reached over a port-forward, verifying the issuer's tokens", func() { + cmd := commandAtPath("agents", "install", "java") + flags := javaDeployFlags{install: agentinstall.Options{Destination: agentinstall.Destination{Kind: "kubernetes", Namespace: "shop", Pod: "orders-0", Container: "orders"}, + PID: 1, Directory: "/tmp/telltale", Service: "orders"}, issuer: "connection://shop/telltale-signing", bind: "127.0.0.1", port: 7070, + jwks: deploy.JWKSSource{Mode: deploy.JWKSPublish, Kind: "ConfigMap", Name: "telltale-jwks"}} + mode, err := flags.mode(cmd.Flags()) + Expect(err).NotTo(HaveOccurred()) + Expect(mode).To(Equal(modeDeployerFlags)) + built, err := flags.builtDeployer() + Expect(err).NotTo(HaveOccurred()) + container, err := deploy.NewJavaContainerSpec(deploy.AttachSpec{Kind: deploy.KindAttach, URL: "portforward://{{.Pod.Name}}.{{.Pod.Namespace}}:7070?kind=pod", + PID: 1, Directory: "/tmp/telltale", Bind: "127.0.0.1", Port: 7070, Service: "orders"}) + Expect(err).NotTo(HaveOccurred()) + Expect(*built).To(Equal(deploy.Deployer{TypeMeta: metav1.TypeMeta{APIVersion: definition.APIVersion, Kind: deploy.Kind}, ObjectMeta: metav1.ObjectMeta{Name: "orders"}, + Spec: deploy.Spec{Selector: deploy.Selector{Kind: deploy.WorkloadPod, Namespace: "shop", Name: "orders-0", Container: "orders"}, JavaContainer: container, + Auth: deploy.AuthSpec{Issuer: "connection://shop/telltale-signing", JWKS: &deploy.JWKSSource{Mode: deploy.JWKSPublish, Kind: "ConfigMap", Name: "telltale-jwks"}}}})) + // The built document is one a Deployer file could hold. + encoded, err := json.Marshal(built) + Expect(err).NotTo(HaveOccurred()) + decoded, err := deploy.Decode(encoded) + Expect(err).NotTo(HaveOccurred()) + Expect(decoded.Spec.Selector).To(Equal(built.Spec.Selector)) + }) + + It("selects workloads by kind and name or labels, and refuses --pod with either", func() { + flags := javaDeployFlags{install: agentinstall.Options{Destination: agentinstall.Destination{Namespace: "shop", Container: "app"}}, workloadKind: "statefulset", workload: "orders"} + built, err := flags.builtDeployer() + Expect(err).NotTo(HaveOccurred()) + Expect(built.Spec.Selector).To(Equal(deploy.Selector{Kind: deploy.WorkloadStatefulSet, Namespace: "shop", Name: "orders", Container: "app"})) + flags.install.Destination.Pod = "orders-0" + _, err = flags.builtDeployer() + Expect(err).To(MatchError(ContainSubstring("--pod selects one pod"))) + _, err = javaDeployFlags{workloadKind: "daemonset"}.builtDeployer() + Expect(err).To(MatchError(ContainSubstring("want Deployment, StatefulSet or Pod"))) + }) + + DescribeTable("refuses flags the chosen mode would ignore", + func(message string, args ...string) { + flags, set := parsedFlags(args...) + _, err := flags.mode(set) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("a hub with a built Deployer", "need no hub", "--issuer", "connection://x/y", "--hub-url", "https://hub.example"), + Entry("an SSH host with a Deployer document", "only a single attach reads these", "--deployer", "kind: Deployer", "--host", "app"), + Entry("selector flags with a Deployer document", "the Deployer document already says them", "--deployer", "kind: Deployer", "--pod", "orders-0"), + Entry("listener flags without an issuer", "which needs a target or --issuer", "--port", "8080"), + Entry("a non-Kubernetes destination for a Deployer", "--destination must be kubernetes", "--issuer", "connection://x/y", "--destination", "ssh"), + ) + + It("runs a single attach dry run under either name, printing the previewed artifact", func() { + for _, path := range [][]string{{"jvm", "deploy"}, {"agents", "install", "java"}} { + root := NewRoot("test") + var out bytes.Buffer + root.SetArgs(append(path, "--pid", "4242", "--directory", "/opt/telltale", "--dry-run")) + root.SetOut(&out) + root.SetErr(&bytes.Buffer{}) + Expect(root.ExecuteContext(context.Background())).To(Succeed(), path) + var result agentinstall.Result + Expect(json.Unmarshal(out.Bytes(), &result)).To(Succeed(), out.String()) + Expect(result).To(HaveField("DryRun", true)) + Expect(result.Jar).To(HavePrefix("/opt/telltale/agent-")) + } + }) +}) + +// changed is a flag set in which names were given on the command line. +func changed(names ...string) *pflag.FlagSet { + set := pflag.NewFlagSet("test", pflag.ContinueOnError) + for _, name := range names { + set.String(name, "", "") + Expect(set.Set(name, "given")).To(Succeed()) + } + return set +} + +var _ = Describe("a Deployer from a [kind/]name target", func() { + DescribeTable("reads kubectl-style targets", + func(target string, kind deploy.WorkloadKind, name string) { + gotKind, gotName, err := parseTarget(target) + Expect(err).NotTo(HaveOccurred()) + Expect([]string{string(gotKind), gotName}).To(Equal([]string{string(kind), name})) + }, + Entry("sts/", "sts/orders", deploy.WorkloadStatefulSet, "orders"), + Entry("statefulset/", "StatefulSet/orders", deploy.WorkloadStatefulSet, "orders"), + Entry("deploy/", "deploy/orders-web", deploy.WorkloadDeployment, "orders-web"), + Entry("deployments/", "deployments/orders-web", deploy.WorkloadDeployment, "orders-web"), + Entry("pod/", "pod/orders-0", deploy.WorkloadPod, "orders-0"), + Entry("a bare pod name", "orders-0", deploy.WorkloadPod, "orders-0"), + ) + + DescribeTable("refuses targets it cannot read", + func(target, message string) { + _, _, err := parseTarget(target) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("an unknown kind", "ds/orders", `kind "ds" is not sts, deploy or pod`), + Entry("no name", "sts/", "want [kind/]name"), + Entry("a nested name", "sts/orders/0", "want [kind/]name"), + ) + + It("defaults a StatefulSet target to JVM discovery, the default issuer, /tmp/telltale and its own name", func() { + flags := javaDeployFlags{target: "sts/orders", install: agentinstall.Options{Destination: agentinstall.Destination{Namespace: "shop"}, Service: "java"}, bind: "127.0.0.1", port: 7070} + mode, err := flags.mode(changed("namespace")) + Expect(err).NotTo(HaveOccurred()) + Expect(mode).To(Equal(modeDeployerFlags)) + flags, err = flags.withDefaults(changed("namespace")) + Expect(err).NotTo(HaveOccurred()) + built, err := flags.builtDeployer() + Expect(err).NotTo(HaveOccurred()) + container, err := deploy.NewJavaContainerSpec(deploy.AttachSpec{Kind: deploy.KindAttach, URL: "portforward://{{.Pod.Name}}.{{.Pod.Namespace}}:7070?kind=pod", + PID: 0, Directory: "/tmp/telltale", Bind: "127.0.0.1", Port: 7070, Service: "orders"}) + Expect(err).NotTo(HaveOccurred()) + Expect(*built).To(Equal(deploy.Deployer{TypeMeta: metav1.TypeMeta{APIVersion: definition.APIVersion, Kind: deploy.Kind}, ObjectMeta: metav1.ObjectMeta{Name: "orders"}, + Spec: deploy.Spec{Selector: deploy.Selector{Kind: deploy.WorkloadStatefulSet, Namespace: "shop", Name: "orders"}, JavaContainer: container, + Auth: deploy.AuthSpec{Issuer: deploy.DefaultIssuer}}})) + }) + + It("defaults a bare pod target to that pod, and keeps what was given", func() { + flags := javaDeployFlags{target: "orders-0", issuer: "connection://shop/signing", install: agentinstall.Options{PID: 7, Directory: "/opt/telltale", Service: "orders"}} + flags, err := flags.withDefaults(changed("issuer", "pid", "directory", "service")) + Expect(err).NotTo(HaveOccurred()) + Expect([]any{flags.install.Destination.Pod, flags.issuer, flags.install.PID, flags.install.Directory, flags.install.Service}). + To(Equal([]any{"orders-0", "connection://shop/signing", 7, "/opt/telltale", "orders"})) + }) + + DescribeTable("refuses a target with another way of selecting pods", + func(flags javaDeployFlags, set *pflag.FlagSet, message string) { + _, err := flags.mode(set) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("--pod", javaDeployFlags{target: "sts/orders"}, changed("pod"), "target sts/orders already selects the pods"), + Entry("--labels", javaDeployFlags{target: "sts/orders"}, changed("labels"), "target sts/orders already selects the pods"), + Entry("a Deployer document", javaDeployFlags{target: "sts/orders", deployer: "kind: Deployer"}, changed("deployer"), "the Deployer document already selects its pods"), + ) + + It("accepts the target positionally under both names, and at most one", func() { + for _, path := range [][]string{{"jvm", "deploy"}, {"agents", "install", "java"}} { + Expect(commandAtPath(path...).Args(commandAtPath(path...), []string{"sts/orders"})).To(Succeed()) + Expect(commandAtPath(path...).Args(commandAtPath(path...), []string{"sts/orders", "pod/orders-0"})).To(HaveOccurred()) + } + }) +}) diff --git a/internal/cli/jvm.go b/internal/cli/jvm.go new file mode 100644 index 0000000..119813f --- /dev/null +++ b/internal/cli/jvm.go @@ -0,0 +1,180 @@ +package cli + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/flanksource/clicky/rpc" + "github.com/flanksource/telltale/internal/jvm" + "github.com/flanksource/telltale/pkg/agentclient" + jvmops "github.com/flanksource/telltale/pkg/tracing/jvm" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// newJVMCommand is `jvm` with `call`, for operations only the live catalog +// knows, such as extension operations. The registry's operations and the +// kinds' JVM presets are placed under it by addJVMOperations and +// addKindCommands. +func newJVMCommand() *cobra.Command { + root := &cobra.Command{Use: "jvm", Short: "Inspect and operate a connected Java runtime"} + root.AddCommand(newJVMTopCommand(), newJVMCallCommand(), newJavaDeployCommand("deploy", "Install the agent into JVMs: a Deployer's pods, or one PID (same command as agents install java)", "jvm/deploy"), newJVMDeployersCommand(), newJVMSnapshotCommand()) + setDeployerResolver(agentclient.Default) + return root +} + +// addJVMOperations places every built-in operation of the Java registry at +// the command it declares, such as jvm.classes.patch at `jvm patch apply`. +func addJVMOperations(root *cobra.Command) { + for _, operation := range jvm.Operations() { + command := commandAt(root, operation.Command) + if command.RunE != nil { + panic(fmt.Sprintf("JVM operation %s is declared as the %q command, which already runs", operation.Name, command.CommandPath())) + } + bindJVMOperation(command, operation) + } +} + +// commandAt is the command at path below root, creating the groups and the +// command that do not exist yet. +func commandAt(root *cobra.Command, path []string) *cobra.Command { + command := root + for _, name := range path { + var next *cobra.Command + for _, child := range command.Commands() { + if child.Name() == name { + next = child + } + } + if next == nil { + next = &cobra.Command{Use: name, Short: name + " commands", Args: cobra.NoArgs} + command.AddCommand(next) + } + command = next + } + return command +} + +func bindJVMOperation(cmd *cobra.Command, operation jvm.Operation) { + request := jvm.Request{Operation: operation.Name} + cmd.Short = operation.Description + cmd.Long = operation.Description + "\n\nRuns the " + operation.Name + " operation." + cmd.Flags().StringVar(&request.Agent, "agent", "", agentFlagUsage) + if operation.Mutates { + cmd.Long += " It changes the runtime, so it needs --allow-mutation and an agent installed with mutations allowed." + cmd.Flags().BoolVar(&request.AllowMutation, "allow-mutation", false, "Allow this call to change the runtime") + } + params, err := bindParamFlags(cmd.Flags(), operation.Params) + if err != nil { + panic(fmt.Sprintf("JVM operation %s: %v", operation.Name, err)) + } + files := bindParamFiles(cmd, params) + cmd.RunE = func(cmd *cobra.Command, _ []string) error { + for _, file := range files { + if err := file.load(cmd); err != nil { + return err + } + } + var err error + if request.Params, err = params.params(); err != nil { + return err + } + value, err := callJVM(cmd.Context(), request) + if err != nil { + return err + } + return printJVMResult(cmd, request, value) + } +} + +// printJVMResult prints an operation's result through its typed model in any +// output format, or as the agent sent it when it has none. +func printJVMResult(cmd *cobra.Command, request jvm.Request, value json.RawMessage) error { + result, err := jvmops.Result(request.Operation, request.Params, value) + if err != nil { + return err + } + if err := printRecords(cmd, result); err != nil { + return err + } + if failed, ok := result.(interface{ Failure() error }); ok { + return failed.Failure() + } + return nil +} + +// agentFlagUsage describes --agent wherever a command names a Java agent. +const agentFlagUsage = "Java agent: its id, or a short name (service, pod, deployer, pid, or a unique prefix); omit it when only one agent is connected or deployed" + +// callJVM sends a typed command's request to its agent; internal tests +// replace it to read the request a command assembles. +var callJVM = jvm.Call + +// paramFile is a ---file flag that fills a source or binary string +// param from a local file: text for a textarea, base64 for a byte string. +type paramFile struct { + flag *pflag.Flag + param *pflag.Flag + base64 bool +} + +func bindParamFiles(cmd *cobra.Command, params paramFlags) []paramFile { + var files []paramFile + for _, param := range params.bound { + if param.schema.Type != "string" || (param.schema.Format != "byte" && param.schema.Format != "textarea") { + continue + } + name := param.flag.Name + "-file" + cmd.Flags().String(name, "", "Read --"+param.flag.Name+" from this local file") + cmd.MarkFlagsMutuallyExclusive(param.flag.Name, name) + if _, required := param.flag.Annotations[cobra.BashCompOneRequiredFlag]; required { + delete(param.flag.Annotations, cobra.BashCompOneRequiredFlag) + cmd.MarkFlagsOneRequired(param.flag.Name, name) + } + files = append(files, paramFile{flag: cmd.Flags().Lookup(name), param: param.flag, base64: param.schema.Format == "byte"}) + } + return files +} + +func (f paramFile) load(cmd *cobra.Command) error { + if !f.flag.Changed { + return nil + } + if _, remote := rpc.RequestFromContext(cmd.Context()); remote { + return fmt.Errorf("--%s reads a file on the machine running the command; send %s in the request instead", f.flag.Name, f.param.Name) + } + content, err := jvm.ReadParamFile(f.flag.Value.String(), f.base64) + if err != nil { + return fmt.Errorf("--%s: %w", f.flag.Name, err) + } + return cmd.Flags().Set(f.param.Name, content) +} + +func newJVMCallCommand() *cobra.Command { + var request jvm.Request + var params, paramsFile string + command := &cobra.Command{Use: "call", Short: "Invoke any operation from the live catalog, including extension operations", Args: cobra.NoArgs} + command.RunE = func(cmd *cobra.Command, _ []string) error { + if paramsFile != "" { + data, err := os.ReadFile(paramsFile) + if err != nil { + return err + } + params = string(data) + } + request.Params = json.RawMessage(params) + value, err := jvm.Call(cmd.Context(), request) + if err != nil { + return err + } + return printJVMResult(cmd, request, value) + } + command.Flags().StringVar(&request.Agent, "agent", "", agentFlagUsage) + command.Flags().StringVar(&request.Operation, "operation", "", "Operation from the live catalog") + command.Flags().StringVar(¶ms, "params", "{}", "JSON operation parameters") + command.Flags().StringVar(¶msFile, "params-file", "", "Read JSON operation parameters from a file") + command.MarkFlagsMutuallyExclusive("params", "params-file") + command.Flags().BoolVar(&request.AllowMutation, "allow-mutation", false, "Allow this call to change the runtime") + return command +} diff --git a/internal/cli/jvm_deploy.go b/internal/cli/jvm_deploy.go new file mode 100644 index 0000000..5246110 --- /dev/null +++ b/internal/cli/jvm_deploy.go @@ -0,0 +1,95 @@ +package cli + +import ( + "context" + "fmt" + "os" + "os/user" + "path/filepath" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/datadir" + "github.com/flanksource/telltale/pkg/deploy" + "github.com/spf13/cobra" +) + +// deployerStore keeps applied deployers in telltale's global config dir. +func deployerStore() (deploy.Store, error) { + dir, err := datadir.Config() + if err != nil { + return deploy.Store{}, err + } + return deploy.Store{Dir: filepath.Join(dir, "deployers")}, nil +} + +// deployerContext is a context the deployer's connection references, its +// issuer's and its port-forwards', resolve in. +func deployerContext(ctx context.Context) (dbcontext.Context, error) { + connections, err := telltale.OpenConnections() + if err != nil { + return dbcontext.Context{}, err + } + return connections.Attach(dbcontext.New().Wrap(ctx)), nil +} + +// operator is the principal execution tokens name: this user on this host. +func operator() (string, error) { + current, err := user.Current() + if err != nil { + return "", fmt.Errorf("token subject: %w", err) + } + host, err := os.Hostname() + if err != nil { + return "", fmt.Errorf("token subject: %w", err) + } + return current.Username + "@" + host, nil +} + +// setDeployerResolver lets every agent lookup of this process reach a stored +// deployer's targets by their id, with tokens minted per request. +func setDeployerResolver(registry *agentclient.Registry) { + registry.SetResolver(deploy.TargetPrefix, func(ctx context.Context, registry *agentclient.Registry, id string) error { + store, err := deployerStore() + if err != nil { + return err + } + subject, err := operator() + if err != nil { + return err + } + return deploy.Resolver{Store: store, Subject: subject, Context: deployerContext}.Resolve(ctx, registry, id) + }) + registry.SetLister(deploy.TargetPrefix, func(context.Context) ([]agentclient.Known, error) { + store, err := deployerStore() + if err != nil { + return nil, err + } + return store.Known() + }) +} + +func newJVMDeployersCommand() *cobra.Command { + return &cobra.Command{ + Use: "deployers", + Short: "List the deployed targets jvm commands can reach, by deployer", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + store, err := deployerStore() + if err != nil { + return err + } + deployers, err := store.List() + if err != nil { + return err + } + // An empty list, not null, when nothing was deployed yet. + targets := []deploy.TargetStatus{} + for _, deployer := range deployers { + targets = append(targets, deployer.Status.Targets...) + } + return printRecords(cmd, targets) + }, + } +} diff --git a/internal/cli/jvm_deploy_internal_test.go b/internal/cli/jvm_deploy_internal_test.go new file mode 100644 index 0000000..134afe9 --- /dev/null +++ b/internal/cli/jvm_deploy_internal_test.go @@ -0,0 +1,142 @@ +package cli + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/json" + "encoding/pem" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + + "github.com/flanksource/clicky/rpc" + "github.com/golang-jwt/jwt/v5" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/deploy" +) + +const ( + deployedIssuer = "connection://example/signing" + deployedTarget = "deployer/orders/shop/orders-0" +) + +// deployedListener is a deployed agent's listener: it admits a request only +// with a token the deployer's issuer signed for this target, and records +// whom each token names. +func deployedListener(key *rsa.PublicKey, subjects *[]string) http.Handler { + admit := func(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + raw, _ := strings.CutPrefix(r.Header.Get(agentclient.TokenHeader), "Bearer ") + claims := jwt.MapClaims{} + if _, err := jwt.ParseWithClaims(raw, claims, func(*jwt.Token) (any, error) { return key, nil }, jwt.WithValidMethods([]string{"RS256"}), + jwt.WithAudience(deployedTarget), jwt.WithIssuer(deployedIssuer), jwt.WithExpirationRequired()); err != nil || claims["scope"] != "jvm:read" { + w.WriteHeader(http.StatusUnauthorized) + return + } + *subjects = append(*subjects, claims["sub"].(string)) + next(w, r) + } + } + mux := http.NewServeMux() + mux.HandleFunc("GET /hello", admit(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{"apiVersion": agentclient.AgentAPIVersion, "protocol": 1, "started": "2026-01-02T03:04:05Z", "runtime": map[string]string{"lang": "java"}, + "ops": []map[string]any{{"name": "jvm.status", "mutates": false, "scope": "jvm:read"}}}) + })) + mux.HandleFunc("POST /call", admit(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"allowMutation":false,"instrumentation":true,"patches":[]}`) + })) + return mux +} + +var _ = Describe("jvm commands on deployed targets", func() { + It("reach a stored deployer's pod by its target id, signing each request with the deployer's issuer", func() { + dir := GinkgoT().TempDir() + GinkgoT().Chdir(dir) + GinkgoT().Setenv("XDG_CONFIG_HOME", filepath.Join(dir, "config")) + DeferCleanup(func(previous *agentclient.Registry) { agentclient.Default = previous }, agentclient.Default) + agentclient.Default = agentclient.NewRegistry() + + key, err := rsa.GenerateKey(rand.Reader, 2048) + Expect(err).NotTo(HaveOccurred()) + der, err := x509.MarshalPKCS8PrivateKey(key) + Expect(err).NotTo(HaveOccurred()) + connection, err := json.Marshal(map[string]any{"apiVersion": "mission-control.flanksource.com/v1", "kind": "Connection", "metadata": map[string]string{"name": "signing", "namespace": "example"}, + "spec": map[string]any{"http": map[string]string{"url": "http://localhost"}, "secrets": map[string]any{"signingKey": map[string]string{"value": string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}))}}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(os.MkdirAll(filepath.Join(dir, ".connections"), 0o700)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(dir, ".connections", "signing.json"), connection, 0o600)).To(Succeed()) + + var subjects []string + listener := httptest.NewServer(deployedListener(&key.PublicKey, &subjects)) + DeferCleanup(listener.Close) + deployer, err := deploy.Decode([]byte(` +apiVersion: telltale.flanksource.com/v1alpha1 +kind: Deployer +metadata: {name: orders} +spec: + selector: {kind: Deployment, namespace: shop, labels: {app: orders}, container: app} + javaContainer: {kind: attach, url: 'http://{{.Pod.IP}}:7070', pid: 1, directory: /tmp/telltale, bind: 0.0.0.0, port: 7070, service: orders} + auth: {issuer: ` + deployedIssuer + `} +`)) + Expect(err).NotTo(HaveOccurred()) + deployer.Status = deploy.Status{Targets: []deploy.TargetStatus{{Target: deployedTarget, Namespace: "shop", Pod: "orders-0", Container: "app", State: deploy.StateDeployed, Endpoint: listener.URL}}} + store, err := deployerStore() + Expect(err).NotTo(HaveOccurred()) + Expect(store.Save(*deployer)).To(Succeed()) + + root := NewRoot("test") + var out bytes.Buffer + root.SetArgs([]string{"jvm", "status", "--agent", deployedTarget}) + root.SetOut(&out) + root.SetErr(&bytes.Buffer{}) + Expect(root.ExecuteContext(context.Background())).To(Succeed()) + Expect(out.String()).To(Equal("mutations denied · instrumentation on · 0 patches\n")) + subject, err := operator() + Expect(err).NotTo(HaveOccurred()) + Expect(subjects).To(Equal([]string{subject, subject})) + + root = NewRoot("test") + out.Reset() + root.SetArgs([]string{"jvm", "deployers", "--format", "json"}) + root.SetOut(&out) + Expect(root.ExecuteContext(context.Background())).To(Succeed()) + var listed []deploy.TargetStatus + Expect(json.Unmarshal(out.Bytes(), &listed)).To(Succeed(), out.String()) + Expect(listed).To(Equal(deployer.Status.Targets)) + }) + + It("refuses a target no deployer deployed, naming the deploy command", func() { + GinkgoT().Setenv("XDG_CONFIG_HOME", GinkgoT().TempDir()) + DeferCleanup(func(previous *agentclient.Registry) { agentclient.Default = previous }, agentclient.Default) + agentclient.Default = agentclient.NewRegistry() + Expect(executeJVM(context.Background(), "metrics", "--agent", "deployer/missing/shop/orders-0")).To(MatchError(ContainSubstring("telltale jvm deploy -f"))) + }) + + It("serves deploy over RPC with the deployer document and a dry run as its inputs", func() { + spec := kindOpenAPI(NewRoot("test")) + Expect(spec.Paths).To(HaveKey("/api/v1/jvm/deploy")) + body := spec.Paths["/api/v1/jvm/deploy"]["post"].RequestBody.Content["application/json"].Schema.Properties + Expect(body).To(And(HaveKeyWithValue("deployer", HaveField("Type", "string")), HaveKeyWithValue("dry-run", HaveField("Type", "boolean")))) + }) + + It("refuses to read a deployer file for a request that arrived over HTTP", func() { + file := filepath.Join(GinkgoT().TempDir(), "deployer.yaml") + Expect(os.WriteFile(file, []byte("kind: Deployer"), 0o600)).To(Succeed()) + remote := rpc.ContextWithRequest(context.Background(), httptest.NewRequest(http.MethodPost, "/api/v1/jvm/deploy", nil)) + Expect(executeJVM(remote, "deploy", "--dry-run", "-f", file)).To(MatchError(ContainSubstring("send deployer in the request instead"))) + }) + + It("takes at most one of a deployer file and a deployer document, and without either attaches to one PID", func() { + Expect(executeJVM(context.Background(), "deploy", "-f", "x.yaml", "--deployer", "kind: Deployer")).To(MatchError(ContainSubstring("none of the others"))) + Expect(executeJVM(context.Background(), "deploy", "--dry-run")).To(MatchError(ContainSubstring("a positive target JVM pid is required"))) + }) +}) diff --git a/internal/cli/jvm_internal_test.go b/internal/cli/jvm_internal_test.go new file mode 100644 index 0000000..c40c233 --- /dev/null +++ b/internal/cli/jvm_internal_test.go @@ -0,0 +1,320 @@ +package cli + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing/fstest" + "time" + + "github.com/flanksource/clicky/rpc" + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/internal/jvm" + "github.com/flanksource/telltale/internal/server" + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/traces" + "github.com/flanksource/telltale/sdk/protocol" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/spf13/cobra" +) + +// jvmCommand is the command at a path of the CLI, such as [jvm patch apply]. +func jvmCommand(path ...string) *cobra.Command { + command, args, err := NewRoot("test").Find(path) + Expect(err).NotTo(HaveOccurred(), strings.Join(path, " ")) + Expect(args).To(BeEmpty(), strings.Join(path, " ")) + return command +} + +// executeJVM runs `telltale jvm ` against a recording agent. +func executeJVM(ctx context.Context, args ...string) error { + root := NewRoot("test") + root.SetArgs(append([]string{"jvm"}, args...)) + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SilenceUsage = true + return root.ExecuteContext(ctx) +} + +// sentJVMRequest runs a generated command against a recording agent and +// returns the request it sent. +func sentJVMRequest(ctx context.Context, args ...string) (jvm.Request, error) { + var sent jvm.Request + DeferCleanup(func(previous func(context.Context, jvm.Request) (json.RawMessage, error)) { callJVM = previous }, callJVM) + callJVM = func(_ context.Context, request jvm.Request) (json.RawMessage, error) { + sent = request + return nil, errNotCalled + } + if err := executeJVM(ctx, args...); !errors.Is(err, errNotCalled) { + return sent, err + } + return sent, nil +} + +// errNotCalled stops a command at the agent call, once its request is assembled. +var errNotCalled = errors.New("agent not called by the test") + +var errNotStarted = errors.New("run not started by the test") + +// startedTrace runs a kind preset and returns the trace spec and duration it +// would start, without starting a run. +func startedTrace(ctx context.Context, args ...string) (telltale.TraceSpec, time.Duration, error) { + registry := agentclient.NewRegistry() + Expect(registry.Add(&agentclient.Agent{Hello: protocol.Hello{AgentID: "a", Runtime: protocol.Runtime{Lang: "java"}, + Probes: []protocol.ProbeSpec{{Name: "jvm_trace"}, {Name: "jvm_jfr"}}}})).To(Succeed()) + // Only the agent added here: no local agents of this machine are discovered. + registry.Dir = GinkgoT().TempDir() + DeferCleanup(func(previous *agentclient.Registry) { agentclient.Default = previous }, agentclient.Default) + agentclient.Default = registry + var started telltale.StartOptions + DeferCleanup(func(previous func(context.Context, telltale.StartOptions) (*telltale.Run, error)) { + startKindRun = previous + }, startKindRun) + startKindRun = func(_ context.Context, options telltale.StartOptions) (*telltale.Run, error) { + started = options + return nil, errNotStarted + } + err := executeJVM(ctx, args...) + if len(started.Traces) != 1 { + return telltale.TraceSpec{}, 0, err + } + return started.Traces[0], started.Duration, err +} + +var _ = Describe("JVM command tree", func() { + It("serves each operation and kind preset at its declared command", func() { + served := map[string]string{} + for _, operation := range jvm.Operations() { + served[strings.Join(operation.Command, " ")] = operation.Name + } + for _, registration := range traces.Registrations() { + for _, preset := range registration.Commands { + served[strings.Join(preset.Path, " ")] = registration.Name + } + } + Expect(served).To(Equal(map[string]string{ + "jvm settings": "jvm.settings", "jvm resources": "jvm.resources", "jvm flags": "jvm.flags", + "jvm decompile": "jvm.classes.source", "jvm decompile bytecode": "jvm.classes.bytecode", + "jvm patch apply": "jvm.classes.patch", "jvm patch restore": "jvm.classes.restore", "jvm patch list": "jvm.classes.patches", + "jvm patch compile": "jvm.classes.compile", "jvm patch lease": "jvm.classes.lease", + "jvm watch": "jvm_trace", "jvm trace": "jvm_trace", "jvm profiler": "jvm_jfr", + "jvm trace instrument": "jvm.classes.instrument", "jvm trace probes": "jvm.probes.list", + "jvm trace read": "jvm.probes.read", "jvm trace freeze": "jvm.probes.freeze", + "jvm classes": "jvm.classes", "jvm loaders": "jvm.loaders", "jvm catalog": "jvm.catalog", "jvm status": "jvm.status", + "jvm jmx search": "jvm.jmx.search", "jvm jmx info": "jvm.jmx.info", "jvm jmx read": "jvm.jmx.read", + "jvm jmx write": "jvm.jmx.write", "jvm jmx invoke": "jvm.jmx.invoke", "jvm metrics": "jvm.metrics", + "jvm telemetry": "jvm.telemetry", + "jvm threads": "jvm.threads", "jvm threads dump": "jvm.threads.dump", "jvm threads stack": "jvm.threads.stack", "jvm heap histogram": "jvm.heap.histogram", + "jvm logging list": "jvm.logging.list", "jvm logging set": "jvm.logging.set", + "jvm groovy eval": "jvm.groovy.eval", "jvm ognl eval": "jvm.ognl.eval", + "jvm extensions list": "jvm.extensions.list", "jvm extensions install": "jvm.extensions.install", + "jvm extensions replace": "jvm.extensions.replace", "jvm extensions unload": "jvm.extensions.unload", + "jvm notifications list": "jvm.notifications.list", + })) + for command := range served { + Expect(jvmCommand(strings.Fields(command)...).RunE).NotTo(BeNil(), command) + } + }) + + It("keeps a runnable preset beside the operations declared below it", func() { + trace := jvmCommand("jvm", "trace") + Expect(trace.Flags().Lookup("targets")).NotTo(BeNil()) + var children []string + for _, child := range trace.Commands() { + children = append(children, child.Name()) + } + Expect(children).To(ConsistOf("instrument", "probes", "read", "freeze")) + for _, removed := range [][]string{{"jvm", "classes", "source"}, {"jvm", "classes", "patch"}, {"jvm", "probes"}} { + _, args, err := NewRoot("test").Find(removed) + Expect(err != nil || len(args) > 0).To(BeTrue(), strings.Join(removed, " ")) + } + }) + + It("starts the preset's kind with its defaults under the flags given", func() { + cases := []struct { + args []string + kind string + expected string + }{ + {[]string{"watch", "--agent", "a", "--targets", `[{"class":"Idle","method":"main"}]`, "--duration", "3s"}, "jvm_trace", + `{"args":true,"return":true,"exception":true,"calls":false,"targets":[{"class":"Idle","method":"main"}],"target":{"agent":"a"}}`}, + {[]string{"trace", "--agent", "a", "--targets", `[{"class":"Idle","method":"main"}]`, "--duration", "3s"}, "jvm_trace", + `{"args":true,"return":true,"exception":true,"calls":true,"targets":[{"class":"Idle","method":"main"}],"target":{"agent":"a"}}`}, + {[]string{"trace", "--agent", "a", "--targets", `[{"class":"Idle","method":"main"}]`, "--calls=false", "--args=false", "--duration", "3s"}, "jvm_trace", + `{"args":false,"return":true,"exception":true,"calls":false,"targets":[{"class":"Idle","method":"main"}],"target":{"agent":"a"}}`}, + {[]string{"profiler", "--agent", "a", "--duration", "3s"}, "jvm_jfr", `{"target":{"seam":"agent","agent":"a"}}`}, + } + for _, c := range cases { + spec, duration, err := startedTrace(context.Background(), c.args...) + Expect(err).To(MatchError(errNotStarted), strings.Join(c.args, " ")) + Expect(duration).To(Equal(3*time.Second), strings.Join(c.args, " ")) + Expect(spec.Kind).To(Equal(c.kind), strings.Join(c.args, " ")) + Expect(spec.Params).To(MatchJSON(c.expected), strings.Join(c.args, " ")) + } + }) + + It("names the agent a capture runs on by its id, from a short name or from the only agent offering the kind", func() { + GinkgoT().Setenv("XDG_CONFIG_HOME", GinkgoT().TempDir()) + cases := []struct { + args []string + expected string + }{ + {[]string{"watch", "--targets", `[{"class":"Idle","method":"main"}]`, "--duration", "3s"}, `{"args":true,"return":true,"exception":true,"calls":false,"targets":[{"class":"Idle","method":"main"}],"target":{"agent":"a"}}`}, + {[]string{"profiler", "--duration", "3s"}, `{"target":{"seam":"agent","agent":"a"}}`}, + } + for _, c := range cases { + spec, _, err := startedTrace(context.Background(), c.args...) + Expect(err).To(MatchError(errNotStarted), strings.Join(c.args, " ")) + Expect(spec.Params).To(MatchJSON(c.expected), strings.Join(c.args, " ")) + } + _, _, err := startedTrace(context.Background(), "watch", "--agent", "billing", "--targets", `[{"class":"Idle","method":"main"}]`, "--duration", "3s") + Expect(err).To(MatchError(`no agent offering jvm_trace matches "billing"; agents: a ()`)) + }) + + It("sends a preset capture requested over HTTP to the server's run API", func() { + remote := rpc.ContextWithRequest(context.Background(), httptest.NewRequest("POST", "/api/v1/jvm/watch", nil)) + _, _, err := startedTrace(remote, "watch", "--agent", "a", "--targets", `[{"class":"Idle","method":"main"}]`, "--duration", "3s") + Expect(err).To(MatchError("telltale jvm watch records a run: start it over HTTP with POST /api/traces and kind jvm_trace")) + }) +}) + +var _ = Describe("typed JVM operation commands", func() { + It("generates a command with a flag per param for every built-in operation", func() { + for _, operation := range jvm.Operations() { + command := jvmCommand(operation.Command...) + Expect(command.RunE).NotTo(BeNil(), operation.Name) + Expect(command.Short).To(Equal(operation.Description)) + Expect(command.Flags().Lookup("agent")).NotTo(BeNil(), operation.Name) + Expect(command.Flags().Lookup("allow-mutation") != nil).To(Equal(operation.Mutates), operation.Name) + for name := range operation.Params.Properties { + Expect(command.Flags().Lookup(flagName([]string{name}))).NotTo(BeNil(), operation.Name+" "+name) + } + } + }) + + It("keeps an operation that is also a group runnable beside its subcommands", func() { + decompile := jvmCommand("jvm", "decompile") + Expect(decompile.Flags().Lookup("class")).NotTo(BeNil()) + Expect(jvmCommand("jvm", "decompile", "bytecode").Short).To(Equal("Read a class's bytecode as base64")) + }) + + It("reaches a reorganized operation through its new command", func() { + request, err := sentJVMRequest(context.Background(), "patch", "list", "--agent", "a") + Expect(err).NotTo(HaveOccurred()) + Expect(request.Operation).To(Equal("jvm.classes.patches")) + request, err = sentJVMRequest(context.Background(), "trace", "read", "--agent", "a", "--key", "k", "--limit", "5") + Expect(err).NotTo(HaveOccurred()) + Expect(request.Operation).To(Equal("jvm.probes.read")) + Expect(request.Params).To(MatchJSON(`{"key":"k","limit":5}`)) + }) + + It("sends typed params as the operation's JSON params", func() { + request, err := sentJVMRequest(context.Background(), "jmx", "write", "--agent", "java/demo", "--allow-mutation", + "--object-name", "example:type=Cache", "--attribute", "Size", "--value", `{"limit":9007199254740993}`) + Expect(err).NotTo(HaveOccurred()) + Expect(request.Agent).To(Equal("java/demo")) + Expect(request.Operation).To(Equal("jvm.jmx.write")) + Expect(request.AllowMutation).To(BeTrue()) + Expect(request.Params).To(MatchJSON(`{"objectName":"example:type=Cache","attribute":"Size","value":{"limit":9007199254740993}}`)) + }) + + It("sends typed lists, integers and an explicit null", func() { + request, err := sentJVMRequest(context.Background(), "jmx", "invoke", "--agent", "a", "--object-name", "o", "--operation", "op", + "--args", `[1,"two"]`, "--signature", "long,java.lang.String") + Expect(err).NotTo(HaveOccurred()) + Expect(request.Params).To(MatchJSON(`{"objectName":"o","operation":"op","args":[1,"two"],"signature":["long","java.lang.String"]}`)) + request, err = sentJVMRequest(context.Background(), "threads", "stack", "--agent", "a", "--id", "7") + Expect(err).NotTo(HaveOccurred()) + Expect(request.Params).To(MatchJSON(`{"id":7}`)) + request, err = sentJVMRequest(context.Background(), "logging", "set", "--agent", "a", "--name", "", "--level", "") + Expect(err).NotTo(HaveOccurred()) + Expect(request.Params).To(MatchJSON(`{"name":"","level":null}`)) + }) + + It("leaves an omitted --agent to the call, which picks the only Java agent", func() { + request, err := sentJVMRequest(context.Background(), "jmx", "search", "--pattern", "java.lang:*") + Expect(err).NotTo(HaveOccurred()) + Expect(request.Agent).To(BeEmpty()) + }) + + DescribeTable("refuses invalid input before calling the agent", func(message string, args ...string) { + _, err := sentJVMRequest(context.Background(), args...) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("a missing required param", "object-name", "jmx", "info", "--agent", "a"), + Entry("a non-array list", "expected a JSON array", "jmx", "invoke", "--agent", "a", "--object-name", "o", "--operation", "op", "--args", `{}`), + Entry("a file and an inline value together", "none of the others", "groovy", "eval", "--agent", "a", "--source", "1", "--source-file", "x.groovy"), + Entry("neither a value nor its file", "at least one of the flags", "extensions", "install", "--agent", "a"), + ) + + It("reads source as text and a JAR as base64 from local files", func() { + dir := GinkgoT().TempDir() + script, jar := filepath.Join(dir, "inspect.groovy"), filepath.Join(dir, "example.jar") + Expect(os.WriteFile(script, []byte("return args.id"), 0o600)).To(Succeed()) + Expect(os.WriteFile(jar, []byte{0x50, 0x4b, 0x03, 0x04}, 0o600)).To(Succeed()) + request, err := sentJVMRequest(context.Background(), "groovy", "eval", "--agent", "a", "--allow-mutation", "--source-file", script, "--args", `{"id":9007199254740993}`) + Expect(err).NotTo(HaveOccurred()) + Expect(request.Params).To(MatchJSON(`{"source":"return args.id","args":{"id":9007199254740993}}`)) + request, err = sentJVMRequest(context.Background(), "extensions", "install", "--agent", "a", "--allow-mutation", "--jar-file", jar) + Expect(err).NotTo(HaveOccurred()) + Expect(request.Params).To(MatchJSON(`{"jar":"` + base64.StdEncoding.EncodeToString([]byte{0x50, 0x4b, 0x03, 0x04}) + `"}`)) + }) + + It("prints a failed compile's diagnostics and exits non-zero", func() { + DeferCleanup(func(previous func(context.Context, jvm.Request) (json.RawMessage, error)) { callJVM = previous }, callJVM) + callJVM = func(context.Context, jvm.Request) (json.RawMessage, error) { + return json.RawMessage(`{"success":false,"classes":[],"diagnostics":[{"kind":"ERROR","line":3,"column":57,"code":"compiler.err.expected","message":"';' expected"}],"classpathEntries":2}`), nil + } + root := NewRoot("test") + var out bytes.Buffer + root.SetArgs([]string{"jvm", "patch", "compile", "--agent", "a", "--class", "Idle", "--source", "static void check(long l) {}"}) + root.SetOut(&out) + root.SetErr(&bytes.Buffer{}) + Expect(root.ExecuteContext(context.Background())).To(MatchError("compile failed with 1 error")) + Expect(out.String()).To(Equal("✗ compile failed · 1 error · 2 classpath entries\nERROR 3:57 ';' expected (compiler.err.expected)\n")) + }) + + It("executes typed operations over HTTP and refuses to read server files for them", func() { + var sent jvm.Request + DeferCleanup(func(previous func(context.Context, jvm.Request) (json.RawMessage, error)) { callJVM = previous }, callJVM) + callJVM = func(_ context.Context, request jvm.Request) (json.RawMessage, error) { + sent = request + return json.RawMessage(`["java.lang:type=Memory"]`), nil + } + handler, err := server.NewHandler(server.HandlerOptions{ + Root: NewRoot("test"), Assets: fstest.MapFS{"index.html": &fstest.MapFile{Data: []byte("ok")}}, + Traces: http.NotFoundHandler(), Top: http.NotFoundHandler(), + }) + Expect(err).NotTo(HaveOccurred()) + post := func(path, body string) *httptest.ResponseRecorder { + request := httptest.NewRequest(http.MethodPost, path, strings.NewReader(body)) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Accept", "application/json") + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + return response + } + response := post("/api/v1/jvm/jmx/search", `{"agent":"java/demo","pattern":"java.lang:*"}`) + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + Expect(sent.Operation).To(Equal("jvm.jmx.search")) + Expect(sent.Params).To(MatchJSON(`{"pattern":"java.lang:*"}`)) + response = post("/api/v1/jvm/extensions/install", `{"agent":"java/demo","allow-mutation":true,"jar-file":"/etc/hosts"}`) + Expect(response.Body.String()).To(ContainSubstring("send jar in the request instead")) + }) + + It("refuses to read a local file for a request that arrived over HTTP", func() { + jar := filepath.Join(GinkgoT().TempDir(), "example.jar") + Expect(os.WriteFile(jar, []byte("jar"), 0o600)).To(Succeed()) + remote := rpc.ContextWithRequest(context.Background(), httptest.NewRequest("POST", "/api/v1/jvm/extensions/install", nil)) + _, err := sentJVMRequest(remote, "extensions", "install", "--agent", "a", "--jar-file", jar) + Expect(err).To(MatchError(ContainSubstring("send jar in the request instead"))) + }) +}) diff --git a/internal/cli/jvm_snapshot.go b/internal/cli/jvm_snapshot.go new file mode 100644 index 0000000..6f7fe63 --- /dev/null +++ b/internal/cli/jvm_snapshot.go @@ -0,0 +1,147 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "time" + + "github.com/flanksource/clicky/rpc" + + "github.com/flanksource/telltale/internal/jvm/snapshot" + "github.com/spf13/cobra" +) + +func newJVMSnapshotCommand() *cobra.Command { + var options snapshot.Options + var pid int + var output, javaHome, server string + command := &cobra.Command{Use: "snapshot", Short: "Build a redacted JVM support ZIP", Args: cobra.NoArgs} + command.RunE = func(cmd *cobra.Command, _ []string) error { + if _, remote := rpc.RequestFromContext(cmd.Context()); remote { + return errors.New("use POST /api/v1/jvm/snapshot to download a snapshot") + } + if err := options.Validate(); err != nil { + return err + } + // The collectors and a JFR recording take up to 7m; workload sampling runs after them. + ctx, cancel := context.WithTimeout(cmd.Context(), 12*time.Minute) + defer cancel() + if output == "" { + identifier := strconv.Itoa(pid) + if options.Agent != "" { + identifier = strings.Map(func(r rune) rune { + if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' { + return r + } + return '-' + }, options.Agent) + } + output = fmt.Sprintf("telltale-jvm-%s-%s.zip", identifier, time.Now().UTC().Format("20060102T150405.000000000Z")) + } + file, err := os.OpenFile(output, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + saved := false + defer func() { + _ = file.Close() + if !saved { + _ = os.Remove(output) + } + }() + var data []byte + var complete bool + if server != "" { + if pid != 0 || options.Agent == "" { + return errors.New("--server requires --agent and cannot use --pid") + } + data, complete, err = remoteSnapshot(ctx, server, options) + } else { + options.Agent, err = snapshot.Resolve(ctx, options.Agent, pid, javaHome) + if err == nil { + var result *snapshot.Result + result, err = snapshot.Collect(ctx, options) + if err == nil { + data, err = result.Zip() + complete = result.Manifest.Complete + } + } + } + if err != nil { + return err + } + if _, err = file.Write(data); err != nil { + return err + } + if err = file.Close(); err != nil { + return err + } + saved = true + if _, err = fmt.Fprintln(cmd.OutOrStdout(), output); err != nil { + return err + } + if !complete { + return fmt.Errorf("partial support bundle saved to %s; see summary.md for collection failures", output) + } + return nil + } + flags := command.Flags() + flags.StringVar(&options.Agent, "agent", "", "Connected Java agent ID") + flags.IntVar(&pid, "pid", 0, "Local JVM PID; attaches the agent if needed") + flags.StringVar(&javaHome, "java-home", "", "Target-compatible JDK for attachment") + flags.StringVar(&server, "server", "", "Telltale hub URL for --agent") + flags.StringVarP(&output, "output", "o", "", "Destination ZIP (must not exist)") + flags.StringVar(&options.JFR, "jfr", "", "Include a bounded JFR recording, e.g. 30s (maximum 5m)") + flags.BoolVar(&options.HeapHistogram, "heap-histogram", false, "Include all-object heap histogram; may pause the JVM") + flags.StringVar(&options.Workload, "workload", "1m", "Sample CPU, memory, GC and threads every 5s for this long into workload.html (5s to 5m; empty disables)") + command.MarkFlagsOneRequired("agent", "pid") + command.MarkFlagsMutuallyExclusive("agent", "pid") + return command +} + +func remoteSnapshot(ctx context.Context, server string, options snapshot.Options) (data []byte, complete bool, err error) { + address, err := url.Parse(server) + if err != nil || address.Host == "" || (address.Scheme != "http" && address.Scheme != "https") || address.User != nil || address.RawQuery != "" || address.Fragment != "" { + return nil, false, fmt.Errorf("invalid server URL") + } + body, err := json.Marshal(options) + if err != nil { + return nil, false, err + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(server, "/")+"/api/v1/jvm/snapshot", bytes.NewReader(body)) + if err != nil { + return nil, false, err + } + request.Header.Set("Content-Type", "application/json") + if token := os.Getenv("TELLTALE_API_TOKEN"); token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + response, err := http.DefaultClient.Do(request) + if err != nil { + return nil, false, err + } + defer func() { err = errors.Join(err, response.Body.Close()) }() + if response.StatusCode != http.StatusOK { + return nil, false, fmt.Errorf("snapshot HTTP %d", response.StatusCode) + } + if response.Header.Get("Content-Type") != "application/zip" { + return nil, false, fmt.Errorf("snapshot returned unexpected content type %q", response.Header.Get("Content-Type")) + } + if status := response.Header.Get("X-Telltale-Snapshot-Status"); status != "complete" && status != "partial" { + return nil, false, fmt.Errorf("snapshot returned invalid completion status %q", status) + } + data, err = io.ReadAll(io.LimitReader(response.Body, 128<<20+1)) + if len(data) > 128<<20 { + return nil, false, fmt.Errorf("snapshot exceeds 128 MiB") + } + return data, response.Header.Get("X-Telltale-Snapshot-Status") == "complete", err +} diff --git a/internal/cli/jvm_top.go b/internal/cli/jvm_top.go new file mode 100644 index 0000000..daeb616 --- /dev/null +++ b/internal/cli/jvm_top.go @@ -0,0 +1,62 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "time" + + tea "charm.land/bubbletea/v2" + "github.com/flanksource/clicky/rpc" + "github.com/flanksource/telltale/internal/jvm/top" + "github.com/flanksource/telltale/internal/jvm/top/tui" + "github.com/flanksource/telltale/internal/shutdown" + "github.com/spf13/cobra" +) + +func newJVMTopCommand() *cobra.Command { + var agent, server string + every := 2 * time.Second + var once bool + cmd := &cobra.Command{Use: "top", Short: "Monitor one JVM's CPU, memory, GC and busiest threads", Args: cobra.NoArgs} + cmd.RunE = func(cmd *cobra.Command, _ []string) error { + ctx, cancel := context.WithCancel(cmd.Context()) + defer cancel() + source := &top.Source{Agent: agent, Server: server} + if every < time.Second { + return fmt.Errorf("--every must be at least 1s") + } + _, remote := rpc.RequestFromContext(cmd.Context()) + if remote && source.Server != "" { + return fmt.Errorf("--server is only supported by the local CLI") + } + if remote || once || !interactive() { + frame, err := source.Next(cmd.Context()) + if err != nil { + return err + } + if remote { + return json.NewEncoder(cmd.OutOrStdout()).Encode(frame) + } + return printRecords(cmd, frame) + } + restore, err := shutdown.Terminal(cmd.Context()) + if err != nil { + return err + } + defer restore() + final, err := tea.NewProgram(tui.New(ctx, source, every), tea.WithContext(ctx)).Run() + if err != nil { + return err + } + if model, ok := final.(tui.Model); ok { + return model.Err() + } + return nil + } + cmd.Flags().StringVar(&agent, "agent", "", agentFlagUsage) + cmd.Flags().StringVar(&server, "server", "", "Telltale hub URL (uses TELLTALE_API_TOKEN)") + cmd.Flags().DurationVar(&every, "every", every, "Refresh interval (at least 1s)") + cmd.Flags().BoolVar(&once, "once", false, "Print one snapshot and exit") + return cmd +} diff --git a/internal/cli/kinddefaults.go b/internal/cli/kinddefaults.go index 67d6acb..ee34271 100644 --- a/internal/cli/kinddefaults.go +++ b/internal/cli/kinddefaults.go @@ -63,10 +63,39 @@ func bindJSONParam(flags *pflag.FlagSet, schema *rpc.OpenAPISchema, name string) } initial = string(raw) } - value := flags.String(name, initial, paramUsage(schema)+" (JSON object)") + value := flags.String(name, initial, paramUsage(schema)+" (JSON "+describeJSON(schema)+")") return func() any { return *value }, nil } +// anyJSON is a param whose values the schema leaves open: an untyped value, +// or a list of them. +func anyJSON(schema *rpc.OpenAPISchema) bool { + return schema.Type == "" || (schema.Type == "array" && (schema.Items == nil || schema.Items.Type == "")) +} + +func describeJSON(schema *rpc.OpenAPISchema) string { + if schema.Type == "" { + return "value" + } + return schema.Type +} + +func readJSONValue(raw, flag string, list bool) (any, error) { + decoder := json.NewDecoder(strings.NewReader(raw)) + decoder.UseNumber() + var value any + if err := decoder.Decode(&value); err != nil { + return nil, fmt.Errorf("--%s: expected JSON: %w", flag, err) + } + if decoder.More() { + return nil, fmt.Errorf("--%s: expected one JSON value", flag) + } + if _, ok := value.([]any); list && !ok { + return nil, fmt.Errorf("--%s: expected a JSON array", flag) + } + return value, nil +} + func readJSONParam(raw, flag string) (map[string]json.RawMessage, error) { var value map[string]json.RawMessage if err := json.Unmarshal([]byte(raw), &value); err != nil { diff --git a/internal/cli/kindflags.go b/internal/cli/kindflags.go index 33ac660..87c30af 100644 --- a/internal/cli/kindflags.go +++ b/internal/cli/kindflags.go @@ -127,6 +127,24 @@ func bindParam(flags *pflag.FlagSet, schema *rpc.OpenAPISchema, path []string) ( param.flag = flags.Lookup(name) flagmeta.SetCategory(param.flag, paramsCategory) flagmeta.SetAllowedValues(param.flag, paramChoices(schema)...) + encoded, err := json.Marshal(schema) + if err != nil { + return paramFlag{}, err + } + var metadata map[string]any + if err := json.Unmarshal(encoded, &metadata); err != nil { + return paramFlag{}, err + } + if schema.Type == "object" || anyJSON(schema) || (schema.Type == "array" && schema.Items != nil && schema.Items.Type == "object") { + for key := range metadata { + if key != "title" && key != "description" && !strings.HasPrefix(key, "x-") { + delete(metadata, key) + } + } + } + if err := flagmeta.SetSchema(param.flag, metadata); err != nil { + return paramFlag{}, err + } if schema.Default != nil { if _, err := param.read(); err != nil { return paramFlag{}, err @@ -148,10 +166,14 @@ func bindParamValue(flags *pflag.FlagSet, schema *rpc.OpenAPISchema, name string return bindTypedParam(schema, func(value float64) *float64 { return flags.Float64(name, value, usage) }) case schema.Type == "array" && schema.Items != nil && schema.Items.Type == "string": return bindTypedParam(schema, func(value []string) *[]string { return flags.StringSlice(name, value, usage) }) + case schema.Type == "array" && schema.Items != nil && schema.Items.Type == "object": + return bindJSONParam(flags, schema, name) case schema.Type == "object" && schema.AdditionalProperties != nil && schema.AdditionalProperties.Type == "string": return bindMapParam(flags, schema, name) case schema.Type == "object" && (schema.AdditionalProperties != nil || schema.Format == "json"): return bindJSONParam(flags, schema, name) + case anyJSON(schema): + return bindJSONParam(flags, schema, name) default: return nil, fmt.Errorf("a %s param cannot be a flag", describeType(schema)) } @@ -280,6 +302,22 @@ func (p paramFlags) params() (json.RawMessage, error) { func (p paramFlag) read() (any, error) { value := p.value() + if p.schema.Type == "array" && p.schema.Items != nil && p.schema.Items.Type == "object" { + var rows []map[string]json.RawMessage + if err := json.Unmarshal([]byte(value.(string)), &rows); err != nil { + return nil, fmt.Errorf("--%s: expected a JSON array of objects: %w", p.flag.Name, err) + } + if rows == nil || slices.ContainsFunc(rows, func(row map[string]json.RawMessage) bool { return row == nil }) { + return nil, fmt.Errorf("--%s: expected a JSON array of objects", p.flag.Name) + } + return rows, nil + } + if anyJSON(p.schema) { + return readJSONValue(value.(string), p.flag.Name, p.schema.Type == "array") + } + if p.schema.Type == "string" && p.schema.Nullable && value == "" { + return nil, nil + } if p.schema.Type == "object" && (p.schema.AdditionalProperties != nil || p.schema.Format == "json") { if p.schema.AdditionalProperties != nil && p.schema.AdditionalProperties.Type == "string" { return readMapParam(value.([]string), p.flag.Name) diff --git a/internal/cli/kindflags_internal_test.go b/internal/cli/kindflags_internal_test.go index 81f6846..21b4548 100644 --- a/internal/cli/kindflags_internal_test.go +++ b/internal/cli/kindflags_internal_test.go @@ -5,7 +5,7 @@ import ( "github.com/flanksource/telltale/pkg/traces" "github.com/flanksource/telltale/pkg/tracing/goprof" "github.com/flanksource/telltale/pkg/tracing/httpproxy" - "github.com/flanksource/telltale/pkg/tracing/process" + "github.com/flanksource/telltale/pkg/tracing/processcapture" "github.com/flanksource/telltale/pkg/tracing/target" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -34,6 +34,22 @@ func parsed(schema *rpc.OpenAPISchema, args ...string) (string, error) { } var _ = Describe("a kind's params as flags", func() { + It("reads method targets from a JSON array without losing their loader", func() { + params, err := parsed(kindSchema("jvm_trace"), "--targets", `[{"class":"example.Worker","method":"compute","loader":"4ac"}]`) + Expect(err).NotTo(HaveOccurred()) + Expect(params).To(MatchJSON(`{"targets":[{"class":"example.Worker","method":"compute","loader":"4ac"}]}`)) + }) + It("reads root-call filters as flags", func() { + params, err := parsed(kindSchema("jvm_trace"), "--targets", `[{"class":"example.Worker","method":"compute"}]`, "--min-cost", "250ms", "--exceptions-only", "--match", `[{"on":"arg","index":1,"path":"customer.name","op":"regex","value":"^acme"}]`) + Expect(err).NotTo(HaveOccurred()) + Expect(params).To(MatchJSON(`{"targets":[{"class":"example.Worker","method":"compute"}],"minCost":"250ms","exceptionsOnly":true,"match":[{"on":"arg","index":1,"path":"customer.name","op":"regex","value":"^acme"}]}`)) + }) + It("refuses non-object entries in target arrays", func() { + for _, value := range []string{`[null]`, `["wrong"]`, `null`, `{}`} { + _, err := parsed(kindSchema("jvm_trace"), "--targets", value) + Expect(err).To(HaveOccurred()) + } + }) It("builds a CPU checkpoint from profile flags", func() { flags := pflag.NewFlagSet("goprof", pflag.ContinueOnError) checkpoint := bindProfileCheckpoint(flags, goprof.CheckpointParams{CheckpointOptions: goprof.CheckpointOptions{Source: goprof.SourceGops, Profile: "heap"}}) @@ -51,7 +67,7 @@ var _ = Describe("a kind's params as flags", func() { It("shows the supported seam but does not offer unimplemented agent targeting", func() { flags := pflag.NewFlagSet("test", pflag.ContinueOnError) - _, err := bindParamFlags(flags, kindSchema(process.SamplesKind)) + _, err := bindParamFlags(flags, kindSchema(processcapture.Kind)) Expect(err).NotTo(HaveOccurred()) Expect(flags.Lookup("seam").Usage).To(ContainSubstring("process")) Expect(flags.Lookup("agent")).To(BeNil()) @@ -59,20 +75,20 @@ var _ = Describe("a kind's params as flags", func() { Expect(flags.Lookup("wait")).To(BeNil()) }) It("names the selector flag when its value is malformed", func() { - schema := kindSchema(process.SamplesKind) + schema := kindSchema(processcapture.Kind) schema.Properties["target"].Extensions["x-telltale-seams"] = []target.Seam{target.Agent, target.Process} _, err := parsed(schema, "--selector", "invalid") Expect(err).To(MatchError(ContainSubstring("--selector:"))) }) It("keeps a run PID separate from an explicit agent", func() { - schema := kindSchema(process.SamplesKind) + schema := kindSchema(processcapture.Kind) schema.Properties["target"].Extensions["x-telltale-seams"] = []target.Seam{target.Agent, target.Process} params, err := parsed(schema, "--pid", "123", "--agent", "worker") Expect(err).NotTo(HaveOccurred()) Expect(params).To(MatchJSON(`{"target":{"agent":"worker"}}`)) }) It("passes mutation consent in the common target", func() { - schema := kindSchema(process.SamplesKind) + schema := kindSchema(processcapture.Kind) schema.Properties["target"].Extensions["x-telltale-seams"] = []target.Seam{target.Agent, target.Process} params, err := parsed(schema, "--allow-mutation") Expect(err).NotTo(HaveOccurred()) @@ -101,8 +117,8 @@ var _ = Describe("a kind's params as flags", func() { Entry("a choice and a list", httpproxy.Kind, []string{"--mode", "mitm", "--hosts", "a.example,b.example"}, `{"mode": "mitm", "hosts": ["a.example", "b.example"]}`), Entry("a repeated list flag", httpproxy.Kind, []string{"--redact", "X-One", "--redact", "X-Two"}, `{"redact": ["X-One", "X-Two"]}`), Entry("a boolean and an integer", "sql", []string{"--params", "--max-statements", "10"}, `{"params": true, "maxStatements": 10}`), - Entry("an object's params, nested", process.TreeKind, []string{"--environment-keys", "PATH", "--interval", "2s"}, `{"environment": {"keys": ["PATH"]}, "interval": "2s"}`), - Entry("a run PID and an embedded struct's params", process.SamplesKind, []string{"--pid", "7", "--facets", "ports"}, `{"facets": ["ports"]}`), + Entry("an object's params, nested", processcapture.Kind, []string{"--environment-keys", "PATH", "--interval", "2s"}, `{"environment": {"keys": ["PATH"]}, "interval": "2s"}`), + Entry("a run PID and an embedded struct's params", processcapture.Kind, []string{"--pid", "7", "--facets", "sockets"}, `{"facets": ["sockets"]}`), ) It("refuses a value outside a param's choices, naming its flag and the choices", func() { @@ -126,9 +142,9 @@ var _ = Describe("a kind's params as flags", func() { _, err := bindParamFlags(flags, schema) Expect(err).To(MatchError(refusal)) }, - Entry("a list of objects", &rpc.OpenAPISchema{Type: "object", Properties: map[string]*rpc.OpenAPISchema{ - "rules": {Type: "array", Items: &rpc.OpenAPISchema{Type: "object"}}, - }}, "param rules: a list of object param cannot be a flag"), + Entry("a list of booleans", &rpc.OpenAPISchema{Type: "object", Properties: map[string]*rpc.OpenAPISchema{ + "rules": {Type: "array", Items: &rpc.OpenAPISchema{Type: "boolean"}}, + }}, "param rules: a list of boolean param cannot be a flag"), Entry("a param whose flag is taken", &rpc.OpenAPISchema{Type: "object", Properties: map[string]*rpc.OpenAPISchema{ "dataDir": {Type: "string"}, }}, "param dataDir: its flag --data-dir is already defined"), diff --git a/internal/cli/kindopenapi_internal_test.go b/internal/cli/kindopenapi_internal_test.go index aad0e63..f3f33ec 100644 --- a/internal/cli/kindopenapi_internal_test.go +++ b/internal/cli/kindopenapi_internal_test.go @@ -6,9 +6,11 @@ import ( "net/http" "net/http/httptest" "slices" + "strings" "testing/fstest" "github.com/flanksource/clicky/rpc" + "github.com/flanksource/telltale/internal/jvm" "github.com/flanksource/telltale/internal/server" "github.com/flanksource/telltale/pkg/traces" . "github.com/onsi/ginkgo/v2" @@ -34,22 +36,73 @@ var _ = Describe("kind argument OpenAPI", func() { It("serves schema-derived arguments for every registered run kind", func() { spec := kindOpenAPI(NewRoot("test")) for _, registration := range traces.Registrations() { - path := "/api/v1/run/{flags}/" + registration.Name + path := "/api/v1/run/" + registration.Name Expect(slices.Sorted(maps.Keys(spec.Paths))).To(ContainElement(path)) Expect(spec.Paths[path]).To(HaveKey("post")) } - http := spec.Paths["/api/v1/run/{flags}/http"]["post"].RequestBody.Content["application/json"].Schema.Properties + http := spec.Paths["/api/v1/run/http"]["post"].RequestBody.Content["application/json"].Schema.Properties Expect(http["mode"].Enum).To(Equal([]any{"connect", "mitm"})) Expect(http["seam"].Enum).To(Equal([]any{"self", "agent", "proxy"})) Expect(http["seam"].Default).To(Equal("self")) Expect(http["hosts"].Items.Type).To(Equal("string")) Expect(http).NotTo(HaveKey("format")) - checkpoint := spec.Paths["/api/v1/run/{flags}/goprof"]["post"].RequestBody.Content["application/json"].Schema.Properties + checkpoint := spec.Paths["/api/v1/run/goprof"]["post"].RequestBody.Content["application/json"].Schema.Properties Expect(checkpoint["profile"].Enum).To(Equal([]any{"cpu", "heap", "allocs", "mutex", "block", "goroutine"})) Expect(checkpoint["profile"].Default).To(Equal("heap")) Expect(checkpoint["seconds"].Type).To(Equal("integer")) }) + It("serves a typed operation for every built-in JVM operation", func() { + spec := kindOpenAPI(NewRoot("test")) + // inputs are an operation's typed inputs, whether the path, query or + // JSON body carries them, and which of them are required. + inputs := func(path string) (map[string]string, []string) { + types, required := map[string]string{}, []string{} + Expect(spec.Paths).To(HaveKey(path)) + for _, operation := range spec.Paths[path] { + for _, parameter := range operation.Parameters { + types[parameter.Name] = parameter.Schema.Type + if parameter.Required { + required = append(required, parameter.Name) + } + } + if operation.RequestBody != nil { + body := operation.RequestBody.Content["application/json"].Schema + for name, property := range body.Properties { + types[name] = property.Type + } + required = append(required, body.Required...) + } + } + return types, required + } + served := 0 + for path, item := range spec.Paths { + if strings.HasPrefix(path, "/api/v1/jvm/") && !slices.Contains([]string{"/api/v1/jvm/call", "/api/v1/jvm/deploy", "/api/v1/jvm/deployers", "/api/v1/jvm/snapshot"}, path) { + served += len(item) + } + } + presets := 0 + for _, registration := range traces.Registrations() { + presets += len(registration.Commands) + } + Expect(served).To(Equal(len(jvm.Operations()) + presets)) + Expect(spec.Paths).NotTo(HaveKey("/api/v1/jvm/classes/source")) + watch, _ := inputs("/api/v1/jvm/watch") + Expect(watch).To(HaveKeyWithValue("calls", "boolean")) + Expect(watch).To(HaveKeyWithValue("duration", "string")) + decompile, _ := inputs("/api/v1/jvm/decompile") + Expect(decompile).To(HaveKeyWithValue("class", "string")) + search, _ := inputs("/api/v1/jvm/jmx/search") + Expect(search).To(HaveKeyWithValue("pattern", "string")) + stack, required := inputs("/api/v1/jvm/threads/{id}/stack") + Expect(stack).To(HaveKeyWithValue("id", "integer")) + Expect(required).To(ContainElement("id")) + Expect(required).NotTo(ContainElement("agent"), "an omitted agent is the only one there is") + install, _ := inputs("/api/v1/jvm/extensions/install") + Expect(install).To(HaveKeyWithValue("jar", "string")) + }) + It("publishes required inputs while preserving optional defaults and list choices", func() { root := &cobra.Command{Use: "test"} command := &cobra.Command{Use: "capture", Run: func(*cobra.Command, []string) {}} diff --git a/internal/cli/records.go b/internal/cli/records.go index 9e73ee4..76d91f9 100644 --- a/internal/cli/records.go +++ b/internal/cli/records.go @@ -19,6 +19,7 @@ func newRecordsCommand() *cobra.Command { var since, from, until, groupBy, view string var cols []string var follow bool + var frame, source, metric string cmd := &cobra.Command{ Use: "records ", Short: "Read, follow, or aggregate a run's stored records", Args: cobra.ExactArgs(1), Long: `Read stored trace records through the kind's query profile. By default, select the newest run containing the kind and return up to 100 rows in profile order. @@ -27,12 +28,12 @@ func newRecordsCommand() *cobra.Command { --filter uses profile column grammar, e.g. pid=123 or rssBytes=>=1048576. --where uses CEL with a row binding, e.g. row.cpuPercent != null && row.cpuPercent > 10. --limit applies after CEL; --all reads every matching row. A continuation cursor is printed to stderr when more rows remain. ---stats aggregates all matching rows unless --limit is explicitly set. Nulls are skipped. Percentiles use nearest rank, exact through 8192 values per column/group, then an 8192-value reservoir; approximate=true identifies sampling. At most 1024 groups, 64 columns, and 1024 column/group pairs are allowed (64 MiB of reservoir values). --group-by alone counts rows. --view runs a named kind view over the seq window; row filters and time bounds require --stats instead.`, - Example: ` telltale records process_samples --json - telltale records process_samples --since 2h --filter 'rssBytes=>=1048576' --where 'row.cpuPercent > 10' --all - telltale records process_samples --run run-2026-10-28T09-00-00Z --follow --format ndjson - telltale records process_samples --stats cpuPercent,rssBytes --group-by pid - telltale records process_samples --group-by pid --format csv +--stats aggregates all matching rows unless --limit is explicitly set. Nulls are skipped. Percentiles use nearest rank, exact through 8192 values per column/group, then an 8192-value reservoir; approximate=true identifies sampling. At most 1024 groups, 64 columns, and 1024 column/group pairs are allowed (64 MiB of reservoir values). --group-by alone counts rows. --view top reads one completed sample, supports native filters, CEL, metric ordering and paging; --frame selects a historical sample. Other named views read the seq window; row filters and time bounds require --stats instead.`, + Example: ` telltale records process --json + telltale records process --result metrics --since 2h --filter 'rssBytes=>=1048576' --where 'row.cpuPercent > 10' --all + telltale records process --result facets --run --follow --format ndjson + telltale records process --result metrics --stats cpuPercent,rssBytes --group-by entity + telltale records process --result metrics --group-by entity --format csv telltale records spans --run receiver --filter service=checkout --limit 5 telltale records spans --run receiver --stats durationMs --group-by operation telltale records spans --run receiver --follow @@ -47,6 +48,24 @@ func newRecordsCommand() *cobra.Command { return err } defer func() { err = errors.Join(err, selected.Close()) }() + if aggregate.View == "top" { + if len(selected.Refs) != 1 { + return fmt.Errorf("top requires exactly one capture; select --run and its kind") + } + result, err := read.Top(cmd.Context(), selected.Refs[0], read.TopQuery{Frame: frame, Source: source, Metric: metric, Query: q}) + if err != nil { + return err + } + if result.NextCursor != "" { + _, _ = fmt.Fprintln(cmd.ErrOrStderr(), "Next cursor:", result.NextCursor) + } + switch clicky.Flags.ResolveFormat() { + case "json", "ndjson", "yaml", "csv": + return printRecords(cmd, result.Rows) + default: + return printRecords(cmd, result.Display) + } + } if aggregate.Follow { return read.FollowSelection(cmd.Context(), selected.Refs, q, recordPrinter(cmd)) } @@ -102,6 +121,7 @@ func newRecordsCommand() *cobra.Command { f := cmd.Flags() bindDataDir(f, &scope.DataDir) f.StringVar(&scope.Run, "run", "", "Run id (default: newest containing the kind)") + f.StringVar(&scope.Result, "result", "", "Named result table, e.g. process --result metrics or --result facets") f.StringSliceVar(&scope.Runs, "runs", nil, "Comma-separated run ids") f.StringVar(&since, "since", "", "Run-start cutoff and default row-time start: duration or RFC3339") f.StringVar(&from, "from", "", "Inclusive row-time start (RFC3339); overrides --since's row window") @@ -118,6 +138,11 @@ func newRecordsCommand() *cobra.Command { f.StringSliceVar(&cols, "stats", nil, "Numeric columns to aggregate, comma-separated") f.StringVar(&groupBy, "group-by", "", "Group aggregates by this column; alone counts rows") f.StringVar(&view, "view", "", "Named ResultView to run over the selected seq window") + f.StringVar(&frame, "frame", "", "Completed sample id for --view top (default: latest)") + f.StringVar(&source, "source", "", "Sample source for --view top") + f.StringVar(&metric, "metric", "", "Profile measurement for --view top") + f.StringVar(&q.Sort, "sort", "", "Native profile column sort key") + f.BoolVar(&q.Desc, "desc", false, "Sort descending") return cmd } diff --git a/internal/cli/records_test.go b/internal/cli/records_test.go index 8ebec90..d4a9cf7 100644 --- a/internal/cli/records_test.go +++ b/internal/cli/records_test.go @@ -2,6 +2,9 @@ package cli_test import ( "context" + "encoding/json" + "net/http" + "net/http/httptest" "path/filepath" "strings" "time" @@ -14,9 +17,28 @@ import ( ) var _ = Describe("records command", func() { + It("keeps followed NDJSON free of command output and final summaries", func() { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, err := w.Write([]byte("command-output")) + Expect(err).NotTo(HaveOccurred()) + })) + defer upstream.Close() + stdout, stderr, err := execute("trace", "run", "--data-dir", GinkgoT().TempDir(), "--follow", "--format", "ndjson", "--", "curl", "-sS", upstream.URL) + Expect(err).NotTo(HaveOccurred(), stderr) + lines := strings.Split(strings.TrimSpace(stdout), "\n") + Expect(lines).To(HaveLen(1)) + var record struct { + Kind string `json:"kind"` + Record map[string]any `json:"record"` + } + Expect(json.Unmarshal([]byte(lines[0]), &record)).To(Succeed()) + Expect(record.Kind).To(Equal("http")) + Expect(record.Record).To(HaveKey("request")) + Expect(stderr).To(ContainSubstring("command-output")) + }) It("registers follow on both run command forms", func() { root := cli.NewRoot("test") - for _, args := range [][]string{{"run", "process_samples"}, {"trace", "run"}} { + for _, args := range [][]string{{"run", "process"}, {"trace", "run"}} { cmd, _, err := root.Find(args) Expect(err).NotTo(HaveOccurred()) Expect(cmd.Flags().Lookup("follow")).NotTo(BeNil()) @@ -24,7 +46,7 @@ var _ = Describe("records command", func() { } }) DescribeTable("refuses invalid selections before opening stores", func(args []string, message string) { - _, _, err := execute(append([]string{"records", "process_samples", "--data-dir", GinkgoT().TempDir()}, args...)...) + _, _, err := execute(append([]string{"records", "process", "--data-dir", GinkgoT().TempDir()}, args...)...) Expect(err).To(MatchError(ContainSubstring(message))) }, Entry("follow across runs", []string{"--runs", "one,two", "--follow"}, "--follow requires a single run"), @@ -68,7 +90,7 @@ var _ = Describe("records command", func() { }) It("includes runnable examples and window semantics in help", func() { page := help("records") - Expect(page).To(ContainSubstring("telltale records process_samples")) + Expect(page).To(ContainSubstring("telltale records process --result metrics")) Expect(page).To(ContainSubstring("--from overrides")) }) }) diff --git a/internal/cli/replay.go b/internal/cli/replay.go new file mode 100644 index 0000000..8fc0e07 --- /dev/null +++ b/internal/cli/replay.go @@ -0,0 +1,107 @@ +package cli + +import ( + "errors" + "fmt" + + tea "charm.land/bubbletea/v2" + "github.com/flanksource/clicky" + "github.com/flanksource/telltale/internal/server" + "github.com/flanksource/telltale/internal/shutdown" + "github.com/flanksource/telltale/internal/traceui" + "github.com/flanksource/telltale/pkg/traces/read" + "github.com/spf13/cobra" +) + +func newTraceReplayCommand() *cobra.Command { + var dataDir string + cmd := &cobra.Command{ + Use: "replay [run-id...]", Short: "Replay recorded captures without rerunning their commands", Args: cobra.MinimumNArgs(1), + Long: "Read the saved captures of one or more trace/run IDs. Records are merged in sequence or timestamp order. --ui opens a local viewer with a run selector; --tui opens the interactive terminal. Replay never executes recorded commands.", + Example: " telltale trace replay buildhost20261107T123000Z\n telltale trace replay --ui\n telltale trace replay --format ndjson --sort timestamp", + RunE: func(cmd *cobra.Command, ids []string) (err error) { + follow, tui, ui, order, err := followOptions(cmd) + if err != nil { + return err + } + if tui && ui { + return fmt.Errorf("replay: --ui and --tui are mutually exclusive") + } + explicitFollow, err := cmd.Flags().GetBool("follow") + if err != nil { + return err + } + if explicitFollow && tui { + return fmt.Errorf("replay: --follow cannot combine with --tui") + } + if tui && clicky.Flags.ResolveFormat() == "ndjson" { + return fmt.Errorf("--tui cannot combine with ndjson") + } + format := clicky.Flags.ResolveFormat() + if format != "ndjson" && format != "pretty" && format != "ansi" { + return fmt.Errorf("replay supports ndjson, ansi or pretty output") + } + selected, err := read.ResolveRuns(read.Scope{DataDir: dataDir, Runs: ids}) + if err != nil { + return err + } + defer func() { err = errors.Join(err, selected.Close()) }() + if ui { + viewer, err := openRunViewer(cmd, server.ViewerOptions{DataDir: dataDir, RunIDs: ids}) + if err != nil { + return err + } + return waitViewer(cmd.Context(), viewer) + } + if tui { + restore, err := shutdown.Terminal(cmd.Context()) + if err != nil { + return err + } + defer restore() + final, err := tea.NewProgram(traceui.New(cmd.Context(), traceui.Options{Refs: selected.Refs, Runs: selected.Runs, Sort: order, Replay: true}), tea.WithContext(cmd.Context())).Run() + restore() + err = errors.Join(err, shutdown.Request(cmd.Context())) + if model, ok := final.(traceui.Model); ok { + err = errors.Join(err, model.Err()) + } + return err + } + printer := newFeedPrinter(cmd.OutOrStdout()) + if follow { + err = read.Feed(cmd.Context(), selected.Refs, read.Query{}, printer.print) + } else { + err = replayRecords(cmd, selected.Refs, order, printer) + } + if err != nil { + return err + } + return printer.stats(cmd.ErrOrStderr()) + }, + } + bindDataDir(cmd.Flags(), &dataDir) + cmd.Flags().Bool("follow", false, "Follow selected captures until their streams seal") + cmd.Flags().Bool("ui", false, "Open a local web viewer for the selected runs until Ctrl+C") + cmd.Flags().Bool("tui", false, "Replay records in an interactive terminal") + cmd.Flags().String("sort", "seq", "Record order: seq or timestamp") + return cmd +} + +func replayRecords(cmd *cobra.Command, refs []read.Ref, order string, printer *feedPrinter) error { + query := read.FeedQuery{Sort: order} + for { + page, err := read.FeedPage(cmd.Context(), refs, query) + if err != nil { + return err + } + for _, record := range page.Rows { + if err := printer.print(record); err != nil { + return err + } + } + if page.NextCursor == "" { + return nil + } + query.Cursor = page.NextCursor + } +} diff --git a/internal/cli/replay_test.go b/internal/cli/replay_test.go new file mode 100644 index 0000000..d8b6f85 --- /dev/null +++ b/internal/cli/replay_test.go @@ -0,0 +1,68 @@ +package cli_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/pkg/traces/read" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("trace replay", func() { + It("reads every selected run without executing its command again", func() { + var requests atomic.Int64 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + _, _ = w.Write([]byte("recorded")) + })) + DeferCleanup(upstream.Close) + targetURL := upstream.URL + "/recorded" + dataDir := GinkgoT().TempDir() + ids := []string{} + for range 2 { + run, err := telltale.Start(context.Background(), telltale.StartOptions{DataDir: dataDir, Command: []string{"curl", "--noproxy", "", "-sS", targetURL}, Traces: []telltale.TraceSpec{{Kind: "http", Params: json.RawMessage(`{"target":{"seam":"proxy"}}`)}}}) + Expect(err).NotTo(HaveOccurred()) + _, err = run.Wait() + Expect(err).NotTo(HaveOccurred()) + ids = append(ids, run.ID()) + } + for _, selected := range [][]string{ids[:1], ids} { + args := append([]string{"trace", "replay", "--data-dir", dataDir, "--format", "ndjson"}, selected...) + stdout, stderr, err := execute(args...) + Expect(err).NotTo(HaveOccurred(), stderr) + rows := []read.Record{} + for _, line := range strings.Split(strings.TrimSpace(stdout), "\n") { + var record read.Record + Expect(json.Unmarshal([]byte(line), &record)).To(Succeed()) + rows = append(rows, record) + } + for _, id := range selected { + Expect(rows).To(ContainElement(And(HaveField("RunID", id), HaveField("Kind", "http"), HaveField("Record", HaveKeyWithValue("url", targetURL))))) + Expect(rows).To(ContainElement(And(HaveField("RunID", id), HaveField("Kind", "process"), HaveField("Result", "metrics")))) + } + Expect(rows).To(HaveEach(HaveField("RunID", BeElementOf(selected)))) + } + Expect(requests.Load()).To(Equal(int64(2))) + stdout, _, err := execute("trace", "replay", "--data-dir", dataDir, ids[0], "missing") + Expect(err).To(HaveOccurred()) + Expect(stdout).To(BeEmpty()) + _, _, err = execute("trace", "replay", "--data-dir", dataDir, ids[0], ids[0]) + Expect(err).To(MatchError(ContainSubstring("selected twice"))) + }) + + DescribeTable("rejects invalid replay requests", func(args []string, message string) { + _, _, err := execute(append([]string{"trace", "replay", "--data-dir", GinkgoT().TempDir()}, args...)...) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("no IDs", nil, "at least 1 arg"), + Entry("unknown ID", []string{"missing"}, "get run missing"), + Entry("path traversal", []string{"../escape"}, "not a run id"), + Entry("bad sort", []string{"missing", "--sort", "invalid"}, "sort"), + ) +}) diff --git a/internal/cli/root.go b/internal/cli/root.go index 720c761..c6a4637 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/flanksource/clicky" + "github.com/flanksource/clicky/rpc" "github.com/flanksource/telltale/internal/otelreceiver" "github.com/flanksource/telltale/internal/server" "github.com/flanksource/telltale/pkg/datadir" @@ -42,15 +43,20 @@ func NewRoot(version string) *cobra.Command { bindDataDir(serve.Flags(), &options.DataDir) serve.Flags().StringVar(&options.RulesFile, "rules", "", "Continuously observe resource rules from this YAML file") serve.Flags().BoolVar(&options.EnforceRules, "enforce-rules", false, "Execute resource rule hooks (default: observation only)") - serve.Flags().BoolVar(&options.OTLP.Enabled, "otlp", false, "Continuously receive OTLP traces into local storage") + serve.Flags().BoolVar(&options.OTLP.Enabled, "otlp", false, "Continuously receive OTLP traces and Profiles into local storage") + serve.Flags().BoolVar(&options.Prometheus, "prometheus", false, "Keep VictoriaMetrics available for Prometheus scrape captures in this data directory") serve.Flags().BoolVar(&options.SelfTrace, "self-trace", false, "Trace HTTP requests to the local OTLP receiver (enables --otlp)") serve.Flags().StringVar(&options.OTLP.GRPCListen, "otlp-grpc-listen", options.OTLP.GRPCListen, "OTLP gRPC listen address") - serve.Flags().DurationVar(&options.OTLP.Retention, "otlp-retention", options.OTLP.Retention, "Received span retention from ingestion time") + serve.Flags().DurationVar(&options.OTLP.Retention, "otlp-retention", options.OTLP.Retention, "Received telemetry retention from ingestion time") serve.Flags().IntVar(&options.OTLP.MaxRequestBytes, "otlp-max-request-bytes", options.OTLP.MaxRequestBytes, "Maximum OTLP request bytes before and after decompression") - serve.Flags().IntVar(&options.OTLP.MaxSpans, "otlp-max-spans", options.OTLP.MaxSpans, "Maximum spans per OTLP request") + serve.Flags().IntVar(&options.OTLP.MaxSpans, "otlp-max-spans", options.OTLP.MaxSpans, "Maximum spans or Profile samples per OTLP request") serve.Flags().IntVar(&options.OTLP.MaxInflight, "otlp-max-inflight", options.OTLP.MaxInflight, "Maximum concurrent OTLP exports across both transports") root.AddCommand(serve, newTraceCommand(), newConnectionCommand(), newRunCommand(), newTopCommand(), newRulesCommand(), newAgentsCommand()) - root.AddCommand(newRecordsCommand(), newDefinitionsCommand()) + root.AddCommand(newRecordsCommand(), newDefinitionsCommand(), newExplainCommand(), newJVMCommand()) + // Kind presets first, so an operation declared below one, such as + // `jvm trace probes`, becomes a subcommand of the runnable preset. + addKindCommands(root) + addJVMOperations(root) builtins, err := definition.Builtins() if err != nil { panic(err) @@ -61,5 +67,6 @@ func NewRoot(version string) *cobra.Command { } root.AddCommand(newPresetCommand(def)) } + rpc.InstallInteractive(root) return root } diff --git a/internal/cli/run.go b/internal/cli/run.go index 68c98e3..1c11279 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -2,13 +2,19 @@ package cli import ( "context" + "encoding/json" + "errors" "fmt" + "maps" "strconv" "strings" + "time" "github.com/flanksource/clicky" "github.com/flanksource/clicky/api" + "github.com/flanksource/clicky/rpc" "github.com/flanksource/telltale" + "github.com/flanksource/telltale/internal/server" "github.com/flanksource/telltale/pkg/traces" "github.com/flanksource/telltale/pkg/traces/read" "github.com/flanksource/telltale/pkg/tracing" @@ -31,7 +37,7 @@ func (f *runFlags) bind(cmd *cobra.Command, checkpointed bool) { if checkpointed { cmd.Flags().DurationVar(&f.options.Every, "every", 0, "Read the checkpointed captures this often, e.g. 10s") } - cmd.Flags().Bool("follow", false, "Print each capture's rows while recording") + bindFollow(cmd) } // newRunCommand is `run` with one subcommand per registered trace kind, each @@ -68,12 +74,39 @@ func newRunCommand() *cobra.Command { if err != nil { panic(err) } - run.AddCommand(newRunKindCommand(registration, plugin)) + run.AddCommand(newRunKindCommand(registration, plugin, nil)) } return run } -func newRunKindCommand(registration traces.Registration, plugin tracing.TracePlugin) *cobra.Command { +// newRunKindCommand is `run `, or, given a preset, the kind served as +// the preset's own command with the preset's param defaults. +// addKindCommands places every registered kind preset at its path below +// root, such as jvm_trace's `jvm watch`. +func addKindCommands(root *cobra.Command) { + kinds, err := traces.Kinds() + if err != nil { + panic(fmt.Sprintf("trace kind catalog: %v", err)) + } + for _, registration := range traces.Registrations() { + for _, preset := range registration.Commands { + plugin, err := traces.Lookup(kinds, registration.Name) + if err != nil { + panic(err) + } + parent := commandAt(root, preset.Path[:len(preset.Path)-1]) + name := preset.Path[len(preset.Path)-1] + for _, existing := range parent.Commands() { + if existing.Name() == name { + panic(fmt.Sprintf("trace kind %s command %v is already the %q command", registration.Name, preset.Path, existing.CommandPath())) + } + } + parent.AddCommand(newRunKindCommand(registration, plugin, &preset)) + } + } +} + +func newRunKindCommand(registration traces.Registration, plugin tracing.TracePlugin, preset *traces.KindCommand) *cobra.Command { var flags runFlags cmd := &cobra.Command{ Use: registration.Name + " [flags] [-- command [args...]]", @@ -86,10 +119,21 @@ func newRunKindCommand(registration traces.Registration, plugin tracing.TracePlu if err != nil { panic(fmt.Sprintf("trace kind %s params: %v", registration.Name, err)) } + if preset != nil { + cmd.Use = preset.Path[len(preset.Path)-1] + " [flags]" + cmd.Short = preset.Summary + cmd.Long = preset.Summary + ".\n\n" + registration.Description + "\n\nThe same capture as `telltale run " + registration.Name + "` with these defaults." + if schema, err = presetSchema(schema, preset.Defaults); err != nil { + panic(fmt.Sprintf("trace kind %s command %v: %v", registration.Name, preset.Path, err)) + } + } params, err := bindParamFlags(cmd.Flags(), schema) if err != nil { panic(fmt.Sprintf("trace kind %s: %v", registration.Name, err)) } + if err := bindConnectionCompletions(cmd, params); err != nil { + panic(err) + } var checkpoint *profileCheckpointFlags if defaults, ok := registration.Checkpoint.(goprof.CheckpointParams); ok { checkpoint = bindProfileCheckpoint(cmd.Flags(), defaults) @@ -100,12 +144,28 @@ func newRunKindCommand(registration traces.Registration, plugin tracing.TracePlu return err } options := flags.options + if preset != nil { + // A run records into a data dir; over HTTP that is the server's, + // which only its run API knows, not one the caller names. + if _, remote := rpc.RequestFromContext(cmd.Context()); remote { + return fmt.Errorf("telltale %s records a run: start it over HTTP with POST /api/traces and kind %s", strings.Join(preset.Path, " "), registration.Name) + } + if raw, err = presetParams(preset.Defaults, raw); err != nil { + return err + } + } if cmd.Flags().Lookup("pid") != nil { options.PID, err = cmd.Flags().GetInt("pid") if err != nil { return err } } + // A run attached to a pid aims its agent captures at that process. + if options.PID == 0 { + if raw, err = withAgentTarget(cmd.Context(), registration, raw); err != nil { + return err + } + } if options.PID == 0 { if err := plugin.ValidateParams(raw); err != nil { return fmt.Errorf("trace %s %w", registration.Name, err) @@ -120,17 +180,80 @@ func newRunKindCommand(registration traces.Registration, plugin tracing.TracePlu } } options.Traces = []telltale.TraceSpec{{Kind: registration.Name, Params: raw, Checkpoint: reading}} - return runTraced(cmd, options, telltale.Start) + return runTraced(cmd, options, startKindRun) + } + if preset == nil { + setHelp(cmd, func(cmd *cobra.Command) api.Text { return kindHelp(cmd, registration).Text }) } - setHelp(cmd, func(cmd *cobra.Command) api.Text { return kindHelp(cmd, registration).Text }) return cmd } +// startKindRun starts the run a kind command arms; internal tests replace it +// to read the trace spec a command builds. +var startKindRun = telltale.Start + +// presetSchema is a kind's params schema with a preset's defaults over its +// own, so help shows them and required params they give are satisfied. +func presetSchema(schema *rpc.OpenAPISchema, defaults map[string]any) (*rpc.OpenAPISchema, error) { + base, err := objectDefaults(schema) + if err != nil { + return nil, err + } + merged := *schema + merged.Default = mergeParams(base, defaults) + return &merged, nil +} + +// presetParams are a preset's defaults with the params its flags set over them. +func presetParams(defaults map[string]any, raw json.RawMessage) (json.RawMessage, error) { + given := map[string]any{} + if len(raw) > 0 { + if err := json.Unmarshal(raw, &given); err != nil { + return nil, err + } + } + return json.Marshal(mergeParams(defaults, given)) +} + +// mergeParams is base with over's values over it, objects merged by key. +func mergeParams(base, over map[string]any) map[string]any { + merged := maps.Clone(base) + if merged == nil { + merged = map[string]any{} + } + for key, value := range over { + nested, isObject := value.(map[string]any) + existing, wasObject := merged[key].(map[string]any) + if isObject && wasObject { + value = mergeParams(existing, nested) + } + merged[key] = value + } + return merged +} + // runTraced starts a run with start, waits for it and prints what it // recorded; a command that failed or a checkpoint that did is an error. func runTraced(cmd *cobra.Command, options telltale.StartOptions, start func(context.Context, telltale.StartOptions) (*telltale.Run, error)) error { + follow, tui, ui, sort, err := followOptions(cmd) + if err != nil { + return err + } + format := clicky.Flags.ResolveFormat() + if tui && format == "ndjson" { + return fmt.Errorf("--tui cannot combine with ndjson") + } + if follow && format != "ndjson" && format != "pretty" && format != "ansi" { + return fmt.Errorf("follow supports ndjson, ansi or pretty output") + } if options.Stdin == nil && options.Stdout == nil && options.Stderr == nil && options.PID == 0 { options.Stdin, options.Stdout, options.Stderr = cmd.InOrStdin(), cmd.OutOrStdout(), cmd.ErrOrStderr() + if format != "pretty" && format != "ansi" { + options.Stdout = cmd.ErrOrStderr() + } + } + if tui { + options.Stdin, options.Stdout, options.Stderr = nil, nil, nil } connections, err := telltale.OpenConnections() if err != nil { @@ -141,18 +264,61 @@ func runTraced(cmd *cobra.Command, options telltale.StartOptions, start func(con if err != nil { return err } + var viewer *server.Viewer + if ui { + viewer, err = openRunViewer(cmd, server.ViewerOptions{DataDir: options.DataDir, RunIDs: []string{started.ID()}}) + if err != nil { + return errors.Join(err, started.Stop()) + } + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = viewer.Close(ctx) + }() + } switch { case options.PID != 0: _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Tracing pid %d into %s; Ctrl+C stops\n", options.PID, started.Dir()) // a lost notice loses nothing recorded case len(options.Command) == 0: _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Recording the host into %s; Ctrl+C stops\n", started.Dir()) } - follow := cmd.Flags().Lookup("follow") != nil && cmd.Flags().Lookup("follow").Value.String() == "true" - result, err := read.Wait(cmd.Context(), started, follow, recordPrinter(cmd)) + var result *telltale.Result + printer := newFeedPrinter(cmd.OutOrStdout()) + human := newFeedPrinter(cmd.ErrOrStderr()) + human.ndjson = false + if tui { + result, err = showTraceTUI(cmd.Context(), started, sort) + } else { + result, err = read.Wait(cmd.Context(), started, true, func(record read.Record) error { + if err := human.print(record); err != nil { + return err + } + if follow && format == "ndjson" { + return printer.print(record) + } + return nil + }) + } if err != nil { return err } - clicky.MustPrint(result) + if !tui { + if err := human.stats(cmd.ErrOrStderr()); err != nil { + return err + } + } + if follow { + if err := writeLine(cmd.ErrOrStderr(), result.Pretty()); err != nil { + return err + } + } else { + clicky.MustPrint(result) + } + if viewer != nil { + if err := waitViewer(cmd.Context(), viewer); err != nil { + return err + } + } switch { case result.ExitCode != 0: return fmt.Errorf("%s exited with code %d", options.Command[0], result.ExitCode) diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index bd79fae..049f2e0 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -95,6 +95,14 @@ var _ = Describe("telltale run ", func() { Expect(page).ToNot(ContainSubstring("\x1b["), "help written to a buffer is plain") }) + It("shows Kubernetes and direct URL examples for Prometheus", func() { + Expect(help("run", "prometheus")).To(SatisfyAll( + ContainSubstring("--endpoints port-forward://metrics-exporter.monitoring:9100/metrics"), + ContainSubstring("--endpoints proxy://metrics-exporter.monitoring:9100/metrics"), + ContainSubstring("--endpoints https://metrics.example.com/metrics"), + )) + }) + DescribeTable("refuses what it cannot run before making a run store", func(kind string, args []string, refusal string) { dataDir := GinkgoT().TempDir() @@ -125,12 +133,12 @@ var _ = Describe("telltale run ", func() { _ = target.Wait() }) dataDir := GinkgoT().TempDir() - _, stderr, err := execute("run", "process_samples", "--data-dir", dataDir, "--pid", strconv.Itoa(target.Process.Pid), "--duration", "1s") + _, stderr, err := execute("run", "process", "--data-dir", dataDir, "--pid", strconv.Itoa(target.Process.Pid), "--duration", "1s") Expect(err).ToNot(HaveOccurred(), stderr) Expect(stderr).To(ContainSubstring("Tracing pid " + strconv.Itoa(target.Process.Pid))) result := manifest(dataDir) Expect(result.PID).To(Equal(target.Process.Pid)) - Expect(result.Captures).To(ConsistOf(HaveField("Kind", "process_samples"))) + Expect(result.Captures).To(ConsistOf(HaveField("Kind", "process"))) }) It("runs goprof and stacks against a child through the agent seam", func() { @@ -156,7 +164,7 @@ var _ = Describe("telltale run ", func() { Expect(manifest(dataDir).PID).To(Equal(os.Getpid())) }, Entry("self", "http", []string{"--seam", "self", "--features", "http"}), - Entry("host", "process_groups", nil), + Entry("host", "process", []string{"--seam", "host"}), ) DescribeTable("reports the same refusal for CLI and library runs", func(kind string, args []string, options telltale.StartOptions) { _, _, cliErr := execute(append([]string{"run", kind}, args...)...) @@ -171,11 +179,11 @@ var _ = Describe("telltale run ", func() { Entry("a command and PID", "stacks", []string{"--pid", strconv.Itoa(os.Getpid()), "--", "true"}, telltale.StartOptions{PID: os.Getpid(), Command: []string{"true"}}), ) It("shows supported seams and run attachment in help", func() { - page := help("run", "process_groups") + page := help("run", "process") Expect(page).To(ContainSubstring("--pid PID")) - Expect(page).To(ContainSubstring("supported seams: [host]")) + Expect(page).To(ContainSubstring("Supported seams: process, host")) Expect(page).NotTo(ContainSubstring("--agent")) - Expect(help("run", "process_samples")).To(ContainSubstring("--seam process")) + Expect(page).To(ContainSubstring("--seam string")) }) }) diff --git a/internal/cli/styled_replay_test.go b/internal/cli/styled_replay_test.go new file mode 100644 index 0000000..4c3462a --- /dev/null +++ b/internal/cli/styled_replay_test.go @@ -0,0 +1,41 @@ +package cli_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + + "github.com/flanksource/clicky" + "github.com/flanksource/telltale" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("styled replay output", func() { + It("keeps HTTP pretty-printing colors and honors no-color", func() { + GinkgoT().Setenv("CLICKY_FORCE_INTERACTIVE", "1") + GinkgoT().Setenv("TERM", "xterm-256color") + GinkgoT().Setenv("NO_COLOR", "") + GinkgoT().Setenv("COLOR", "true") + saved := *clicky.Flags + DeferCleanup(func() { *clicky.Flags = saved; clicky.UseFormatter(saved.FormatOptions) }) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("recorded")) })) + DeferCleanup(server.Close) + dataDir := GinkgoT().TempDir() + run, err := telltale.Start(context.Background(), telltale.StartOptions{DataDir: dataDir, Command: []string{"curl", "--noproxy", "", "-sS", server.URL + "/orders"}, + Traces: []telltale.TraceSpec{{Kind: "http", Params: json.RawMessage(`{"target":{"seam":"proxy"}}`)}}}) + Expect(err).NotTo(HaveOccurred()) + _, err = run.Wait() + Expect(err).NotTo(HaveOccurred()) + stdout, stderr, err := execute("trace", "replay", run.ID(), "--data-dir", dataDir, "--format", "ansi") + Expect(err).NotTo(HaveOccurred(), stderr) + Expect(clicky.Flags.NoColor).To(BeFalse(), "explicit color environment should keep colors") + Expect(stdout).To(ContainSubstring("\x1b[")) + Expect(stdout).To(ContainSubstring("GET")) + stdout, stderr, err = execute("trace", "replay", run.ID(), "--data-dir", dataDir, "--format", "ansi", "--no-color") + Expect(err).NotTo(HaveOccurred(), stderr) + Expect(stdout).NotTo(ContainSubstring("\x1b[")) + Expect(stdout).To(ContainSubstring("GET")) + }) +}) diff --git a/internal/cli/top.go b/internal/cli/top.go index 7355b0f..729c9b6 100644 --- a/internal/cli/top.go +++ b/internal/cli/top.go @@ -6,16 +6,17 @@ import ( "errors" "fmt" "os" - "os/signal" - "syscall" "time" tea "charm.land/bubbletea/v2" "github.com/flanksource/clicky" "github.com/flanksource/telltale" + "github.com/flanksource/telltale/internal/shutdown" "github.com/flanksource/telltale/internal/topui" "github.com/flanksource/telltale/pkg/tracing/groups" "github.com/flanksource/telltale/pkg/tracing/process" + "github.com/flanksource/telltale/pkg/tracing/processcapture" + "github.com/flanksource/telltale/pkg/tracing/target" "github.com/spf13/cobra" "golang.org/x/term" ) @@ -41,8 +42,8 @@ func newTopCommand() *cobra.Command { Long: "Sorts every process on the host into the tool that owns it — claude, codex, cmux, gavel, go test, vitest, agent-browser, chrome, " + "brew services and more — as a tree of groups, each with its checkout, agent session, cmux workspace and what started it, " + "and shows what each group and those under it use, full screen. The processes no tool holds are kept under other (the user's) " + - "and system (root's and the role accounts'), by program.\n\n" + - "Keys: ↑↓ move, ←→ fold, enter lists the selected group's processes, s sorts, S reverses, / filters, o shows other processes, " + + "and system (root's and the role accounts'), by program. Gavel groups add the test and lint progress and slots their process publishes.\n\n" + + "Keys: ↑↓ move, ←→ fold, enter lists the selected group's processes in the tree, f opens the process pane, s sorts, S reverses, / filters, o shows other processes, " + "b cycles grouping, n reads network bytes, q quits.\n\n" + "Nothing is kept unless --record records a run into the data dir, which `telltale trace list` and `telltale serve` show. " + "Rules come from the built-in catalog and groups.yaml in ./.telltale, else the global config dir (or --catalog).", @@ -86,8 +87,7 @@ func (f topFlags) run(cmd *cobra.Command) error { return err } } - ctx, stop := signal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM) - defer stop() + ctx := cmd.Context() if f.record { return f.recordRun(ctx) } @@ -170,9 +170,9 @@ func (f topFlags) printOnce(ctx context.Context, top *groups.Top) error { // store commits until it is stopped or its duration passes, then what it // recorded. func (f topFlags) recordRun(ctx context.Context) error { - raw, err := json.Marshal(f.params) + raw, err := json.Marshal(processcapture.Params{Options: processcapture.Options{Options: process.Options{Interval: f.params.Interval}, Catalog: f.params.Catalog, Net: f.params.Net, Facets: []string{processcapture.Files, processcapture.Sockets, "connections"}}, Target: target.Target{Seam: target.Host}}) if err != nil { - return fmt.Errorf("encode %s params: %w", groups.Kind, err) + return fmt.Errorf("encode %s params: %w", processcapture.Kind, err) } connections, err := telltale.OpenConnections() if err != nil { @@ -180,13 +180,13 @@ func (f topFlags) recordRun(ctx context.Context) error { } run, err := telltale.Start(ctx, telltale.StartOptions{ DataDir: f.dataDir, Duration: f.duration, Connections: connections, - Traces: []telltale.TraceSpec{{Kind: groups.Kind, Params: raw}}, + Traces: []telltale.TraceSpec{{Kind: processcapture.Kind, Params: raw}}, }) if err != nil { return err } if interactive() { - follower, err := groups.NewFollower(run.Runtime(), run.Info().Captures[0].Ref) + follower, err := processcapture.NewFollower(run.Runtime(), run.Info().Captures[0].Ref) if err != nil { return errors.Join(err, run.Stop()) } @@ -226,7 +226,14 @@ func ended(done <-chan struct{}) bool { func (f topFlags) show(ctx context.Context, options topui.Options) error { options.GroupBy = groups.GroupBy(f.groupBy) model := topui.New(ctx, options) + restore, err := shutdown.Terminal(ctx) + if err != nil { + return err + } + defer restore() final, err := tea.NewProgram(model, tea.WithContext(ctx)).Run() + restore() + shutdownErr := shutdown.Request(ctx) if last, ok := final.(topui.Model); ok { last.Close() if err == nil { @@ -234,7 +241,7 @@ func (f topFlags) show(ctx context.Context, options topui.Options) error { } } if errors.Is(err, tea.ErrProgramKilled) && ctx.Err() != nil { - return nil + return shutdownErr } - return err + return errors.Join(err, shutdownErr) } diff --git a/internal/cli/trace.go b/internal/cli/trace.go index 6b9e7be..2c9602b 100644 --- a/internal/cli/trace.go +++ b/internal/cli/trace.go @@ -13,7 +13,7 @@ import ( func newTraceCommand() *cobra.Command { trace := &cobra.Command{Use: "trace", Short: "Record what a command does"} - trace.AddCommand(newTraceRunCommand(), newTraceListCommand()) + trace.AddCommand(newTraceRunCommand(), newTraceListCommand(), newTraceReplayCommand()) return trace } @@ -38,7 +38,7 @@ func newTraceRunCommand() *cobra.Command { bindDataDir(run.Flags(), &options.DataDir) run.Flags().DurationVar(&options.Duration, "duration", 0, "Stop the command after this long, e.g. 30s") run.Flags().DurationVar(&options.Every, "every", 0, "Read the checkpointed captures this often, e.g. 10s") - run.Flags().Bool("follow", false, "Print each capture's rows while recording") + bindFollow(run) setHelp(run, func(cmd *cobra.Command) api.Text { return traceRunHelp(cmd).Text }) return run } diff --git a/internal/jvm/catalog.go b/internal/jvm/catalog.go new file mode 100644 index 0000000..9798889 --- /dev/null +++ b/internal/jvm/catalog.go @@ -0,0 +1,99 @@ +package jvm + +import ( + _ "embed" + "encoding/json" + "fmt" + "strings" + + "github.com/flanksource/clicky/rpc" +) + +// catalog.json is the Java operation registry's built-in catalog, written by +// `./sdk/java/gradlew -p sdk/java jvmCatalog`; CatalogSnapshotTest fails when +// it is stale. +// +//go:embed catalog.json +var catalogJSON []byte + +// Operation is one built-in operation of the Java diagnostics registry. +type Operation struct { + Name string + // Command is the CLI command, and API path, the registry serves the + // operation as, such as [jvm patch apply] for jvm.classes.patch. + Command []string + Title string + Description string + Params *rpc.OpenAPISchema + Result *rpc.OpenAPISchema + Mutates bool +} + +var operations = mustDecodeCatalog(catalogJSON) + +// Operations are the built-in operations every Java agent serves, in registry order. +func Operations() []Operation { return operations } + +func mustDecodeCatalog(data []byte) []Operation { + decoded, err := decodeCatalog(data) + if err != nil { + panic(fmt.Sprintf("embedded JVM operation catalog: %v", err)) + } + return decoded +} + +func decodeCatalog(data []byte) ([]Operation, error) { + var entries []struct { + Name string `json:"name"` + Title string `json:"title"` + Description string `json:"description"` + Command string `json:"command"` + ParamsSchema map[string]any `json:"paramsSchema"` + ResultSchema map[string]any `json:"resultSchema"` + Mutates bool `json:"mutates"` + } + if err := json.Unmarshal(data, &entries); err != nil { + return nil, err + } + decoded := make([]Operation, 0, len(entries)) + for _, entry := range entries { + command := strings.Fields(entry.Command) + if !strings.HasPrefix(entry.Name, "jvm.") || entry.Description == "" || len(command) < 2 || command[0] != "jvm" { + return nil, fmt.Errorf("operation %q needs a jvm. name, a description and a jvm command", entry.Name) + } + params, err := schema(entry.ParamsSchema) + if err != nil { + return nil, fmt.Errorf("%s params: %w", entry.Name, err) + } + if params.Type != "object" { + return nil, fmt.Errorf("%s params: expected an object schema, got %q", entry.Name, params.Type) + } + result, err := schema(entry.ResultSchema) + if err != nil { + return nil, fmt.Errorf("%s result: %w", entry.Name, err) + } + decoded = append(decoded, Operation{Name: entry.Name, Command: command, Title: entry.Title, Description: entry.Description, Params: params, Result: result, Mutates: entry.Mutates}) + } + return decoded, nil +} + +// schema decodes a JSON Schema into an OpenAPI schema. A closed object's +// `additionalProperties: false` has no OpenAPI field to land in and is +// dropped: only declared properties become flags or request fields anyway. +func schema(value map[string]any) (*rpc.OpenAPISchema, error) { + if closed, ok := value["additionalProperties"].(bool); ok { + if closed { + return nil, fmt.Errorf("additionalProperties: true is not supported; use a schema") + } + delete(value, "additionalProperties") + } + raw, err := json.Marshal(value) + if err != nil { + return nil, err + } + var decoded rpc.OpenAPISchema + if err := json.Unmarshal(raw, &decoded); err != nil { + return nil, err + } + return &decoded, nil +} diff --git a/internal/jvm/catalog.json b/internal/jvm/catalog.json new file mode 100644 index 0000000..357f9ef --- /dev/null +++ b/internal/jvm/catalog.json @@ -0,0 +1,1102 @@ +[ + { + "name": "jvm.extensions.list", + "title": "jvm.extensions.list", + "description": "List loaded diagnostic extensions", + "command": "jvm extensions list", + "paramsSchema": { + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": false + }, + "resultSchema": { + "type": "array", + "items": { + "type": "object" + } + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.extensions.install", + "title": "jvm.extensions.install", + "description": "Install a diagnostic extension JAR", + "command": "jvm extensions install", + "paramsSchema": { + "type": "object", + "properties": { + "jar": { + "type": "string", + "title": "jar", + "format": "byte" + } + }, + "required": [ + "jar" + ], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": true, + "scope": "jvm:instrument" + }, + { + "name": "jvm.extensions.replace", + "title": "jvm.extensions.replace", + "description": "Replace a loaded diagnostic extension with a new JAR", + "command": "jvm extensions replace", + "paramsSchema": { + "type": "object", + "properties": { + "jar": { + "type": "string", + "title": "jar", + "format": "byte" + } + }, + "required": [ + "jar" + ], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": true, + "scope": "jvm:instrument" + }, + { + "name": "jvm.extensions.unload", + "title": "jvm.extensions.unload", + "description": "Unload a diagnostic extension", + "command": "jvm extensions unload", + "paramsSchema": { + "type": "object", + "properties": { + "id": { + "type": "string", + "title": "id" + } + }, + "required": [ + "id" + ], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": true, + "scope": "jvm:instrument" + }, + { + "name": "jvm.catalog", + "title": "jvm.catalog", + "description": "List the operations this runtime supports", + "command": "jvm catalog", + "paramsSchema": { + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": false + }, + "resultSchema": { + "type": "array", + "items": { + "type": "object" + } + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.status", + "title": "jvm.status", + "description": "Report mutation permission, instrumentation and active patches", + "command": "jvm status", + "paramsSchema": { + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.jmx.search", + "title": "jvm.jmx.search", + "description": "Search MBean names matching an ObjectName pattern", + "command": "jvm jmx search", + "paramsSchema": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "title": "pattern" + } + }, + "required": [], + "additionalProperties": false + }, + "resultSchema": { + "type": "array", + "items": { + "type": "string" + } + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.jmx.info", + "title": "jvm.jmx.info", + "description": "Describe an MBean's attributes, operations and notifications", + "command": "jvm jmx info", + "paramsSchema": { + "type": "object", + "properties": { + "objectName": { + "type": "string", + "title": "objectName" + } + }, + "required": [ + "objectName" + ], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.jmx.read", + "title": "jvm.jmx.read", + "description": "Read one MBean attribute, or all of them", + "command": "jvm jmx read", + "paramsSchema": { + "type": "object", + "properties": { + "objectName": { + "type": "string", + "title": "objectName" + }, + "attribute": { + "type": "string", + "title": "attribute" + } + }, + "required": [ + "objectName" + ], + "additionalProperties": false + }, + "resultSchema": {}, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.jmx.write", + "title": "jvm.jmx.write", + "description": "Write an MBean attribute", + "command": "jvm jmx write", + "paramsSchema": { + "type": "object", + "properties": { + "objectName": { + "type": "string", + "title": "objectName" + }, + "attribute": { + "type": "string", + "title": "attribute" + }, + "value": {} + }, + "required": [ + "objectName", + "attribute", + "value" + ], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": true, + "scope": "jvm:write" + }, + { + "name": "jvm.jmx.invoke", + "title": "jvm.jmx.invoke", + "description": "Invoke an MBean operation", + "command": "jvm jmx invoke", + "paramsSchema": { + "type": "object", + "properties": { + "objectName": { + "type": "string", + "title": "objectName" + }, + "operation": { + "type": "string", + "title": "operation" + }, + "args": { + "type": "array", + "title": "args", + "items": {} + }, + "signature": { + "type": "array", + "title": "signature", + "items": { + "type": "string" + } + } + }, + "required": [ + "objectName", + "operation" + ], + "additionalProperties": false + }, + "resultSchema": {}, + "mutates": true, + "scope": "jvm:write" + }, + { + "name": "jvm.metrics", + "title": "jvm.metrics", + "description": "Snapshot memory, GC, thread, class-loading and runtime metrics", + "command": "jvm metrics", + "paramsSchema": { + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.telemetry", + "title": "jvm.telemetry", + "description": "Read native activity telemetry from every loaded application without arming it or using JMX", + "command": "jvm telemetry", + "paramsSchema": { + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": false + }, + "resultSchema": { + "type": "object", + "properties": { + "capturedAt": { + "type": "string", + "title": "capturedAt" + }, + "applications": { + "type": "array", + "items": { + "type": "object", + "properties": { + "loader": { + "type": "string", + "title": "loader" + }, + "label": { + "type": "string", + "title": "label" + }, + "status": { + "type": "object", + "title": "status", + "additionalProperties": {} + }, + "histograms": { + "type": "array", + "title": "histograms", + "items": {} + } + } + } + } + } + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.settings", + "title": "jvm.settings", + "description": "Read OS/JVM properties, environment and resource settings", + "command": "jvm settings", + "paramsSchema": { + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.resources", + "title": "jvm.resources", + "description": "Read filesystem capacity and Linux process and cgroup resources", + "command": "jvm resources", + "paramsSchema": { + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.flags", + "title": "jvm.flags", + "description": "Read effective HotSpot VM flags", + "command": "jvm flags", + "paramsSchema": { + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": false + }, + "resultSchema": { + "type": "string" + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.threads", + "title": "jvm.threads", + "description": "Sample thread states and CPU over a short interval", + "command": "jvm threads", + "paramsSchema": { + "type": "object", + "properties": { + "sampleMs": { + "type": "integer", + "title": "sampleMs" + } + }, + "required": [], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.threads.dump", + "title": "jvm.threads.dump", + "description": "Dump every thread's stack as text", + "command": "jvm threads dump", + "paramsSchema": { + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": false + }, + "resultSchema": { + "type": "string" + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.heap.histogram", + "title": "jvm.heap.histogram", + "description": "List the heap's classes by retained bytes (GC.class_histogram)", + "command": "jvm heap histogram", + "paramsSchema": { + "type": "object", + "properties": { + "top": { + "type": "integer", + "title": "top", + "description": "Rows to return, largest first (default 100)." + }, + "classPattern": { + "type": "string", + "title": "classPattern", + "description": "Glob over class names, e.g. com.example.*" + }, + "live": { + "type": "boolean", + "title": "live", + "description": "Count only reachable objects. Forces a full GC on this JVM first." + } + }, + "required": [], + "additionalProperties": false + }, + "resultSchema": { + "type": "object", + "properties": { + "collectedAt": { + "type": "string", + "title": "collectedAt" + }, + "live": { + "type": "boolean", + "title": "live" + }, + "totalInstances": { + "type": "integer", + "title": "totalInstances" + }, + "totalBytes": { + "type": "integer", + "title": "totalBytes" + }, + "matched": { + "type": "integer", + "title": "matched" + }, + "rows": { + "type": "array", + "title": "rows", + "items": {} + } + } + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.threads.stack", + "title": "jvm.threads.stack", + "description": "Read one thread's stack and lock state", + "command": "jvm threads stack", + "paramsSchema": { + "type": "object", + "properties": { + "id": { + "type": "integer", + "title": "id" + } + }, + "required": [ + "id" + ], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.logging.list", + "title": "jvm.logging.list", + "description": "List java.util.logging loggers and their levels", + "command": "jvm logging list", + "paramsSchema": { + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": false + }, + "resultSchema": { + "type": "array", + "items": { + "type": "object" + } + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.logging.set", + "title": "jvm.logging.set", + "description": "Set a logger's level, or inherit its parent's with null", + "command": "jvm logging set", + "paramsSchema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "title": "name" + }, + "level": { + "type": "string", + "nullable": true, + "title": "level", + "description": "Use null to inherit the parent logger level." + } + }, + "required": [ + "name", + "level" + ], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": true, + "scope": "jvm:write" + }, + { + "name": "jvm.classes", + "title": "jvm.classes", + "description": "Page through loaded classes by name and loader", + "command": "jvm classes", + "paramsSchema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "title": "query" + }, + "loader": { + "type": "string", + "title": "loader" + }, + "limit": { + "type": "integer", + "title": "limit" + }, + "offset": { + "type": "integer", + "title": "offset" + }, + "includePlatform": { + "type": "boolean", + "title": "includePlatform" + } + }, + "required": [], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.loaders", + "title": "jvm.loaders", + "description": "List class loaders and their identities", + "command": "jvm loaders", + "paramsSchema": { + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": false + }, + "resultSchema": { + "type": "array", + "items": { + "type": "object" + } + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.groovy.eval", + "title": "jvm.groovy.eval", + "description": "Execute a Groovy script once in an application loader", + "command": "jvm groovy eval", + "paramsSchema": { + "type": "object", + "properties": { + "source": { + "type": "string", + "title": "source", + "format": "textarea" + }, + "loader": { + "type": "string", + "title": "loader" + }, + "args": { + "type": "object", + "title": "args", + "additionalProperties": {} + }, + "timeoutSeconds": { + "type": "integer", + "title": "timeoutSeconds" + } + }, + "required": [ + "source" + ], + "additionalProperties": false + }, + "resultSchema": { + "type": "object", + "properties": { + "value": {}, + "graph": { + "type": "object", + "title": "graph", + "additionalProperties": {} + }, + "stdout": { + "type": "string", + "title": "stdout" + }, + "stderr": { + "type": "string", + "title": "stderr" + }, + "elapsedMs": { + "type": "integer", + "title": "elapsedMs" + } + } + }, + "mutates": true, + "scope": "jvm:eval" + }, + { + "name": "jvm.ognl.eval", + "title": "jvm.ognl.eval", + "description": "Evaluate an OGNL expression in an application loader", + "command": "jvm ognl eval", + "paramsSchema": { + "type": "object", + "properties": { + "expression": { + "type": "string", + "title": "expression", + "format": "textarea" + }, + "loader": { + "type": "string", + "title": "loader" + } + }, + "required": [ + "expression" + ], + "additionalProperties": false + }, + "resultSchema": { + "type": "object", + "properties": { + "value": {}, + "graph": { + "type": "object", + "title": "graph", + "additionalProperties": {} + } + } + }, + "mutates": true, + "scope": "jvm:eval" + }, + { + "name": "jvm.classes.source", + "title": "jvm.classes.source", + "description": "Decompile a class with its descriptors and line mappings", + "command": "jvm decompile", + "paramsSchema": { + "type": "object", + "properties": { + "class": { + "type": "string", + "title": "class" + }, + "loader": { + "type": "string", + "title": "loader" + }, + "method": { + "type": "string", + "title": "method" + } + }, + "required": [ + "class" + ], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.classes.bytecode", + "title": "jvm.classes.bytecode", + "description": "Read a class's bytecode as base64", + "command": "jvm decompile bytecode", + "paramsSchema": { + "type": "object", + "properties": { + "class": { + "type": "string", + "title": "class" + }, + "loader": { + "type": "string", + "title": "loader" + }, + "method": { + "type": "string", + "title": "method" + } + }, + "required": [ + "class" + ], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.classes.compile", + "title": "jvm.classes.compile", + "description": "Preview compiling replacement source without applying it", + "command": "jvm patch compile", + "paramsSchema": { + "type": "object", + "properties": { + "class": { + "type": "string", + "title": "class" + }, + "loader": { + "type": "string", + "title": "loader" + }, + "method": { + "type": "string", + "title": "method" + }, + "source": { + "type": "string", + "title": "source", + "format": "textarea" + }, + "classpath": { + "type": "string", + "title": "classpath" + } + }, + "required": [ + "class", + "source" + ], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.classes.patch", + "title": "jvm.classes.patch", + "description": "Compile and apply replacement source under a restore lease", + "command": "jvm patch apply", + "paramsSchema": { + "type": "object", + "properties": { + "class": { + "type": "string", + "title": "class" + }, + "loader": { + "type": "string", + "title": "loader" + }, + "method": { + "type": "string", + "title": "method" + }, + "source": { + "type": "string", + "title": "source", + "format": "textarea" + }, + "classpath": { + "type": "string", + "title": "classpath" + }, + "leaseSeconds": { + "type": "integer", + "title": "leaseSeconds" + } + }, + "required": [ + "class", + "source" + ], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": true, + "scope": "jvm:instrument" + }, + { + "name": "jvm.classes.instrument", + "title": "jvm.classes.instrument", + "description": "Add a method probe to a class under a restore lease", + "command": "jvm trace instrument", + "paramsSchema": { + "type": "object", + "properties": { + "class": { + "type": "string", + "title": "class" + }, + "loader": { + "type": "string", + "title": "loader" + }, + "method": { + "type": "string", + "title": "method" + }, + "probe": { + "type": "object", + "title": "probe", + "additionalProperties": {} + }, + "leaseSeconds": { + "type": "integer", + "title": "leaseSeconds" + } + }, + "required": [ + "class", + "probe" + ], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": true, + "scope": "jvm:instrument" + }, + { + "name": "jvm.classes.restore", + "title": "jvm.classes.restore", + "description": "Restore a class's original bytecode, or remove one probe", + "command": "jvm patch restore", + "paramsSchema": { + "type": "object", + "properties": { + "class": { + "type": "string", + "title": "class" + }, + "loader": { + "type": "string", + "title": "loader" + }, + "method": { + "type": "string", + "title": "method" + }, + "probe": { + "type": "object", + "title": "probe", + "additionalProperties": {} + } + }, + "required": [ + "class" + ], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": true, + "scope": "jvm:instrument" + }, + { + "name": "jvm.classes.lease", + "title": "jvm.classes.lease", + "description": "Renew or change a patched class's restore lease", + "command": "jvm patch lease", + "paramsSchema": { + "type": "object", + "properties": { + "class": { + "type": "string", + "title": "class" + }, + "loader": { + "type": "string", + "title": "loader" + }, + "method": { + "type": "string", + "title": "method" + }, + "seconds": { + "type": "integer", + "title": "seconds" + } + }, + "required": [ + "class" + ], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": true, + "scope": "jvm:instrument" + }, + { + "name": "jvm.classes.patches", + "title": "jvm.classes.patches", + "description": "List active bytecode patches and probes", + "command": "jvm patch list", + "paramsSchema": { + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": false + }, + "resultSchema": { + "type": "array", + "items": { + "type": "object" + } + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.probes.read", + "title": "jvm.probes.read", + "description": "Read buffered events of a method probe", + "command": "jvm trace read", + "paramsSchema": { + "type": "object", + "properties": { + "key": { + "type": "string", + "title": "key" + }, + "since": { + "type": "integer", + "title": "since" + }, + "limit": { + "type": "integer", + "title": "limit" + } + }, + "required": [ + "key" + ], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.probes.list", + "title": "jvm.probes.list", + "description": "List method probe keys", + "command": "jvm trace probes", + "paramsSchema": { + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": false, + "scope": "jvm:read" + }, + { + "name": "jvm.probes.freeze", + "title": "jvm.probes.freeze", + "description": "Stop a method probe from recording further events", + "command": "jvm trace freeze", + "paramsSchema": { + "type": "object", + "properties": { + "key": { + "type": "string", + "title": "key" + } + }, + "required": [ + "key" + ], + "additionalProperties": false + }, + "resultSchema": { + "type": "object" + }, + "mutates": true, + "scope": "jvm:trace" + }, + { + "name": "jvm.notifications.list", + "title": "jvm.notifications.list", + "description": "List JMX notification subscriptions and their queues", + "command": "jvm notifications list", + "paramsSchema": { + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": false + }, + "resultSchema": { + "type": "array", + "items": { + "type": "object" + } + }, + "mutates": false, + "scope": "jvm:read" + } +] diff --git a/internal/jvm/files.go b/internal/jvm/files.go new file mode 100644 index 0000000..13d69a0 --- /dev/null +++ b/internal/jvm/files.go @@ -0,0 +1,34 @@ +package jvm + +import ( + "encoding/base64" + "errors" + "fmt" + "io" + "os" +) + +// paramFileLimit bounds a file read into one operation param, matching the +// agent's 16 MiB extension upload limit. +const paramFileLimit = 16 << 20 + +// ReadParamFile is a local file's content as an operation param: its text, or +// base64 for a binary param such as an extension JAR. +func ReadParamFile(path string, encode bool) (content string, err error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer func() { err = errors.Join(err, file.Close()) }() + data, err := io.ReadAll(io.LimitReader(file, paramFileLimit+1)) + if err != nil { + return "", err + } + if len(data) > paramFileLimit { + return "", fmt.Errorf("%s exceeds 16 MiB", path) + } + if encode { + return base64.StdEncoding.EncodeToString(data), nil + } + return string(data), nil +} diff --git a/internal/jvm/files_test.go b/internal/jvm/files_test.go new file mode 100644 index 0000000..caf3ba9 --- /dev/null +++ b/internal/jvm/files_test.go @@ -0,0 +1,86 @@ +package jvm + +import ( + "encoding/base64" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestJVM(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "JVM operations") } + +var _ = Describe("ReadParamFile", func() { + write := func(data []byte) string { + path := filepath.Join(GinkgoT().TempDir(), "param") + Expect(os.WriteFile(path, data, 0o600)).To(Succeed()) + return path + } + + It("reads text as is and binary content as base64", func() { + jar := []byte{0x50, 0x4b, 0x03, 0x04, 0x00, 0xff} + Expect(ReadParamFile(write([]byte("return 3 + 4")), false)).To(Equal("return 3 + 4")) + Expect(ReadParamFile(write(jar), true)).To(Equal(base64.StdEncoding.EncodeToString(jar))) + }) + + It("refuses a file over the 16 MiB upload limit", func() { + _, err := ReadParamFile(write(make([]byte, paramFileLimit+1)), true) + Expect(err).To(MatchError(ContainSubstring("exceeds 16 MiB"))) + }) +}) + +var _ = Describe("embedded operation catalog", func() { + It("decodes every built-in operation with an object params schema", func() { + names := make([]string, 0, len(Operations())) + for _, operation := range Operations() { + names = append(names, operation.Name) + Expect(operation.Params.Type).To(Equal("object"), operation.Name) + Expect(operation.Description).NotTo(BeEmpty(), operation.Name) + } + Expect(names).To(ContainElements("jvm.jmx.search", "jvm.classes", "jvm.classes.source", "jvm.groovy.eval", "jvm.extensions.install", "jvm.logging.set")) + Expect(slices.Compact(slices.Sorted(slices.Values(names)))).To(HaveLen(len(names))) + }) + + It("places every operation at its declared command", func() { + commands := map[string]string{} + for _, operation := range Operations() { + commands[operation.Name] = strings.Join(operation.Command, " ") + } + Expect(commands).To(Equal(map[string]string{ + "jvm.extensions.list": "jvm extensions list", "jvm.extensions.install": "jvm extensions install", + "jvm.extensions.replace": "jvm extensions replace", "jvm.extensions.unload": "jvm extensions unload", + "jvm.catalog": "jvm catalog", "jvm.status": "jvm status", + "jvm.jmx.search": "jvm jmx search", "jvm.jmx.info": "jvm jmx info", "jvm.jmx.read": "jvm jmx read", + "jvm.jmx.write": "jvm jmx write", "jvm.jmx.invoke": "jvm jmx invoke", "jvm.metrics": "jvm metrics", + "jvm.telemetry": "jvm telemetry", + "jvm.settings": "jvm settings", "jvm.resources": "jvm resources", "jvm.flags": "jvm flags", + "jvm.threads": "jvm threads", "jvm.threads.dump": "jvm threads dump", "jvm.threads.stack": "jvm threads stack", "jvm.heap.histogram": "jvm heap histogram", + "jvm.logging.list": "jvm logging list", "jvm.logging.set": "jvm logging set", + "jvm.classes": "jvm classes", "jvm.loaders": "jvm loaders", + "jvm.groovy.eval": "jvm groovy eval", "jvm.ognl.eval": "jvm ognl eval", + "jvm.classes.source": "jvm decompile", "jvm.classes.bytecode": "jvm decompile bytecode", + "jvm.classes.compile": "jvm patch compile", "jvm.classes.patch": "jvm patch apply", + "jvm.classes.restore": "jvm patch restore", "jvm.classes.lease": "jvm patch lease", + "jvm.classes.patches": "jvm patch list", "jvm.classes.instrument": "jvm trace instrument", + "jvm.probes.list": "jvm trace probes", "jvm.probes.read": "jvm trace read", "jvm.probes.freeze": "jvm trace freeze", + "jvm.notifications.list": "jvm notifications list", + })) + }) + + It("refuses an operation without a description, a command or a closed params object", func() { + for _, data := range []string{ + `[{"name":"jvm.x","command":"jvm x","paramsSchema":{"type":"object"}}]`, + `[{"name":"jvm.x","description":"d","paramsSchema":{"type":"object"}}]`, + `[{"name":"jvm.x","description":"d","command":"x","paramsSchema":{"type":"object"}}]`, + `[{"name":"jvm.x","description":"d","command":"jvm x","paramsSchema":{"type":"object","additionalProperties":true}}]`, + `[{"name":"jvm.x","description":"d","command":"jvm x","paramsSchema":{"type":"string"}}]`, + } { + _, err := decodeCatalog([]byte(data)) + Expect(err).To(HaveOccurred(), data) + } + }) +}) diff --git a/internal/jvm/service.go b/internal/jvm/service.go new file mode 100644 index 0000000..5d19373 --- /dev/null +++ b/internal/jvm/service.go @@ -0,0 +1,41 @@ +package jvm + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/flanksource/telltale/pkg/agentclient" +) + +type Request struct { + Agent string `json:"agent"` + Operation string `json:"operation"` + Params json.RawMessage `json:"params"` + AllowMutation bool `json:"allowMutation"` +} + +func Call(ctx context.Context, request Request) (json.RawMessage, error) { + if request.Operation == "" { + return nil, fmt.Errorf("an operation is required") + } + if len(request.Params) == 0 { + request.Params = json.RawMessage(`{}`) + } + var params map[string]json.RawMessage + if err := json.Unmarshal(request.Params, ¶ms); err != nil { + return nil, fmt.Errorf("operation parameters: %w", err) + } + if params == nil { + return nil, fmt.Errorf("operation parameters must be an object") + } + // The agent may be named by a short name, or not at all when only one Java agent is there. + agent, err := agentclient.Default.Lookup(ctx, request.Agent, agentclient.Want{Runtime: "java"}) + if err != nil { + return nil, err + } + if agent.Hello.Runtime.Lang != "java" { + return nil, fmt.Errorf("agent %q is not a Java runtime", agent.Hello.AgentID) + } + return agent.Call(ctx, agentclient.CallRequest{Op: request.Operation, Params: request.Params, AllowMutation: request.AllowMutation}) +} diff --git a/internal/jvm/snapshot/collect.go b/internal/jvm/snapshot/collect.go new file mode 100644 index 0000000..a5585fe --- /dev/null +++ b/internal/jvm/snapshot/collect.go @@ -0,0 +1,167 @@ +package snapshot + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "slices" + "strings" + "time" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/tracing" +) + +const maxArtifact = 64 << 20 + +type caller func(context.Context, string) (json.RawMessage, error) + +func Collect(ctx context.Context, options Options) (*Result, error) { + if err := options.Validate(); err != nil { + return nil, err + } + agent := agentclient.Default.ByID(options.Agent) + if agent == nil || agent.Hello.Runtime.Lang != "java" { + return nil, fmt.Errorf("connected Java agent %q is required", options.Agent) + } + result, err := collect(ctx, options, workloadTarget(agent.Hello.Resource, agent.Hello.Service), func(ctx context.Context, op string) (json.RawMessage, error) { + return agent.Call(ctx, agentclient.CallRequest{Op: op, Params: json.RawMessage(`{}`)}) + }) + if err != nil { + return nil, err + } + identity, err := json.Marshal(tracing.MaskAttribute("identity", agent.Hello)) + if err != nil { + return nil, err + } + identity, err = sanitized(identity, false) + result.add("identity.json", identity, err) + if options.JFR != "" { + data, captureErr := profile(ctx, agent, options.JFR) + result.add("recording.jfr", data, captureErr) + } + result.finish() + return result, nil +} + +func collect(ctx context.Context, options Options, target WorkloadTarget, call caller) (*Result, error) { + if err := options.Validate(); err != nil { + return nil, err + } + result := &Result{Manifest: Manifest{Version: 1, Agent: options.Agent, StartedAt: time.Now().UTC(), Complete: true}, Files: map[string][]byte{}} + operations := []struct{ op, file string }{ + {"jvm.settings", "settings.json"}, {"jvm.resources", "resources.json"}, {"jvm.flags", "vm-flags.txt"}, {"jvm.metrics", "metrics.json"}, + {"jvm.threads.dump", "threads.txt"}, {"jvm.loaders", "loaders.json"}, + {"jvm.logging.list", "logging.json"}, {"jvm.status", "status.json"}, + {"jvm.telemetry", "telemetry.json"}, + } + if options.HeapHistogram { + operations = append(operations, struct{ op, file string }{"jvm.heap.histogram", "heap-histogram.json"}) + } + for _, operation := range operations { + if err := ctx.Err(); err != nil { + result.add(operation.file, nil, err) + continue + } + bounded, cancel := context.WithTimeout(ctx, 30*time.Second) + raw, err := call(bounded, operation.op) + cancel() + var data []byte + if err == nil { + data, err = sanitized(raw, strings.HasSuffix(operation.file, ".txt")) + } + result.add(operation.file, data, err) + } + addTelemetryReport(result) + if count := options.workloadSamples(); count > 0 { + addWorkload(ctx, result, target, call, count) + } + result.finish() + return result, nil +} + +// addWorkload adds the sampled series and the page that summarises it. +func addWorkload(ctx context.Context, result *Result, target WorkloadTarget, call caller, count int) { + samples, err := collectWorkload(ctx, call, count) + var series, page []byte + if err == nil { + series, err = json.MarshalIndent(samples, "", " ") + } + result.add("workload.json", series, err) + if err == nil { + var panel workloadPanel + if panel, err = newWorkloadPanel(target, samples); err == nil { + page, err = workloadHTML(panel) + } + } + result.add("workload.html", page, err) +} + +func (r *Result) add(name string, data []byte, err error) { + total := len(data) + for _, existing := range r.Files { + total += len(existing) + } + if err == nil && total > 128<<20 { + err = fmt.Errorf("bundle exceeds 128 MiB") + } + if err == nil && len(data) > maxArtifact { + err = fmt.Errorf("artifact exceeds 64 MiB") + } + outcome := Outcome{File: name, Status: "ok"} + if err != nil { + outcome.Status = "failed" + outcome.Error = maskArguments(tracing.MaskAttribute("error", err.Error())).(string) + r.Manifest.Complete = false + } else { + outcome.SHA256 = fmt.Sprintf("%x", sha256.Sum256(data)) + r.Files[name] = data + } + r.Manifest.Collectors = append(r.Manifest.Collectors, outcome) +} + +func (r *Result) finish() { + r.Manifest.CompletedAt = time.Now().UTC() + var summary strings.Builder + fmt.Fprintf(&summary, "# JVM support snapshot\n\nAgent: %s\n\nComplete: %t\n\n", r.Manifest.Agent, r.Manifest.Complete) + for _, outcome := range r.Manifest.Collectors { + fmt.Fprintf(&summary, "- %s: %s %s\n", outcome.File, outcome.Status, outcome.Error) + } + summary.WriteString("\nHeap histogram, when requested, includes unreachable objects and can pause the JVM. A timed-out native heap walk may continue in the JVM.\n") + r.Files["summary.md"] = []byte(summary.String()) +} + +func (r *Result) Zip() ([]byte, error) { + manifest, err := json.MarshalIndent(r.Manifest, "", " ") + if err != nil { + return nil, err + } + files := make(map[string][]byte, len(r.Files)) + for name, data := range r.Files { + files[name] = data + } + files["manifest.json"] = manifest + var output bytes.Buffer + writer := zip.NewWriter(&output) + names := make([]string, 0, len(files)) + for name := range files { + names = append(names, name) + } + slices.Sort(names) + for _, name := range names { + entry, err := writer.Create(name) + if err != nil { + return nil, err + } + if _, err := entry.Write(files[name]); err != nil { + return nil, err + } + } + if err := writer.Close(); err != nil { + return nil, err + } + return output.Bytes(), nil +} diff --git a/internal/jvm/snapshot/live_test.go b/internal/jvm/snapshot/live_test.go new file mode 100644 index 0000000..6e6d2b2 --- /dev/null +++ b/internal/jvm/snapshot/live_test.go @@ -0,0 +1,58 @@ +package snapshot + +import ( + "bytes" + "context" + "encoding/json" + "os/exec" + "path/filepath" + "time" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/tracing/jfr/decode" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("disposable JVM support snapshot", func() { + It("attaches by PID, masks properties and arguments, and closes only its own recording", func() { + directory := GinkgoT().TempDir() + output, err := exec.Command("javac", "-d", directory, "../../../pkg/agentinstall/testdata/Workload.java").CombinedOutput() + Expect(err).NotTo(HaveOccurred(), string(output)) + child := exec.Command("java", "-Dexample.password=snapshot-private-value", "-cp", directory, "Workload") + Expect(child.Start()).To(Succeed()) + DeferCleanup(func() { _ = child.Process.Kill(); _ = child.Wait() }) + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + previous := agentclient.Default + agentclient.Default = agentclient.NewRegistry() + DeferCleanup(func() { agentclient.Default = previous }) + id, err := Resolve(ctx, "", child.Process.Pid, "") + Expect(err).NotTo(HaveOccurred()) + agent := agentclient.Default.ByID(id) + recording, err := agent.Arm(ctx, agentclient.ArmRequest{Probe: "jvm_jfr", Params: json.RawMessage(`{"engine":"jfr","settings":"profile"}`)}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + cleanup, done := context.WithTimeout(context.Background(), 10*time.Second) + defer done() + Expect(recording.Disarm(cleanup)).To(Succeed()) + Expect(agent.Close()).To(Succeed()) + }) + result, err := Collect(ctx, Options{Agent: id, JFR: "1s", HeapHistogram: true}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Manifest.Complete).To(BeTrue(), "%+v", result.Manifest.Collectors) + Expect(string(result.Files["heap-histogram.json"])).To(ContainSubstring("java.lang.String")) + for name, content := range result.Files { + Expect(bytes.Contains(content, []byte("snapshot-private-value"))).To(BeFalse(), name) + } + profile, err := decode.Decode(result.Files["recording.jfr"], decode.DecodeOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(profile.Samples).NotTo(BeEmpty()) + checkpoint, err := recording.Checkpoint(ctx, json.RawMessage(`{}`)) + Expect(err).NotTo(HaveOccurred()) + Expect(checkpoint.Blobs).To(HaveLen(1)) + again, err := Resolve(ctx, "", child.Process.Pid, filepath.Join(directory, "not-a-jdk")) + Expect(err).NotTo(HaveOccurred()) + Expect(again).To(Equal(id)) + }) +}) diff --git a/internal/jvm/snapshot/profile.go b/internal/jvm/snapshot/profile.go new file mode 100644 index 0000000..cf077d9 --- /dev/null +++ b/internal/jvm/snapshot/profile.go @@ -0,0 +1,45 @@ +package snapshot + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/flanksource/telltale/pkg/agentclient" +) + +func profile(ctx context.Context, agent *agentclient.Agent, duration string) (data []byte, err error) { + if err := ctx.Err(); err != nil { + return nil, err + } + window, err := time.ParseDuration(duration) + if err != nil { + return nil, err + } + session, err := agent.Arm(ctx, agentclient.ArmRequest{Probe: "jvm_jfr", Params: json.RawMessage(`{"engine":"jfr","settings":"profile","supportSnapshot":true}`), StopAt: time.Now().Add(window + 15*time.Second)}) + if err != nil { + return nil, err + } + defer func() { + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + err = errors.Join(err, session.Disarm(cleanup)) + }() + timer := time.NewTimer(window) + defer timer.Stop() + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-timer.C: + } + checkpoint, err := session.Checkpoint(ctx, json.RawMessage(`{}`)) + if err != nil { + return nil, err + } + if len(checkpoint.Blobs) != 1 { + return nil, fmt.Errorf("expected one JFR artifact, received %d", len(checkpoint.Blobs)) + } + return scrub(ctx, checkpoint.Blobs[0].Data) +} diff --git a/internal/jvm/snapshot/redact.go b/internal/jvm/snapshot/redact.go new file mode 100644 index 0000000..89db7b8 --- /dev/null +++ b/internal/jvm/snapshot/redact.go @@ -0,0 +1,70 @@ +package snapshot + +import ( + "bytes" + "encoding/json" + "fmt" + "regexp" + + "github.com/flanksource/telltale/pkg/tracing" +) + +// secretWords name a secret in a key or an argument, in any case: tracing's +// masking misses camelCase property names such as trustStorePassword. +const secretWords = `(?:password|passwd|secret|token|credential|api[_-]?key)` + +var ( + assignments = regexp.MustCompile(`(?i)([-D]*[a-z0-9_.-]*` + secretWords + `[a-z0-9_.-]*\s*[=:]\s*)("[^"]*"|'[^']*'|[^\s,;]+)`) + secretKey = regexp.MustCompile(`(?i)` + secretWords) +) + +// masked is what tracing.MaskAttribute replaces a secret with. +const masked = "****" + +func sanitized(raw []byte, text bool) ([]byte, error) { + var value any + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + if err := decoder.Decode(&value); err != nil { + return nil, err + } + value = tracing.MaskAttribute("snapshot", maskArguments(value)) + if text { + content, ok := value.(string) + if !ok { + return nil, fmt.Errorf("expected text artifact") + } + return []byte(assignments.ReplaceAllString(content, "${1}[REDACTED]")), nil + } + // Inspect decoded strings so JSON escaping cannot hide argument assignments. + return json.MarshalIndent(value, "", " ") +} + +func maskArguments(value any) any { + switch v := value.(type) { + case string: + return assignments.ReplaceAllString(v, "${1}[REDACTED]") + case map[string]any: + if key, ok := v["key"].(string); ok { + if child, exists := v["value"]; exists { + if _, scalar := child.(string); scalar && secretKey.MatchString(key) { + v["value"] = masked + } else { + v["value"] = tracing.MaskAttribute(key, child) + } + } + } + for key, child := range v { + if _, scalar := child.(string); scalar && secretKey.MatchString(key) { + v[key] = masked + continue + } + v[key] = maskArguments(child) + } + case []any: + for i, child := range v { + v[i] = maskArguments(child) + } + } + return value +} diff --git a/internal/jvm/snapshot/scrub.go b/internal/jvm/snapshot/scrub.go new file mode 100644 index 0000000..8fbc256 --- /dev/null +++ b/internal/jvm/snapshot/scrub.go @@ -0,0 +1,45 @@ +package snapshot + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + + clickyexec "github.com/flanksource/clicky/exec" + "github.com/flanksource/telltale/internal/safefs" +) + +const supportEvents = "jdk.ExecutionSample,jdk.NativeMethodSample,jdk.ObjectAllocationSample,jdk.ObjectAllocationInNewTLAB,jdk.ObjectAllocationOutsideTLAB,jdk.JavaMonitorEnter,jdk.ThreadPark,jdk.GarbageCollection,jdk.GCHeapSummary" + +// Scrubbing rebuilds constant pools as well as removing foreign events: another +// recording may enable sensitive events JVM-wide despite our own settings. +func scrub(ctx context.Context, data []byte) (result []byte, err error) { + if err := os.MkdirAll(".tmp/jvm-snapshot", 0700); err != nil { + return nil, err + } + directory, err := os.MkdirTemp(".tmp/jvm-snapshot", "profile-") + if err != nil { + return nil, err + } + defer func() { + err = errors.Join(err, safefs.SafeRemove(directory, safefs.RemoveContext{AllowedRoot: ".tmp/jvm-snapshot"})) + }() + input, output := filepath.Join(directory, "raw.jfr"), filepath.Join(directory, "support.jfr") + if err := os.WriteFile(input, data, 0600); err != nil { + return nil, err + } + process := clickyexec.NewExec("jfr", "scrub", "--include-events", supportEvents, input, output).WithContext(ctx).WithCaptureLimit(4096).Run() + if process.Err != nil { + return nil, fmt.Errorf("scrub JFR (requires JDK 19+ jfr tool on Telltale host): %w: %s", process.Err, process.GetStderr()) + } + info, err := os.Stat(output) + if err != nil { + return nil, err + } + if info.Size() > maxArtifact { + return nil, fmt.Errorf("scrubbed JFR exceeds 64 MiB") + } + return os.ReadFile(output) +} diff --git a/internal/jvm/snapshot/snapshot_test.go b/internal/jvm/snapshot/snapshot_test.go new file mode 100644 index 0000000..34bd502 --- /dev/null +++ b/internal/jvm/snapshot/snapshot_test.go @@ -0,0 +1,92 @@ +package snapshot + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestSnapshot(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "JVM snapshot") } + +var _ = Describe("support archive", func() { + It("retains evidence and masks secrets when a collector fails", func() { + result, err := collect(context.Background(), Options{}, WorkloadTarget{}, func(_ context.Context, op string) (json.RawMessage, error) { + if op == "jvm.threads.dump" { + return nil, errors.New("thread dump denied") + } + return json.RawMessage(`{"password":"snapshot-private-value","java.version":"17"}`), nil + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Manifest.Complete).To(BeFalse()) + data, err := result.Zip() + Expect(err).NotTo(HaveOccurred()) + archive, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + Expect(err).NotTo(HaveOccurred()) + Expect(archive.File).NotTo(BeEmpty()) + for _, file := range archive.File { + stream, err := file.Open() + Expect(err).NotTo(HaveOccurred()) + content, err := io.ReadAll(stream) + Expect(err).NotTo(HaveOccurred()) + Expect(stream.Close()).To(Succeed()) + for _, collector := range result.Manifest.Collectors { + if collector.File == file.Name { + Expect(collector.SHA256).To(Equal(fmt.Sprintf("%x", sha256.Sum256(content)))) + } + } + } + for _, content := range result.Files { + Expect(string(content)).NotTo(ContainSubstring("snapshot-private-value")) + } + Expect(string(result.Files["summary.md"])).To(ContainSubstring("thread dump denied")) + }) + It("rejects invalid recording durations", func() { + Expect((Options{JFR: "-1s"}).Validate()).NotTo(Succeed()) + Expect((Options{JFR: "bogus"}).Validate()).NotTo(Succeed()) + }) + It("masks quoted assignments and embedded URLs without losing integer precision", func() { + data, err := sanitized([]byte(`{"arguments":["-Dexample.password=\"private with spaces\""],"environment":{"JAVA_TOOL_OPTIONS":"-Dauth.token=private-token"},"url":"https://user:private-pass@example.com/path","counter":9007199254740993}`), false) + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).NotTo(ContainSubstring("private")) + Expect(string(data)).To(ContainSubstring("9007199254740993")) + }) + It("does not invoke collectors after cancellation", func() { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + called := false + result, err := collect(ctx, Options{HeapHistogram: true}, WorkloadTarget{}, func(context.Context, string) (json.RawMessage, error) { called = true; return nil, nil }) + Expect(err).NotTo(HaveOccurred()) + Expect(called).To(BeFalse()) + Expect(result.Manifest.Complete).To(BeFalse()) + }) + It("masks JMX tabular system-property values", func() { + data, err := sanitized([]byte(`{"SystemProperties":[{"key":"example.password","value":"private-value"}]}`), false) + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).NotTo(ContainSubstring("private-value")) + }) + + It("masks settings properties whose camelCase key names a secret, and keeps the others", func() { + data, err := sanitized([]byte(`{"properties":{"javax.net.ssl.trustStorePassword":"private-store","javax.net.ssl.keyStorePassword":"private-key","example.apiKey":"private-api","java.version":"1.8.0_302"}}`), false) + Expect(err).NotTo(HaveOccurred()) + var settings struct{ Properties map[string]string } + Expect(json.Unmarshal(data, &settings)).To(Succeed()) + Expect(settings.Properties).To(Equal(map[string]string{"javax.net.ssl.trustStorePassword": "****", "javax.net.ssl.keyStorePassword": "****", + "example.apiKey": "****", "java.version": "1.8.0_302"})) + }) + + It("masks the value of a JMX key/value pair whose camelCase key names a secret", func() { + data, err := sanitized([]byte(`{"runtime":{"attributes":[{"value":[{"key":"javax.net.ssl.trustStorePassword","value":"private-store"},{"key":"java.version","value":"1.8.0_302"}]}]}}`), false) + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).NotTo(ContainSubstring("private-store")) + Expect(string(data)).To(ContainSubstring("1.8.0_302")) + }) +}) diff --git a/internal/jvm/snapshot/target.go b/internal/jvm/snapshot/target.go new file mode 100644 index 0000000..1e4d944 --- /dev/null +++ b/internal/jvm/snapshot/target.go @@ -0,0 +1,50 @@ +package snapshot + +import ( + "context" + "fmt" + "path/filepath" + "strconv" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/agentinstall" + "github.com/flanksource/telltale/pkg/datadir" +) + +func Resolve(ctx context.Context, agent string, pid int, javaHome string) (string, error) { + if (agent == "") == (pid == 0) || pid < 0 { + return "", fmt.Errorf("specify exactly one connected agent or positive local PID") + } + if agent != "" { + // Lookup connects a deployed target (deployer/…) or names a short one, + // as every other --agent does; ByID alone sees only connected agents. + connected, err := agentclient.Default.Lookup(ctx, agent, agentclient.Want{Runtime: "java"}) + if err != nil { + return "", err + } + return connected.Hello.AgentID, nil + } + if err := agentclient.Default.Discover(ctx, ""); err != nil { + return "", err + } + if connected := agentclient.Default.ByPID(pid); connected != nil { + return connected.Hello.AgentID, nil + } + directory, err := datadir.Cache() + if err != nil { + return "", err + } + result, err := (agentinstall.Installer{Runtime: agentinstall.Java{}}).Install(ctx, agentinstall.Options{ + Destination: agentinstall.Destination{Kind: "local"}, PID: pid, Directory: filepath.Join(directory, "java"), JavaHome: javaHome, Service: "java", + }) + if err != nil { + return "", err + } + if err := agentclient.Default.Discover(ctx, ""); err != nil { + return "", err + } + if result.Agent == "" { + return "", fmt.Errorf("no agent discovered for PID %s", strconv.Itoa(pid)) + } + return result.Agent, nil +} diff --git a/internal/jvm/snapshot/telemetry.go b/internal/jvm/snapshot/telemetry.go new file mode 100644 index 0000000..a04cf7d --- /dev/null +++ b/internal/jvm/snapshot/telemetry.go @@ -0,0 +1,119 @@ +package snapshot + +import ( + "bytes" + _ "embed" + "encoding/json" + "fmt" + "html/template" + "math" + "slices" + "strconv" + "time" +) + +//go:embed telemetry.html +var telemetryPage string + +var telemetryTemplate = template.Must(template.New("telemetry").Parse(telemetryPage)) + +type telemetrySnapshot struct { + CapturedAt time.Time `json:"capturedAt"` + Applications []telemetryApplication `json:"applications"` +} + +type telemetryApplication struct { + Loader string `json:"loader"` + Label string `json:"label"` + Status struct { + Armed bool `json:"armed"` + Mode string `json:"mode"` + SeriesCount int `json:"seriesCount"` + Errors int64 `json:"errors"` + AbandonedFrames int64 `json:"abandonedFrames"` + Overflow int64 `json:"overflow"` + LastError string `json:"lastError"` + } `json:"status"` + Histograms []telemetryHistogram `json:"histograms"` +} + +type telemetryHistogram struct { + Name string `json:"name"` + Labels map[string]string `json:"labels"` + Count int64 `json:"count"` + SumSeconds float64 `json:"sumSeconds"` + Buckets map[string]int64 `json:"buckets"` +} + +type telemetryBucket struct { + Bound string + Count int64 + limit float64 +} + +func (h telemetryHistogram) BucketRows() ([]telemetryBucket, error) { + rows := make([]telemetryBucket, 0, len(h.Buckets)) + for bound, count := range h.Buckets { + limit, err := strconv.ParseFloat(bound, 64) + if err != nil || math.IsNaN(limit) || limit < 0 || math.IsInf(limit, 1) && bound != "+Inf" || count < 0 || count > h.Count { + return nil, fmt.Errorf("invalid %s histogram bucket %q: %d", h.Name, bound, count) + } + rows = append(rows, telemetryBucket{Bound: bound, Count: count, limit: limit}) + } + slices.SortFunc(rows, func(a, b telemetryBucket) int { + if a.limit < b.limit { + return -1 + } + if a.limit > b.limit { + return 1 + } + return 0 + }) + for i := 1; i < len(rows); i++ { + if rows[i].Count < rows[i-1].Count { + return nil, fmt.Errorf("%s histogram buckets are not cumulative", h.Name) + } + } + return rows, nil +} + +func addTelemetryReport(result *Result) { + data, ok := result.Files["telemetry.json"] + var page []byte + var err error + if !ok { + err = fmt.Errorf("telemetry.json is unavailable; see its collector outcome") + } else { + page, err = telemetryHTML(data) + } + result.add("telemetry.html", page, err) +} + +func telemetryHTML(data []byte) ([]byte, error) { + var snapshot telemetrySnapshot + if err := json.Unmarshal(data, &snapshot); err != nil { + return nil, fmt.Errorf("decode native telemetry: %w", err) + } + if snapshot.CapturedAt.IsZero() || snapshot.Applications == nil { + return nil, fmt.Errorf("native telemetry requires capturedAt and an applications list") + } + for _, application := range snapshot.Applications { + if application.Loader == "" || application.Status.Mode == "" || application.Histograms == nil { + return nil, fmt.Errorf("native telemetry requires loader, status mode and histograms for every application") + } + for _, histogram := range application.Histograms { + infinity, exists := histogram.Buckets["+Inf"] + if histogram.Name == "" || histogram.Labels == nil || histogram.Count < 0 || histogram.SumSeconds < 0 || !exists || infinity != histogram.Count { + return nil, fmt.Errorf("invalid %s native telemetry histogram", histogram.Name) + } + } + } + var escaped bytes.Buffer + json.HTMLEscape(&escaped, data) + var output bytes.Buffer + err := telemetryTemplate.Execute(&output, struct { + telemetrySnapshot + JSON template.JS + }{snapshot, template.JS(escaped.String())}) + return output.Bytes(), err +} diff --git a/internal/jvm/snapshot/telemetry.html b/internal/jvm/snapshot/telemetry.html new file mode 100644 index 0000000..c13a30b --- /dev/null +++ b/internal/jvm/snapshot/telemetry.html @@ -0,0 +1,63 @@ + + + + + + Activity telemetry — JVM snapshot + + + +
+

Activity telemetry

+ + {{range .Applications}} +
+

{{.Label}}

+ +
+
Histogram series
{{.Status.SeriesCount}}
+
Errors
{{.Status.Errors}}
+
Abandoned frames
{{.Status.AbandonedFrames}}
+
Overflow
{{.Status.Overflow}}
+
+ {{if .Status.LastError}}
{{.Status.LastError}}
{{end}} + {{range .Histograms}} +
+

{{.Name}}

+
{{range $name, $value := .Labels}}{{$name}}={{$value}}{{end}}
+
+
Observations
{{.Count}}
+
Total duration (seconds)
{{.SumSeconds}}
+
+
+ Cumulative histogram buckets + + + {{range .BucketRows}}{{end}} +
Upper bound (seconds)Observations
{{.Bound}}{{.Count}}
+
+
+ {{else}}

No completed histogram series in this application.

{{end}} +
+ {{else}}

No loaded activity telemetry implementations were found in this JVM.

{{end}} +
+ + + diff --git a/internal/jvm/snapshot/telemetry_test.go b/internal/jvm/snapshot/telemetry_test.go new file mode 100644 index 0000000..1630e20 --- /dev/null +++ b/internal/jvm/snapshot/telemetry_test.go @@ -0,0 +1,77 @@ +package snapshot + +import ( + "context" + "encoding/json" + "errors" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const nativeTelemetryFixture = `{"capturedAt":"2026-11-16T15:00:00Z","applications":[{"loader":"loader-a","label":"PASJava","status":{"armed":true,"mode":"boot","seriesCount":1,"errors":0,"abandonedFrames":0,"overflow":0},"histograms":[{"name":"oipa_activity_processing_seconds","labels":{"transaction":"Premium ","outcome":"success","token":"private-label-value"},"count":9007199254740993,"sumSeconds":0.025,"buckets":{"0.005":0,"0.025":1,"+Inf":9007199254740993}}]}]}` + +var _ = Describe("native activity telemetry snapshot", func() { + It("archives the read-only agent data and HTML with the same inline JSON", func() { + var called []string + result, err := collect(context.Background(), Options{}, WorkloadTarget{}, func(_ context.Context, op string) (json.RawMessage, error) { + called = append(called, op) + if op == "jvm.telemetry" { + return json.RawMessage(nativeTelemetryFixture), nil + } + if op == "jvm.flags" || op == "jvm.threads.dump" { + return json.RawMessage(`"diagnostic text"`), nil + } + return json.RawMessage(`{}`), nil + }) + Expect(err).NotTo(HaveOccurred()) + Expect(called).To(ContainElement("jvm.telemetry")) + Expect(result.Manifest.Complete).To(BeTrue(), "%+v", result.Manifest.Collectors) + Expect(result.Files).To(HaveKey("telemetry.json")) + Expect(result.Files).To(HaveKey("telemetry.html")) + data, page := string(result.Files["telemetry.json"]), string(result.Files["telemetry.html"]) + Expect(data).To(ContainSubstring("9007199254740993")) + Expect(data).NotTo(ContainSubstring("private-label-value")) + Expect(page).NotTo(ContainSubstring("private-label-value")) + Expect(strings.Count(page, "")).To(Equal(1)) + Expect(page).To(ContainSubstring("PASJava")) + Expect(page).To(ContainSubstring("9007199254740993")) + island := strings.SplitN(strings.SplitN(page, `id="telemetry-data">`, 2)[1], "", 2)[0] + Expect(island).To(MatchJSON(data)) + Expect(island).To(ContainSubstring("9007199254740993")) + }) + + It("keeps a partial archive and explicit failures when telemetry cannot be read", func() { + result, err := collect(context.Background(), Options{}, WorkloadTarget{}, func(_ context.Context, op string) (json.RawMessage, error) { + if op == "jvm.telemetry" { + return nil, errors.New("native snapshot API is missing") + } + return json.RawMessage(`{}`), nil + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Manifest.Complete).To(BeFalse()) + Expect(result.Files).NotTo(HaveKey("telemetry.html")) + Expect(string(result.Files["summary.md"])).To(ContainSubstring("native snapshot API is missing")) + Expect(result.Files).To(HaveKey("metrics.json")) + }) + + It("opens without a server or external assets when no telemetry is loaded", func() { + page, err := telemetryHTML([]byte(`{"capturedAt":"2026-11-16T15:00:00Z","applications":[]}`)) + Expect(err).NotTo(HaveOccurred()) + Expect(string(page)).To(ContainSubstring("No loaded activity telemetry")) + Expect(string(page)).NotTo(ContainSubstring("src=")) + Expect(string(page)).NotTo(ContainSubstring("href=")) + }) + + DescribeTable("rejects invalid telemetry instead of rendering invented measurements", func(data string) { + _, err := telemetryHTML([]byte(data)) + Expect(err).To(HaveOccurred()) + }, Entry("missing applications", `{"capturedAt":"2026-11-16T15:00:00Z"}`), + Entry("null applications", `{"capturedAt":"2026-11-16T15:00:00Z","applications":null}`), + Entry("missing capture time", `{"applications":[]}`), + Entry("infinity differs from count", strings.Replace(nativeTelemetryFixture, `"+Inf":9007199254740993`, `"+Inf":1`, 1)), + Entry("negative sum", strings.Replace(nativeTelemetryFixture, `"sumSeconds":0.025`, `"sumSeconds":-0.025`, 1)), + Entry("decreasing cumulative buckets", strings.Replace(nativeTelemetryFixture, `"0.005":0`, `"0.005":2`, 1)), + Entry("invalid boundary", strings.Replace(nativeTelemetryFixture, `"0.005":0`, `"unknown":0`, 1))) +}) diff --git a/internal/jvm/snapshot/types.go b/internal/jvm/snapshot/types.go new file mode 100644 index 0000000..467a60b --- /dev/null +++ b/internal/jvm/snapshot/types.go @@ -0,0 +1,61 @@ +package snapshot + +import ( + "errors" + "fmt" + "time" +) + +type Options struct { + Agent string `json:"agent"` + JFR string `json:"jfr,omitempty"` + HeapHistogram bool `json:"heapHistogram,omitempty"` + // Workload is how long to sample for workload.html; empty disables it. + Workload string `json:"workload,omitempty"` +} + +func (o Options) Validate() error { + if o.JFR != "" { + duration, err := time.ParseDuration(o.JFR) + if err != nil || duration <= 0 || duration > 5*time.Minute { + return errors.New("jfr must be a positive duration at most 5m") + } + } + if o.Workload != "" { + window, err := time.ParseDuration(o.Workload) + if err != nil || window < workloadInterval || window > 5*time.Minute { + return fmt.Errorf("workload must be a duration from %s (the sampling interval) to 5m", workloadInterval) + } + } + return nil +} + +// workloadSamples is how many samples span the Workload window, 0 when disabled. +func (o Options) workloadSamples() int { + window, err := time.ParseDuration(o.Workload) + if err != nil { + return 0 + } + return int(window/workloadInterval) + 1 +} + +type Outcome struct { + File string `json:"file"` + Status string `json:"status"` + Error string `json:"error,omitempty"` + SHA256 string `json:"sha256,omitempty"` +} + +type Manifest struct { + Version int `json:"version"` + Agent string `json:"agent"` + StartedAt time.Time `json:"startedAt"` + CompletedAt time.Time `json:"completedAt"` + Complete bool `json:"complete"` + Collectors []Outcome `json:"collectors"` +} + +type Result struct { + Manifest Manifest + Files map[string][]byte +} diff --git a/internal/jvm/snapshot/workload.go b/internal/jvm/snapshot/workload.go new file mode 100644 index 0000000..e630624 --- /dev/null +++ b/internal/jvm/snapshot/workload.go @@ -0,0 +1,330 @@ +package snapshot + +import ( + "bytes" + "context" + _ "embed" + "encoding/json" + "fmt" + "math" + "slices" + "strconv" + "strings" + "time" + + "github.com/flanksource/telltale/pkg/tracing/jvm/jvmruntime" +) + +// workloadPage renders workload.json with the published clicky-ui WorkloadPanel. +// +//go:embed workload.html +var workloadPage []byte + +// workloadMarker is the page's empty data island; workloadHTML fills it. +const workloadMarker = `` + +// workloadInterval is the time between workload samples. +var workloadInterval = 5 * time.Second + +// WorkloadTarget names the container the workload panel describes. +type WorkloadTarget struct { + Kind string `json:"kind,omitempty"` + Name string `json:"name"` + Namespace string `json:"namespace,omitempty"` + Container string `json:"container"` +} + +// workloadTarget reads the Kubernetes resource attributes an agent reports. +func workloadTarget(resource map[string]string, service string) WorkloadTarget { + target := WorkloadTarget{Name: resource["telltale.deployer"], Namespace: resource["k8s.namespace.name"], Container: resource["k8s.container.name"]} + if pod := resource["k8s.pod.name"]; target.Name == "" && pod != "" { + target.Kind, target.Name = "pod", pod + } + if target.Name == "" { + target.Name = service + } + if target.Name == "" { + target.Name = "jvm" + } + if target.Container == "" { + target.Container = target.Name + } + return target +} + +// workloadSample is one poll of the container and JVM the panel summarises. +type workloadSample struct { + At time.Time `json:"at"` + CPUCores float64 `json:"cpuCores"` + CPULimit float64 `json:"cpuLimit"` + Memory int64 `json:"memory"` + MemoryLimit int64 `json:"memoryLimit"` + Eden int64 `json:"eden"` + Survivor int64 `json:"survivor"` + Old int64 `json:"old"` + HeapMax int64 `json:"heapMax"` + Metaspace int64 `json:"metaspace"` + CodeCache int64 `json:"codeCache"` + Direct int64 `json:"direct"` + GCCount int64 `json:"gcCount"` + GCTimeMs int64 `json:"gcTimeMs"` + Live int `json:"live"` + Daemon int `json:"daemon"` + Blocked int `json:"blocked"` +} + +func sampleWorkload(ctx context.Context, call caller) (workloadSample, error) { + raw := map[string]json.RawMessage{} + for _, op := range []string{jvmruntime.MetricsOp, jvmruntime.ResourcesOp, jvmruntime.ThreadsOp} { + bounded, cancel := context.WithTimeout(ctx, 30*time.Second) + data, err := call(bounded, op) + cancel() + if err != nil { + return workloadSample{}, fmt.Errorf("%s: %w", op, err) + } + raw[op] = data + } + metrics, err := jvmruntime.DecodeMetrics(raw[jvmruntime.MetricsOp]) + if err != nil { + return workloadSample{}, err + } + resources, err := jvmruntime.DecodeResources(raw[jvmruntime.ResourcesOp]) + if err != nil { + return workloadSample{}, err + } + threads, err := jvmruntime.DecodeThreads(raw[jvmruntime.ThreadsOp]) + if err != nil { + return workloadSample{}, err + } + return newWorkloadSample(metrics, resources, threads) +} + +// newWorkloadSample reads container usage from the cgroup v2 files and JVM +// usage from its memory pools. Heap pools that are neither eden nor survivor +// (old, tenured, or a single-generation ZGC/Shenandoah heap) count as old. +func newWorkloadSample(metrics jvmruntime.Metrics, resources jvmruntime.Resources, threads jvmruntime.ThreadSample) (workloadSample, error) { + if metrics.ProcessCPUPercent == nil { + return workloadSample{}, fmt.Errorf("%s has no processCpuPercent", jvmruntime.MetricsOp) + } + cgroup := resources.Cgroups["/sys/fs/cgroup"] + if cgroup == nil { + return workloadSample{}, fmt.Errorf("%s has no cgroup v2 at /sys/fs/cgroup; the workload panel needs a container's cgroup", jvmruntime.ResourcesOp) + } + processors := float64(metrics.Runtime.Processors) + sample := workloadSample{At: metrics.CapturedAt, CPUCores: *metrics.ProcessCPUPercent / 100 * processors, CPULimit: processors} + cpuMax := strings.Fields(cgroup["cpu.max"]) + if len(cpuMax) != 2 { + return workloadSample{}, fmt.Errorf("cgroup cpu.max %q: want \" \"", cgroup["cpu.max"]) + } + if cpuMax[0] != "max" { + quota, quotaErr := strconv.ParseFloat(cpuMax[0], 64) + period, periodErr := strconv.ParseFloat(cpuMax[1], 64) + if quotaErr != nil || periodErr != nil || period <= 0 { + return workloadSample{}, fmt.Errorf("cgroup cpu.max %q: want numeric quota and period", cgroup["cpu.max"]) + } + sample.CPULimit = quota / period + } + var err error + if sample.Memory, err = strconv.ParseInt(strings.TrimSpace(cgroup["memory.current"]), 10, 64); err != nil { + return workloadSample{}, fmt.Errorf("cgroup memory.current: %w", err) + } + if limit := strings.TrimSpace(cgroup["memory.max"]); limit != "max" { + if sample.MemoryLimit, err = strconv.ParseInt(limit, 10, 64); err != nil { + return workloadSample{}, fmt.Errorf("cgroup memory.max: %w", err) + } + } + for _, pool := range metrics.Memory.Heap { + name := strings.ToLower(pool.Name) + switch { + case strings.Contains(name, "eden"): + sample.Eden += pool.Used + case strings.Contains(name, "survivor"): + sample.Survivor += pool.Used + default: + sample.Old += pool.Used + } + sample.HeapMax += max(0, pool.Max) + } + for _, pool := range metrics.Memory.NonHeap { + name := strings.ToLower(pool.Name) + switch { + case name == "metaspace": + sample.Metaspace += pool.Used + case strings.Contains(name, "code"): + sample.CodeCache += pool.Used + } + } + for _, pool := range metrics.Memory.Buffer { + if pool.Name == "direct" { + sample.Direct += pool.Used + } + } + for _, collector := range metrics.GarbageCollectors { + sample.GCCount += collector.CollectionCount + sample.GCTimeMs += collector.CollectionTime + } + sample.Live = len(threads.Threads) + for _, thread := range threads.Threads { + if thread.Daemon { + sample.Daemon++ + } + if thread.State == "BLOCKED" { + sample.Blocked++ + } + } + return sample, nil +} + +// statistics is the clicky-ui WorkloadResourceStatistics: CPU in cores, memory in bytes. +type statistics struct { + Average float64 `json:"average"` + P90 float64 `json:"p90"` + P95 float64 `json:"p95"` + Limit float64 `json:"limit"` +} + +type gcStatistics struct { + Count int64 `json:"count"` + TimeMs int64 `json:"timeMs"` + P90Ms float64 `json:"p90Ms"` + P95Ms float64 `json:"p95Ms"` + MaxMs float64 `json:"maxMs"` + PercentP90 float64 `json:"percentP90"` + PercentP95 float64 `json:"percentP95"` +} + +// workloadPanel is the clicky-ui WorkloadPanelProps the page renders. +type workloadPanel struct { + Workload map[string]any `json:"workload"` + Container string `json:"container"` + ObservationWindowMs int64 `json:"observationWindowMs"` + CPU statistics `json:"cpu"` + Memory statistics `json:"memory"` + JVM struct { + Heap statistics `json:"heap"` + Regions map[string]float64 `json:"regions"` + GC gcStatistics `json:"gc"` + Threads map[string]int `json:"threads"` + } `json:"jvm"` +} + +func percentile(values []float64, p float64) float64 { + if len(values) == 0 { + return 0 + } + sorted := slices.Sorted(slices.Values(values)) + return sorted[max(0, int(math.Ceil(p/100*float64(len(sorted))))-1)] +} + +func summarise(samples []workloadSample, value func(workloadSample) float64, limit float64) statistics { + values := make([]float64, len(samples)) + sum := 0.0 + for i, sample := range samples { + values[i] = value(sample) + sum += values[i] + } + return statistics{Average: sum / float64(len(values)), P90: percentile(values, 90), P95: percentile(values, 95), Limit: limit} +} + +// newWorkloadPanel summarises samples over their window. GC pause figures are +// the average pause within each sampling interval, since the JVM only exposes +// cumulative collection counts and times. +func newWorkloadPanel(target WorkloadTarget, samples []workloadSample) (workloadPanel, error) { + if len(samples) < 2 { + return workloadPanel{}, fmt.Errorf("the workload panel needs at least 2 samples, got %d", len(samples)) + } + first, last := samples[0], samples[len(samples)-1] + panel := workloadPanel{ + Workload: map[string]any{"name": target.Name, "status": map[string]string{"label": "Running", "health": "healthy"}}, + Container: target.Container, + ObservationWindowMs: last.At.Sub(first.At).Milliseconds(), + CPU: summarise(samples, func(s workloadSample) float64 { return s.CPUCores }, last.CPULimit), + Memory: summarise(samples, func(s workloadSample) float64 { return float64(s.Memory) }, float64(last.MemoryLimit)), + } + if panel.ObservationWindowMs <= 0 { + return workloadPanel{}, fmt.Errorf("workload samples span %s; want a positive window", last.At.Sub(first.At)) + } + if target.Kind != "" { + panel.Workload["kind"] = target.Kind + } + if target.Namespace != "" { + panel.Workload["namespace"] = target.Namespace + } + average := func(value func(workloadSample) int64) float64 { + return summarise(samples, func(s workloadSample) float64 { return float64(value(s)) }, 0).Average + } + panel.JVM.Regions = map[string]float64{ + "eden": average(func(s workloadSample) int64 { return s.Eden }), + "survivor": average(func(s workloadSample) int64 { return s.Survivor }), + "old": average(func(s workloadSample) int64 { return s.Old }), + "metaspace": average(func(s workloadSample) int64 { return s.Metaspace }), + "codeCache": average(func(s workloadSample) int64 { return s.CodeCache }), + "direct": average(func(s workloadSample) int64 { return s.Direct }), + } + panel.JVM.Heap = summarise(samples, func(s workloadSample) float64 { return float64(s.Eden + s.Survivor + s.Old) }, float64(last.HeapMax)) + regions := 0.0 + for _, bytes := range panel.JVM.Regions { + regions += bytes + } + if regions > panel.Memory.Average { + return workloadPanel{}, fmt.Errorf("JVM memory regions average %.0f bytes, more than the container's %.0f", regions, panel.Memory.Average) + } + var pauses, percents []float64 + for i, sample := range samples[1:] { + count, timeMs := sample.GCCount-samples[i].GCCount, sample.GCTimeMs-samples[i].GCTimeMs + if count < 0 || timeMs < 0 { + return workloadPanel{}, fmt.Errorf("GC totals went backwards between %s and %s; the JVM restarted", samples[i].At, sample.At) + } + if count == 0 { + continue + } + panel.JVM.GC.Count += count + panel.JVM.GC.TimeMs += timeMs + pauses = append(pauses, float64(timeMs)/float64(count)) + percents = append(percents, min(100, float64(timeMs)/float64(max(1, sample.At.Sub(samples[i].At).Milliseconds()))*100)) + } + if len(pauses) > 0 { + panel.JVM.GC.P90Ms, panel.JVM.GC.P95Ms, panel.JVM.GC.MaxMs = percentile(pauses, 90), percentile(pauses, 95), slices.Max(pauses) + panel.JVM.GC.PercentP90, panel.JVM.GC.PercentP95 = percentile(percents, 90), percentile(percents, 95) + } + peak := 0 + for _, sample := range samples { + peak = max(peak, sample.Live) + } + panel.JVM.Threads = map[string]int{"live": last.Live, "daemon": last.Daemon, "blocked": last.Blocked, "peak": peak} + return panel, nil +} + +// collectWorkload samples count times, workloadInterval apart. +func collectWorkload(ctx context.Context, call caller, count int) ([]workloadSample, error) { + var samples []workloadSample + for i := range count { + if i > 0 { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(workloadInterval): + } + } + sample, err := sampleWorkload(ctx, call) + if err != nil { + return nil, err + } + samples = append(samples, sample) + } + return samples, nil +} + +// workloadHTML fills the page's data island; json.Marshal escapes <, > and &, +// so no value can close the script element. +func workloadHTML(panel workloadPanel) ([]byte, error) { + if !bytes.Contains(workloadPage, []byte(workloadMarker)) { + return nil, fmt.Errorf("workload.html has no %s", workloadMarker) + } + data, err := json.Marshal(panel) + if err != nil { + return nil, err + } + island := strings.Replace(workloadMarker, ">null<", ">"+string(data)+"<", 1) + return bytes.Replace(workloadPage, []byte(workloadMarker), []byte(island), 1), nil +} diff --git a/internal/jvm/snapshot/workload.html b/internal/jvm/snapshot/workload.html new file mode 100644 index 0000000..83feb65 --- /dev/null +++ b/internal/jvm/snapshot/workload.html @@ -0,0 +1,67 @@ + + + + + + Workload panel + + + + + + +

Loading clicky-ui…

+ + + + + + + + + + diff --git a/internal/jvm/snapshot/workload_test.go b/internal/jvm/snapshot/workload_test.go new file mode 100644 index 0000000..99acf71 --- /dev/null +++ b/internal/jvm/snapshot/workload_test.go @@ -0,0 +1,139 @@ +package snapshot + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/telltale/pkg/tracing/jvm/jvmruntime" +) + +func agentFixture(op string) json.RawMessage { + data, err := os.ReadFile(filepath.Join("..", "..", "..", "pkg", "tracing", "jvm", "jvmruntime", "testdata", op+".json")) + Expect(err).NotTo(HaveOccurred()) + return data +} + +var _ = Describe("workload panel", func() { + start := time.Date(2026, 11, 16, 12, 0, 0, 0, time.UTC) + sample := func(offset time.Duration, cpu float64, memory, eden, gcCount, gcTimeMs int64, live, daemon, blocked int) workloadSample { + return workloadSample{At: start.Add(offset), CPUCores: cpu, CPULimit: 2, Memory: memory, MemoryLimit: 4000, Eden: eden, Survivor: 10, Old: 200, + HeapMax: 1000, Metaspace: 50, CodeCache: 20, Direct: 5, GCCount: gcCount, GCTimeMs: gcTimeMs, Live: live, Daemon: daemon, Blocked: blocked} + } + series := []workloadSample{ + sample(0, 0.5, 1000, 100, 0, 0, 10, 5, 0), + sample(5*time.Second, 1.0, 1200, 200, 2, 20, 12, 5, 0), + sample(10*time.Second, 1.5, 1400, 300, 3, 50, 11, 6, 1), + } + + It("reads container usage from cgroup v2 and JVM usage from the agent's pools", func() { + metrics, err := jvmruntime.DecodeMetrics(agentFixture(jvmruntime.MetricsOp)) + Expect(err).NotTo(HaveOccurred()) + resources, err := jvmruntime.DecodeResources(agentFixture(jvmruntime.ResourcesOp)) + Expect(err).NotTo(HaveOccurred()) + threads, err := jvmruntime.DecodeThreads(agentFixture(jvmruntime.ThreadsOp)) + Expect(err).NotTo(HaveOccurred()) + got, err := newWorkloadSample(metrics, resources, threads) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(Equal(workloadSample{ + At: metrics.CapturedAt, CPUCores: 0.09098106439764791 / 100 * 12, CPULimit: 2, Memory: 612368384, MemoryLimit: 2147483648, + Eden: 8388608, Survivor: 17704008, Old: 17925592, HeapMax: 8589934592, + Metaspace: 55839576, CodeCache: 1739520 + 14589696 + 1571264, Direct: 0, + GCCount: 15 + 6 + 1, GCTimeMs: 45 + 5 + 21, Live: 11, Daemon: 10, Blocked: 0, + })) + }) + + It("refuses a JVM that runs outside a cgroup v2 container", func() { + metrics, err := jvmruntime.DecodeMetrics(agentFixture(jvmruntime.MetricsOp)) + Expect(err).NotTo(HaveOccurred()) + _, err = newWorkloadSample(metrics, jvmruntime.Resources{Linux: map[string]string{"status": "not-applicable"}}, jvmruntime.ThreadSample{}) + Expect(err).To(MatchError(ContainSubstring("no cgroup v2"))) + }) + + It("summarises averages, percentiles, per-interval GC pauses and the thread peak", func() { + panel, err := newWorkloadPanel(WorkloadTarget{Kind: "pod", Name: "api-0", Namespace: "lab", Container: "api"}, series) + Expect(err).NotTo(HaveOccurred()) + Expect(panel.Workload).To(HaveKeyWithValue("kind", "pod")) + Expect(panel.Workload).To(HaveKeyWithValue("namespace", "lab")) + Expect(panel.Container).To(Equal("api")) + Expect(panel.ObservationWindowMs).To(Equal(int64(10000))) + Expect(panel.CPU).To(Equal(statistics{Average: 1, P90: 1.5, P95: 1.5, Limit: 2})) + Expect(panel.Memory).To(Equal(statistics{Average: 1200, P90: 1400, P95: 1400, Limit: 4000})) + Expect(panel.JVM.Heap).To(Equal(statistics{Average: 410, P90: 510, P95: 510, Limit: 1000})) + Expect(panel.JVM.Regions).To(Equal(map[string]float64{"eden": 200, "survivor": 10, "old": 200, "metaspace": 50, "codeCache": 20, "direct": 5})) + // Intervals: 2 collections in 20ms (10ms each, 0.4% of 5s), then 1 in 30ms (0.6%). + Expect(panel.JVM.GC).To(Equal(gcStatistics{Count: 3, TimeMs: 50, P90Ms: 30, P95Ms: 30, MaxMs: 30, PercentP90: 0.6, PercentP95: 0.6})) + Expect(panel.JVM.Threads).To(Equal(map[string]int{"live": 11, "daemon": 6, "blocked": 1, "peak": 12})) + }) + + It("refuses JVM regions larger than the container's memory", func() { + small := []workloadSample{sample(0, 1, 100, 100, 0, 0, 1, 0, 0), sample(time.Second, 1, 100, 100, 0, 0, 1, 0, 0)} + _, err := newWorkloadPanel(WorkloadTarget{Name: "api", Container: "api"}, small) + Expect(err).To(MatchError(ContainSubstring("more than the container's"))) + }) + + It("refuses GC totals that go backwards", func() { + restarted := []workloadSample{sample(0, 1, 1000, 100, 5, 50, 1, 0, 0), sample(time.Second, 1, 1000, 100, 1, 5, 1, 0, 0)} + _, err := newWorkloadPanel(WorkloadTarget{Name: "api", Container: "api"}, restarted) + Expect(err).To(MatchError(ContainSubstring("went backwards"))) + }) + + It("embeds the panel so no value can close its script element", func() { + panel, err := newWorkloadPanel(WorkloadTarget{Name: "", Container: "api"}, series) + Expect(err).NotTo(HaveOccurred()) + page, err := workloadHTML(panel) + Expect(err).NotTo(HaveOccurred()) + Expect(string(page)).NotTo(ContainSubstring(workloadMarker)) + Expect(strings.Count(string(page), "")).To(Equal(strings.Count(string(workloadPage), ""))) + island := strings.SplitN(strings.SplitN(string(page), `id="workload">`, 2)[1], "", 2)[0] + var embedded workloadPanel + Expect(json.Unmarshal([]byte(island), &embedded)).To(Succeed()) + Expect(embedded.Workload["name"]).To(Equal("")) + }) + + It("names the workload from the agent's Kubernetes resource attributes", func() { + Expect(workloadTarget(map[string]string{"telltale.deployer": "api", "k8s.namespace.name": "lab", "k8s.container.name": "app", "k8s.pod.name": "api-1"}, "")). + To(Equal(WorkloadTarget{Name: "api", Namespace: "lab", Container: "app"})) + Expect(workloadTarget(map[string]string{"k8s.pod.name": "cycle-0"}, "")).To(Equal(WorkloadTarget{Kind: "pod", Name: "cycle-0", Container: "cycle-0"})) + Expect(workloadTarget(nil, "billing")).To(Equal(WorkloadTarget{Name: "billing", Container: "billing"})) + }) + + It("rejects workload windows outside the sampling interval to 5m", func() { + Expect((Options{Workload: "1s"}).Validate()).NotTo(Succeed()) + Expect((Options{Workload: "6m"}).Validate()).NotTo(Succeed()) + Expect((Options{Workload: "1m"}).Validate()).To(Succeed()) + Expect((Options{Workload: "1m"}).workloadSamples()).To(Equal(13)) + }) + + It("adds workload.json and workload.html to the archive", func() { + original := workloadInterval + workloadInterval = time.Millisecond + DeferCleanup(func() { workloadInterval = original }) + polls := 0 + result, err := collect(context.Background(), Options{Workload: "2ms"}, WorkloadTarget{Name: "api", Container: "api"}, func(_ context.Context, op string) (json.RawMessage, error) { + switch op { + case jvmruntime.ResourcesOp, jvmruntime.ThreadsOp: + return agentFixture(op), nil + case jvmruntime.MetricsOp: + default: + return json.RawMessage(`{}`), nil + } + var metrics map[string]any + Expect(json.Unmarshal(agentFixture(op), &metrics)).To(Succeed()) + polls++ + metrics["capturedAt"] = start.Add(time.Duration(polls) * time.Second) + return json.Marshal(metrics) + }) + Expect(err).NotTo(HaveOccurred()) + var samples []workloadSample + Expect(json.Unmarshal(result.Files["workload.json"], &samples)).To(Succeed()) + Expect(samples).To(HaveLen(3)) + Expect(string(result.Files["workload.html"])).To(ContainSubstring(`"observationWindowMs":2000`)) + }) +}) diff --git a/internal/jvm/top/source.go b/internal/jvm/top/source.go new file mode 100644 index 0000000..eff4d09 --- /dev/null +++ b/internal/jvm/top/source.go @@ -0,0 +1,127 @@ +package top + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "sync" + "time" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/tracing/jvm/jvmruntime" +) + +type Source struct { + mu sync.Mutex + Agent string + Server string + local *agentclient.Agent +} + +func (s *Source) Next(ctx context.Context) (Frame, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := ctx.Err(); err != nil { + return Frame{}, err + } + if s.Server != "" { + var frame Frame + err := s.remote(ctx, "/api/v1/jvm/top", map[string]any{"agent": s.Agent, "once": true, "format": "json"}, &frame, true) + if err == nil && (frame.AgentID == "" || frame.SampledAt.IsZero() || frame.Threads.Threads == nil) { + err = fmt.Errorf("server returned an incomplete JVM Top frame") + } + if err == nil { + s.Agent = frame.AgentID + } + return frame, err + } + if s.local == nil { + agent, err := agentclient.Default.Lookup(ctx, s.Agent, agentclient.Want{Runtime: "java"}) + if err != nil { + return Frame{}, err + } + if agent.Hello.Runtime.Lang != "java" { + return Frame{}, fmt.Errorf("agent %q is not Java", agent.Hello.AgentID) + } + s.local, s.Agent = agent, agent.Hello.AgentID + } + return Sample(ctx, s.Agent, s.local.Call) +} + +func (s *Source) Stack(ctx context.Context, id int64) (jvmruntime.ThreadStack, error) { + s.mu.Lock() + defer s.mu.Unlock() + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + if id <= 0 { + return jvmruntime.ThreadStack{}, fmt.Errorf("thread ID must be positive") + } + if s.Server != "" { + var stack jvmruntime.ThreadStack + err := s.remote(ctx, "/api/agents/"+url.PathEscape(s.Agent)+"/call", map[string]any{"operation": "jvm.threads.stack", "params": map[string]any{"id": id}}, &stack, false) + return stack, err + } + if s.local == nil { + return jvmruntime.ThreadStack{}, fmt.Errorf("sample JVM before inspecting a stack") + } + raw, err := s.local.Call(ctx, agentclient.CallRequest{Op: jvmruntime.ThreadStackOp, Params: json.RawMessage(fmt.Sprintf(`{"id":%d}`, id))}) + if err != nil { + return jvmruntime.ThreadStack{}, err + } + return jvmruntime.DecodeThreadStack(raw) +} + +func (s *Source) remote(ctx context.Context, path string, body any, result any, command bool) (err error) { + address, err := url.Parse(s.Server) + if err != nil || address.Host == "" || (address.Scheme != "http" && address.Scheme != "https") || address.User != nil || address.RawQuery != "" || address.Fragment != "" { + return fmt.Errorf("invalid server URL") + } + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + data, err := json.Marshal(body) + if err != nil { + return err + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(s.Server, "/")+path, bytes.NewReader(data)) + if err != nil { + return err + } + request.Header.Set("Content-Type", "application/json") + if token := os.Getenv("TELLTALE_API_TOKEN"); token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + response, err := http.DefaultClient.Do(request) + if err != nil { + return err + } + defer func() { err = errors.Join(err, response.Body.Close()) }() + data, err = io.ReadAll(io.LimitReader(response.Body, 16<<20)) + if err != nil { + return err + } + if response.StatusCode != http.StatusOK { + return fmt.Errorf("JVM Top HTTP %d: %s", response.StatusCode, data) + } + if command { + var envelope struct { + Success bool `json:"success"` + Output string `json:"output"` + Error string `json:"error"` + } + if err := json.Unmarshal(data, &envelope); err != nil { + return err + } + if !envelope.Success { + return fmt.Errorf("JVM Top: %s", envelope.Error) + } + data = []byte(envelope.Output) + } + return json.Unmarshal(data, result) +} diff --git a/internal/jvm/top/table.go b/internal/jvm/top/table.go new file mode 100644 index 0000000..798b515 --- /dev/null +++ b/internal/jvm/top/table.go @@ -0,0 +1,38 @@ +package top + +import ( + "fmt" + "github.com/flanksource/clicky/api" + "github.com/flanksource/telltale/pkg/tracing/jvm/jvmruntime" +) + +type Thread struct{ jvmruntime.Thread } + +func (Thread) Columns() []api.ColumnDef { + return []api.ColumnDef{ + api.Column("id").Label("ID").Build(), api.Column("cpu").Label("CPU%/core").Build(), + api.Column("state").Label("State").Build(), api.Column("name").Label("Thread").MaxWidth(50).Build(), + api.Column("cpuTime").Label("CPU ms").Build(), api.Column("blocked").Label("Blocked").Build(), + api.Column("waited").Label("Waited").Build(), api.Column("owner").Label("Lock owner").Build(), + } +} +func (t Thread) Row() map[string]any { + cpu, cpuTime, owner := "unavailable", "unavailable", "—" + if t.CPUPercent != nil { + cpu = fmt.Sprintf("%.1f", *t.CPUPercent) + } + if t.CPUTimeMs != nil { + cpuTime = fmt.Sprint(*t.CPUTimeMs) + } + if t.LockOwnerName != nil { + owner = *t.LockOwnerName + } + return map[string]any{"id": t.ID, "cpu": cpu, "state": t.State, "name": t.Name, "cpuTime": cpuTime, "blocked": fmt.Sprint(t.BlockedCount), "waited": fmt.Sprint(t.WaitedCount), "owner": owner} +} +func Table(rows []jvmruntime.Thread) api.TextTable { + display := make([]Thread, len(rows)) + for i, row := range rows { + display[i] = Thread{row} + } + return api.NewTableFrom(display) +} diff --git a/internal/jvm/top/top.go b/internal/jvm/top/top.go new file mode 100644 index 0000000..1405f56 --- /dev/null +++ b/internal/jvm/top/top.go @@ -0,0 +1,126 @@ +// Package top samples one Java runtime for terminal and web dashboards. +package top + +import ( + "cmp" + "context" + "encoding/json" + "fmt" + "slices" + "strings" + "time" + + "github.com/flanksource/clicky/api" + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/tracing/jvm/jvmruntime" +) + +type Frame struct { + AgentID string `json:"agentId"` + SampledAt time.Time `json:"sampledAt"` + Metrics jvmruntime.Metrics `json:"metrics"` + Threads jvmruntime.ThreadSample `json:"threads"` +} + +type Caller func(context.Context, agentclient.CallRequest) (json.RawMessage, error) + +func Sample(ctx context.Context, id string, call Caller) (Frame, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + frame := Frame{AgentID: id} + raw, err := call(ctx, agentclient.CallRequest{Op: jvmruntime.MetricsOp, Params: json.RawMessage(`{}`)}) + if err != nil { + return frame, fmt.Errorf("JVM metrics: %w", err) + } + if frame.Metrics, err = jvmruntime.DecodeMetrics(raw); err != nil { + return frame, err + } + raw, err = call(ctx, agentclient.CallRequest{Op: jvmruntime.ThreadsOp, Params: json.RawMessage(`{"sampleMs":200}`)}) + if err != nil { + return frame, fmt.Errorf("JVM threads: %w", err) + } + if frame.Threads, err = jvmruntime.DecodeThreads(raw); err != nil { + return frame, err + } + frame.SampledAt = time.Now().UTC() + return frame, nil +} + +func Order(rows []jvmruntime.Thread, by string, reverse bool, query string) []jvmruntime.Thread { + filtered := make([]jvmruntime.Thread, 0, len(rows)) + for _, row := range rows { + if strings.Contains(strings.ToLower(fmt.Sprintf("%d %s %s", row.ID, row.Name, row.State)), strings.ToLower(query)) { + filtered = append(filtered, row) + } + } + slices.SortFunc(filtered, func(a, b jvmruntime.Thread) int { + var order int + switch by { + case "name": + order = cmp.Compare(a.Name, b.Name) + case "state": + order = cmp.Compare(a.State, b.State) + case "id": + order = cmp.Compare(a.ID, b.ID) + default: + if a.CPUPercent == nil && b.CPUPercent != nil { + return 1 + } + if a.CPUPercent != nil && b.CPUPercent == nil { + return -1 + } + if a.CPUPercent != nil && b.CPUPercent != nil { + order = cmp.Compare(*b.CPUPercent, *a.CPUPercent) + } + } + if reverse { + order = -order + } + if order == 0 { + return cmp.Compare(a.ID, b.ID) + } + return order + }) + return filtered +} + +func (f Frame) Summary() string { + cpu := "unavailable" + if f.Metrics.ProcessCPUPercent != nil { + cpu = fmt.Sprintf("%.1f%%", *f.Metrics.ProcessCPUPercent) + } + out := fmt.Sprintf("JVM Top · %s · Java %s · uptime %s\nCPU %s of capacity · %d threads · sampled %s", f.AgentID, f.Metrics.Runtime.JavaVersion, (time.Duration(f.Metrics.Runtime.Uptime) * time.Millisecond).Round(time.Second), cpu, len(f.Threads.Threads), f.SampledAt.Format(time.TimeOnly)) + for _, area := range []struct { + name string + pools []jvmruntime.MemoryPool + }{{"Heap", f.Metrics.Memory.Heap}, {"Non-heap", f.Metrics.Memory.NonHeap}, {"Buffers", f.Metrics.Memory.Buffer}} { + var used, committed int64 + for _, pool := range area.pools { + used += pool.Used + committed += pool.Committed + } + out += fmt.Sprintf("\n%s %s used / %s committed", area.name, api.HumanizeBytes(used), api.HumanizeBytes(committed)) + } + for _, gc := range f.Metrics.GarbageCollectors { + out += fmt.Sprintf("\nGC %s: %d collections / %dms (lifetime)", gc.Name, gc.CollectionCount, gc.CollectionTime) + } + states := map[string]int{} + for _, thread := range f.Threads.Threads { + states[thread.State]++ + } + out += "\nThreads" + for _, state := range []string{"RUNNABLE", "BLOCKED", "WAITING", "TIMED_WAITING"} { + out += fmt.Sprintf(" · %s %d", state, states[state]) + } + if !f.Threads.CPUTimeEnabled { + out += "\nThread CPU unavailable: JVM CPU timing disabled" + } + if len(f.Threads.Deadlocked) > 0 { + out += fmt.Sprintf("\nDEADLOCK: threads %v", f.Threads.Deadlocked) + } + return out +} + +func (f Frame) Pretty() api.Text { + return api.Text{}.Append(f.Summary()).NewLine().Add(Table(Order(f.Threads.Threads, "cpu", false, ""))) +} diff --git a/internal/jvm/top/top_test.go b/internal/jvm/top/top_test.go new file mode 100644 index 0000000..5ab83b8 --- /dev/null +++ b/internal/jvm/top/top_test.go @@ -0,0 +1,58 @@ +package top + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/tracing/jvm/jvmruntime" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestTop(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "JVM Top") } + +var _ = Describe("JVM Top", func() { + It("decodes both operations and preserves unavailable CPU", func() { + frame, err := Sample(context.Background(), "fixture", func(_ context.Context, request agentclient.CallRequest) (json.RawMessage, error) { + return os.ReadFile("../../../pkg/tracing/jvm/jvmruntime/testdata/" + request.Op + ".json") + }) + Expect(err).NotTo(HaveOccurred()) + Expect(frame.AgentID).To(Equal("fixture")) + Expect(frame.Threads.Threads).NotTo(BeEmpty()) + Expect(frame.Metrics.Memory.Heap).NotTo(BeNil()) + }) + It("rejects malformed agent responses", func() { + _, err := Sample(context.Background(), "fixture", func(context.Context, agentclient.CallRequest) (json.RawMessage, error) { + return json.RawMessage(`{}`), nil + }) + Expect(err).To(HaveOccurred()) + }) + It("propagates cancellation and operation failures without making an empty frame", func() { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := Sample(ctx, "fixture", func(ctx context.Context, _ agentclient.CallRequest) (json.RawMessage, error) { return nil, ctx.Err() }) + Expect(err).To(MatchError(ContainSubstring("context canceled"))) + }) + It("rejects failed and incomplete remote snapshots", func() { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + Expect(json.NewEncoder(w).Encode(map[string]any{"success": true, "output": "{}"})).To(Succeed()) + })) + defer server.Close() + source := Source{Server: server.URL} + _, err := source.Next(context.Background()) + Expect(err).To(MatchError(ContainSubstring("incomplete"))) + _, err = source.Stack(context.Background(), 0) + Expect(err).To(MatchError(ContainSubstring("positive"))) + }) + It("sorts available CPU first with deterministic ties", func() { + cpu := 25.0 + rows := []jvmruntime.Thread{{ID: 3}, {ID: 2, CPUPercent: &cpu}, {ID: 1, CPUPercent: &cpu}} + Expect(Order(rows, "cpu", false, "")).To(Equal([]jvmruntime.Thread{rows[2], rows[1], rows[0]})) + Expect(Order(rows, "cpu", true, "3")).To(Equal(rows[:1])) + }) +}) diff --git a/internal/jvm/top/tui/model.go b/internal/jvm/top/tui/model.go new file mode 100644 index 0000000..4e73bae --- /dev/null +++ b/internal/jvm/top/tui/model.go @@ -0,0 +1,193 @@ +package tui + +import ( + "context" + "strconv" + "time" + + tea "charm.land/bubbletea/v2" + "github.com/flanksource/clicky/api" + "github.com/flanksource/clicky/tui/table" + "github.com/flanksource/telltale/internal/jvm/top" +) + +type Model struct { + ctx context.Context + source *top.Source + every time.Duration + frame *top.Frame + table table.Model + inspection *table.Model + width, height, sortIndex int + query string + paused, reverse, typing, busy, help bool + generation int + err error +} +type frameMsg struct { + frame top.Frame + err error + generation int +} +type stackMsg struct { + text string + err error + generation int +} +type tickMsg struct{ generation int } + +var sorts = []string{"cpu", "name", "state", "id"} + +func New(ctx context.Context, source *top.Source, every time.Duration) Model { + return Model{ctx: ctx, source: source, every: every, busy: true} +} +func (m Model) Err() error { return m.err } +func (m Model) Init() tea.Cmd { return m.read() } +func (m Model) read() tea.Cmd { + return func() tea.Msg { frame, err := m.source.Next(m.ctx); return frameMsg{frame, err, m.generation} } +} +func (m *Model) refreshTable() { + if m.frame == nil { + return + } + rows := top.Order(m.frame.Threads.Threads, sorts[m.sortIndex], m.reverse, m.query) + ids := make([]string, len(rows)) + for i, row := range rows { + ids[i] = strconv.FormatInt(row.ID, 10) + } + m.err = m.table.Set(top.Table(rows), ids) +} +func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.WindowSizeMsg: + m.width, m.height = msg.Width, msg.Height + case frameMsg: + m.busy = false + if msg.generation != m.generation || m.paused { + if !m.paused { + m.busy = true + return m, m.read() + } + return m, nil + } + if msg.err != nil { + m.err = msg.err + return m, tea.Quit + } + m.frame = &msg.frame + m.refreshTable() + if m.err != nil { + return m, tea.Quit + } + return m, tea.Tick(m.every, func(time.Time) tea.Msg { return tickMsg{m.generation} }) + case tickMsg: + if !m.paused && !m.busy && msg.generation == m.generation { + m.busy = true + return m, m.read() + } + case stackMsg: + if msg.generation != m.generation || m.inspection == nil { + return m, nil + } + if msg.err != nil { + m.setStack("Stack unavailable: " + msg.err.Error()) + } else { + m.setStack(msg.text) + } + case tea.KeyPressMsg: + return m.key(msg) + } + return m, nil +} +func (m Model) key(key tea.KeyPressMsg) (tea.Model, tea.Cmd) { + name := key.String() + if name == "ctrl+c" { + return m, tea.Quit + } + if m.inspection != nil { + switch name { + case "q": + return m, tea.Quit + case "esc", "d": + m.inspection = nil + return m, nil + case "space", "P": + m.inspection = nil + default: + m.inspection.Key(name) + return m, nil + } + } + if m.typing { + switch name { + case "enter": + m.typing = false + case "esc": + m.typing = false + m.query = "" + case "backspace": + runes := []rune(m.query) + if len(runes) > 0 { + m.query = string(runes[:len(runes)-1]) + } + default: + m.query += key.Text + } + m.refreshTable() + return m, nil + } + switch name { + case "q": + return m, tea.Quit + case "?": + m.help = !m.help + case "space", "P": + m.paused = !m.paused + m.generation++ + if !m.paused { + if !m.busy { + m.busy = true + return m, m.read() + } + } + case "enter", "d": + if m.table.Selected == "" { + return m, nil + } + id, err := strconv.ParseInt(m.table.Selected, 10, 64) + if err != nil { + m.err = err + return m, tea.Quit + } + m.paused = true + view := m.table + view.Detail = true + view.DetailOffset = 0 + m.inspection = &view + m.generation++ + m.setStack("Reading stack…") + return m, func() tea.Msg { + stack, err := m.source.Stack(m.ctx, id) + return stackMsg{stack.Text, err, m.generation} + } + case "esc": + m.help = false + m.query = "" + m.refreshTable() + case "/": + m.typing = true + case "s": + m.sortIndex = (m.sortIndex + 1) % len(sorts) + m.refreshTable() + case "S": + m.reverse = !m.reverse + m.refreshTable() + default: + m.table.Key(name) + } + return m, nil +} +func (m *Model) setStack(text string) { + m.inspection.Table.RowDetail = make([]api.Textable, len(m.inspection.IDs)) + m.inspection.Table.RowDetail[m.inspection.Index()] = api.Text{}.Append(text) +} diff --git a/internal/jvm/top/tui/model_test.go b/internal/jvm/top/tui/model_test.go new file mode 100644 index 0000000..a43d38f --- /dev/null +++ b/internal/jvm/top/tui/model_test.go @@ -0,0 +1,102 @@ +package tui + +import ( + tea "charm.land/bubbletea/v2" + "context" + "github.com/charmbracelet/x/ansi" + "github.com/flanksource/telltale/internal/jvm/top" + "github.com/flanksource/telltale/pkg/tracing/jvm/jvmruntime" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "strings" + "testing" + "time" +) + +func TestTopTUI(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "JVM Top TUI") } + +var _ = Describe("JVM Top lifecycle", func() { + It("uses the shared wrapped detail viewport and remains paused when inspection closes", func() { + m := New(context.Background(), &top.Source{}, time.Second) + m.width, m.height = 40, 10 + next, _ := m.Update(frameMsg{frame: top.Frame{Threads: jvmruntime.ThreadSample{Threads: []jvmruntime.Thread{{ID: 1, Name: "worker"}}}}}) + m = next.(Model) + next, _ = m.key(tea.KeyPressMsg{Text: "d"}) + m = next.(Model) + next, _ = m.Update(stackMsg{text: strings.Repeat("wide stack frame with source filename\n", 30) + "last frame", generation: m.generation}) + m = next.(Model) + for range 100 { + next, _ = m.key(tea.KeyPressMsg{Code: tea.KeyDown}) + m = next.(Model) + } + Expect(ansi.Strip(m.View().Content)).To(ContainSubstring("last frame")) + next, _ = m.key(tea.KeyPressMsg{Code: tea.KeyEscape}) + m = next.(Model) + Expect(m.paused).To(BeTrue()) + Expect(m.inspection).To(BeNil()) + Expect(m.table.Selected).To(Equal("1")) + }) + It("uses a compact meter header and keeps its footer at the bottom after resizing", func() { + m := New(context.Background(), &top.Source{}, time.Second) + frame := top.Frame{AgentID: "fixture", Metrics: jvmruntime.Metrics{ProcessCPUPercent: new(float64(25)), Memory: jvmruntime.Memory{Heap: []jvmruntime.MemoryPool{{Used: 128, Committed: 256}}}}, Threads: jvmruntime.ThreadSample{Threads: []jvmruntime.Thread{{ID: 1, Name: "worker"}}}} + next, _ := m.Update(frameMsg{frame: frame}) + m = next.(Model) + for _, size := range []tea.WindowSizeMsg{{Width: 120, Height: 30}, {Width: 80, Height: 24}, {Width: 40, Height: 10}} { + next, _ = m.Update(size) + m = next.(Model) + lines := strings.Split(ansi.Strip(m.View().Content), "\n") + Expect(lines).To(HaveLen(size.Height)) + Expect(lines[0]).To(ContainSubstring("telltale jvm top")) + Expect(lines[1]).To(ContainSubstring("[")) + Expect(lines[len(lines)-1]).To(ContainSubstring("P pause")) + for _, line := range lines { + Expect(ansi.StringWidth(line)).To(BeNumerically("<=", size.Width)) + } + } + }) + It("supports host Top's pause and help keys and Escape clears the filter", func() { + m := New(context.Background(), &top.Source{}, time.Second) + next, _ := m.key(tea.KeyPressMsg{Text: "P"}) + m = next.(Model) + Expect(m.paused).To(BeTrue()) + m.typing = true + m.query = "worker" + next, _ = m.key(tea.KeyPressMsg{Code: tea.KeyEscape}) + m = next.(Model) + Expect(m.query).To(BeEmpty()) + next, _ = m.key(tea.KeyPressMsg{Text: "?"}) + m = next.(Model) + m.width, m.height = 120, 24 + Expect(ansi.Strip(m.View().Content)).To(ContainSubstring("JVM Top help")) + }) + It("preserves selection on reorder and ignores frames while paused", func() { + m := New(context.Background(), &top.Source{}, time.Second) + frame := top.Frame{Threads: jvmruntime.ThreadSample{Threads: []jvmruntime.Thread{{ID: 1, Name: "worker"}, {ID: 2, Name: "main"}}}} + next, _ := m.Update(frameMsg{frame: frame}) + m = next.(Model) + m.table.Selected = "2" + m.refreshTable() + Expect(m.table.Selected).To(Equal("2")) + m.paused = true + next, _ = m.Update(frameMsg{frame: top.Frame{AgentID: "other"}}) + Expect(next.(Model).frame.AgentID).To(BeEmpty()) + }) + It("never schedules an overlapping sample when resumed before a pending sample finishes", func() { + m := New(context.Background(), &top.Source{}, time.Second) + next, _ := m.key(tea.KeyPressMsg{Code: tea.KeySpace}) + m = next.(Model) + next, cmd := m.key(tea.KeyPressMsg{Code: tea.KeySpace}) + Expect(cmd).To(BeNil()) + m = next.(Model) + _, cmd = m.Update(frameMsg{generation: 0}) + Expect(cmd).NotTo(BeNil()) + }) + It("clips output on small terminals and reports sampling failures", func() { + m := New(context.Background(), &top.Source{}, time.Second) + next, _ := m.Update(tea.WindowSizeMsg{Width: 20, Height: 5}) + Expect(next.(Model).View().Content).NotTo(BeEmpty()) + next, cmd := m.Update(frameMsg{err: context.DeadlineExceeded}) + Expect(next.(Model).Err()).To(MatchError(context.DeadlineExceeded)) + Expect(cmd).NotTo(BeNil()) + }) +}) diff --git a/internal/jvm/top/tui/view.go b/internal/jvm/top/tui/view.go new file mode 100644 index 0000000..d1f25c9 --- /dev/null +++ b/internal/jvm/top/tui/view.go @@ -0,0 +1,120 @@ +package tui + +import ( + "fmt" + "strings" + "time" + + tea "charm.land/bubbletea/v2" + "github.com/flanksource/clicky/api" + "github.com/flanksource/telltale/internal/terminalui" + "github.com/flanksource/telltale/pkg/tracing/jvm/jvmruntime" +) + +func (m Model) View() tea.View { + if m.width <= 0 || m.height <= 0 { + return tea.NewView("Reading JVM…") + } + header := m.header() + footer := m.footer() + bodyHeight := max(0, m.height-min(len(header), max(1, m.height-3))-1) + header = header[:min(len(header), max(1, m.height-3))] + body := m.table.View(m.width, bodyHeight) + if m.inspection != nil { + header = []string{terminalui.Title.Render("JVM thread stack · paused")} + bodyHeight = max(0, m.height-2) + body = m.inspection.View(m.width, bodyHeight) + } + if m.help { + header = []string{terminalui.Title.Render("JVM Top help")} + bodyHeight = max(0, m.height-2) + body = "↑↓ or j/k select · s sort · S reverse\n/ filter · enter keeps filter · esc clears filter\nd or enter inspect stack and freeze\nP or space pause/resume\nIn stack: ↑↓ scroll · esc or d close · P resume\n? close help · q or Ctrl+C quit" + } + lines := append(header, terminalui.PadLines(body, bodyHeight)...) + lines = append(lines, footer) + lines = lines[:min(len(lines), m.height)] + for i := range lines { + lines[i] = terminalui.Clip(lines[i], m.width) + } + view := tea.NewView(strings.Join(lines, "\n")) + view.AltScreen = true + return view +} + +func (m Model) header() []string { + direction := "↓" + if m.reverse { + direction = "↑" + } + title := "telltale jvm top · sort " + sorts[m.sortIndex] + direction + if m.paused { + title += " · paused" + } + if m.typing || m.query != "" { + title += " · filter " + m.query + if m.typing { + title += "▏" + } + } + if m.frame == nil { + return terminalui.PadLines(terminalui.Title.Render(terminalui.Fit(title, m.width))+"\n"+terminalui.Muted.Render("Reading JVM…"), 6) + } + f := m.frame + title += fmt.Sprintf(" · %s · %s · Java %s · up %s", f.SampledAt.Local().Format(time.TimeOnly), f.AgentID, f.Metrics.Runtime.JavaVersion, (time.Duration(f.Metrics.Runtime.Uptime) * time.Millisecond).Round(time.Second)) + width := max(1, min(60, m.width/3)) + cpu := "CPU " + terminalui.Muted.Render("unavailable") + if value := f.Metrics.ProcessCPUPercent; value != nil { + cpu = "CPU " + terminalui.Bar(width, []terminalui.Share{{Percent: *value, Style: terminalui.CPU}}) + fmt.Sprintf(" %.1f%% of capacity", *value) + } + used, committed := memory(f.Metrics.Memory.Heap) + heap := "HEAP " + terminalui.Muted.Render("unavailable") + if committed > 0 { + heap = "HEAP " + terminalui.Bar(width, []terminalui.Share{{Percent: 100 * float64(used) / float64(committed), Style: terminalui.Memory}}) + fmt.Sprintf(" %s / %s committed", api.HumanizeBytes(used), api.HumanizeBytes(committed)) + } + nonHeap, _ := memory(f.Metrics.Memory.NonHeap) + buffers, _ := memory(f.Metrics.Memory.Buffer) + gc := []string{} + for _, collector := range f.Metrics.GarbageCollectors { + gc = append(gc, fmt.Sprintf("%s %s / %sms", collector.Name, counter(collector.CollectionCount), counter(collector.CollectionTime))) + } + states := map[string]int{} + for _, thread := range f.Threads.Threads { + states[thread.State]++ + } + threads := fmt.Sprintf("THR %d · runnable %d · blocked %d · waiting %d", len(f.Threads.Threads), states["RUNNABLE"], states["BLOCKED"], states["WAITING"]+states["TIMED_WAITING"]) + if !f.Threads.CPUTimeEnabled { + threads += " · CPU timing off" + } + return []string{terminalui.Title.Render(terminalui.Fit(title, m.width)), cpu, heap, fmt.Sprintf("MEM non-heap %s · buffers %s", api.HumanizeBytes(nonHeap), api.HumanizeBytes(buffers)), "GC " + strings.Join(gc, " · ") + " (lifetime count / time)", threads} +} + +func (m Model) footer() string { + text := "P pause/resume · d stack · / filter · ? help · q quit" + if m.typing { + text = "type to filter · enter keeps it · esc clears it" + } + if m.inspection != nil { + text = "↑↓ scroll · esc close · P or space resume live · q quit" + } + if m.err != nil { + return terminalui.Status.Render(terminalui.Fit(m.err.Error(), m.width)) + } + if m.frame != nil && len(m.frame.Threads.Deadlocked) > 0 { + return terminalui.Status.Render(terminalui.Fit(fmt.Sprintf("DEADLOCK threads %v · d inspect · ? help", m.frame.Threads.Deadlocked), m.width)) + } + return terminalui.Muted.Render(terminalui.Fit(text, m.width)) +} + +func memory(pools []jvmruntime.MemoryPool) (used, committed int64) { + for _, pool := range pools { + used += pool.Used + committed += pool.Committed + } + return +} +func counter(value int64) string { + if value < 0 { + return "unavailable" + } + return fmt.Sprint(value) +} diff --git a/internal/otelreceiver/grpc.go b/internal/otelreceiver/grpc.go index c647d57..f35ea7d 100644 --- a/internal/otelreceiver/grpc.go +++ b/internal/otelreceiver/grpc.go @@ -3,6 +3,7 @@ package otelreceiver import ( "context" + "go.opentelemetry.io/collector/pdata/pprofile/pprofileotlp" "go.opentelemetry.io/collector/pdata/ptrace/ptraceotlp" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -26,5 +27,6 @@ func (r *Receiver) GRPCServer(requireAuth bool) *grpc.Server { return handler(ctx, request) })) ptraceotlp.RegisterGRPCServer(server, r) + pprofileotlp.RegisterGRPCServer(server, &profilesGRPC{receiver: r}) return server } diff --git a/internal/otelreceiver/http.go b/internal/otelreceiver/http.go index 5916a21..5f62a91 100644 --- a/internal/otelreceiver/http.go +++ b/internal/otelreceiver/http.go @@ -10,6 +10,7 @@ import ( "net/http" "time" + "go.opentelemetry.io/collector/pdata/pprofile/pprofileotlp" "go.opentelemetry.io/collector/pdata/ptrace/ptraceotlp" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -40,7 +41,13 @@ func (r *Receiver) HTTPHandler(requireAuth bool) http.Handler { return } } - body, err := r.exportHTTP(request, format) + var body []byte + if request.URL.Path == "/v1/profiles" { + w.Header().Set(pprofileotlp.DevelopmentVersionHeader, pprofileotlp.DevelopmentVersion) + body, err = r.exportProfilesHTTP(request, format) + } else { + body, err = r.exportHTTP(request, format) + } if err != nil { writeError(w, protocolError{format: format, err: err}) return diff --git a/internal/otelreceiver/normalize.go b/internal/otelreceiver/normalize.go index e843a55..c01d051 100644 --- a/internal/otelreceiver/normalize.go +++ b/internal/otelreceiver/normalize.go @@ -1,126 +1,11 @@ package otelreceiver import ( - "encoding/base64" - "fmt" - "math" - "time" - - "github.com/flanksource/telltale/pkg/tracing" - sdk "github.com/flanksource/telltale/sdk" - "go.opentelemetry.io/collector/pdata/pcommon" - "go.opentelemetry.io/collector/pdata/ptrace" + otlpspans "github.com/flanksource/telltale/pkg/otlp/spans" "go.opentelemetry.io/collector/pdata/ptrace/ptraceotlp" ) -type spanContext struct { - resource ptrace.ResourceSpans - scope ptrace.ScopeSpans - span ptrace.Span -} - func normalize(request ptraceotlp.ExportRequest, options Options) (batch, error) { - result := batch{Spans: []sdk.Span{}} - if request.Traces().SpanCount() > options.MaxSpans { - return result, fmt.Errorf("%w: at most %d spans", ErrLimit, options.MaxSpans) - } - size := 0 - resources := request.Traces().ResourceSpans() - for i := 0; i < resources.Len(); i++ { - resource := resources.At(i) - for j := 0; j < resource.ScopeSpans().Len(); j++ { - scope := resource.ScopeSpans().At(j) - for k := 0; k < scope.Spans().Len(); k++ { - span, err := convertSpan(spanContext{resource: resource, scope: scope, span: scope.Spans().At(k)}) - if err != nil { - result.Rejected++ - result.Reason = err.Error() - continue - } - size += len(span.Native) - if size > options.MaxRequestBytes*8 { - return batch{}, fmt.Errorf("%w: expanded span contexts exceed %d bytes; reduce the batch size", ErrLimit, options.MaxRequestBytes*8) - } - result.Spans = append(result.Spans, span) - } - } - } - return result, nil -} - -func convertSpan(input spanContext) (sdk.Span, error) { - span := input.span - if span.TraceID().IsEmpty() || span.SpanID().IsEmpty() || span.StartTimestamp() == 0 || span.EndTimestamp() < span.StartTimestamp() || uint64(span.EndTimestamp()) > math.MaxInt64 || span.SpanID() == span.ParentSpanID() { - return sdk.Span{}, fmt.Errorf("invalid span: nonzero trace/span IDs, valid timestamps and a distinct parent are required") - } - native := ptraceotlp.NewExportRequest() - resource := native.Traces().ResourceSpans().AppendEmpty() - input.resource.Resource().CopyTo(resource.Resource()) - resource.SetSchemaUrl(input.resource.SchemaUrl()) - scope := resource.ScopeSpans().AppendEmpty() - input.scope.Scope().CopyTo(scope.Scope()) - scope.SetSchemaUrl(input.scope.SchemaUrl()) - span.CopyTo(scope.Spans().AppendEmpty()) - span = scope.Spans().At(0) - if err := redact(spanContext{resource: resource, scope: scope, span: span}); err != nil { - return sdk.Span{}, err - } - encoded, err := native.MarshalProto() - if err != nil { - return sdk.Span{}, fmt.Errorf("encode OTLP span: %w", err) - } - jsonData, err := native.MarshalJSON() - if err != nil { - return sdk.Span{}, err - } - digest, err := nativeDigest(jsonData) - if err != nil { - return sdk.Span{}, err - } - parent := "" - if !span.ParentSpanID().IsEmpty() { - parent = span.ParentSpanID().String() - } - record := sdk.Span{ - TraceID: span.TraceID().String(), SpanID: span.SpanID().String(), ParentID: parent, - Operation: span.Name(), Kind: span.Kind().String(), - StartedAt: span.StartTimestamp().AsTime(), - DurationMs: float64(span.EndTimestamp()-span.StartTimestamp()) / float64(time.Millisecond), - Status: span.Status().Code().String(), - Attributes: span.Attributes().AsRaw(), Resource: resource.Resource().Attributes().AsRaw(), - Native: base64.StdEncoding.EncodeToString(encoded), Digest: digest, - Source: sdk.SpanSource{Type: sdk.SpanSourceOTLP}, - } - if err := record.Normalize(); err != nil { - return sdk.Span{}, fmt.Errorf("invalid span: %w", err) - } - return record, nil -} - -func redact(input spanContext) error { - span := input.span - maps := []pcommon.Map{input.resource.Resource().Attributes(), input.scope.Scope().Attributes(), span.Attributes()} - for i := 0; i < span.Events().Len(); i++ { - event := span.Events().At(i) - event.SetName(sdk.Mask(event.Name())) - maps = append(maps, event.Attributes()) - } - for i := 0; i < span.Links().Len(); i++ { - maps = append(maps, span.Links().At(i).Attributes()) - } - for _, attributes := range maps { - values := attributes.AsRaw() - if len(values) != attributes.Len() { - return fmt.Errorf("invalid span: duplicate attribute keys") - } - for key, value := range values { - values[key] = tracing.MaskAttribute(key, value) - } - if err := attributes.FromRaw(values); err != nil { - return fmt.Errorf("redact OTLP attributes: %w", err) - } - } - span.SetName(sdk.Mask(span.Name())) - span.Status().SetMessage(sdk.Mask(span.Status().Message())) - return nil + converted, err := otlpspans.Normalize(request, otlpspans.Options{MaxSpans: options.MaxSpans, MaxRequestBytes: options.MaxRequestBytes}) + return batch{Spans: converted.Spans, Rejected: converted.Rejected, Reason: converted.Reason}, err } diff --git a/internal/otelreceiver/profiles_collector_test.go b/internal/otelreceiver/profiles_collector_test.go new file mode 100644 index 0000000..55548da --- /dev/null +++ b/internal/otelreceiver/profiles_collector_test.go @@ -0,0 +1,101 @@ +package otelreceiver + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "time" + + "github.com/flanksource/telltale/pkg/otlp/profiles" + "github.com/flanksource/telltale/pkg/traces/read" + "github.com/flanksource/telltale/pkg/tracing" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.opentelemetry.io/collector/pdata/pprofile" +) + +func collectorPipeline(endpoint, queueSizer string) string { + GinkgoHelper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + address := listener.Addr().String() + Expect(listener.Close()).To(Succeed()) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + command := exec.CommandContext(ctx, "go", "run", ".", "-listen", address, "-export", endpoint, "-queue-sizer", queueSizer) + command.Dir = "../../hack/collector-fixture" + command.Env = append(os.Environ(), "GOWORK=off") + command.Stderr = GinkgoWriter + output, err := command.StdoutPipe() + Expect(err).NotTo(HaveOccurred()) + input, err := command.StdinPipe() + Expect(err).NotTo(HaveOccurred()) + Expect(command.Start()).To(Succeed()) + ready := make(chan string, 1) + go func() { + scanner := bufio.NewScanner(output) + if scanner.Scan() { + ready <- scanner.Text() + } else { + ready <- fmt.Sprintf("Collector exited before ready: %v", scanner.Err()) + } + }() + DeferCleanup(func() { + Expect(input.Close()).To(Succeed()) + Expect(command.Wait()).To(Succeed()) + cancel() + }) + var message string + Eventually(ready).WithTimeout(150 * time.Second).Should(Receive(&message)) + Expect(message).To(Equal("ready")) + return "http://" + address + "/v1development/profiles" +} + +var _ = Describe("released Collector Profiles pipeline", func() { + DescribeTable("preserves active, updated and empty snapshots through the 0.163 receiver and queued OTLP exporter", func(queueSizer string) { + receiver, store := testReceiver(DefaultOptions()) + endpoint := collectorPipeline(serveGRPC(receiver, false), queueSizer) + client := &http.Client{Timeout: 10 * time.Second} + for sequence := int64(1); sequence <= 3; sequence++ { + snapshot := topSnapshot(sequence) + if sequence == 3 { + snapshot.Rows, snapshot.Observed = nil, 0 + } + body, err := profiles.Encode([]profiles.Snapshot{snapshot}) + Expect(err).NotTo(HaveOccurred()) + request, err := http.NewRequest(http.MethodPost, endpoint, bytes.NewReader(body)) + Expect(err).NotTo(HaveOccurred()) + request.Header.Set("Content-Type", protobufType) + request.Header.Set("OTLP-Profiles-Development-Version", profiles.DevelopmentVersion) + response, err := client.Do(request) + Expect(err).NotTo(HaveOccurred()) + responseBody, err := io.ReadAll(response.Body) + Expect(err).NotTo(HaveOccurred()) + Expect(response.Body.Close()).To(Succeed()) + Expect(response.StatusCode).To(Equal(http.StatusOK), string(responseBody)) + } + Eventually(func(g Gomega) { + stored, err := store.ReadProfiles(context.Background(), ProfileReadOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(stored)).To(Equal(3), "Collector must preserve the clearing profile") + g.Expect(stored[0].Rows[0].ElapsedNS).To(Equal(int64(time.Second))) + g.Expect(stored[1].Rows[0].ElapsedNS).To(Equal(int64(2 * time.Second))) + g.Expect(stored[2].Rows).To(BeEmpty()) + g.Expect(stored[2].Observed).To(BeZero()) + native, err := (&pprofile.ProtoUnmarshaler{}).UnmarshalProfiles(stored[2].Native) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(native.SampleCount()).To(Equal(1)) + ref := read.Ref{Runtime: store.Runtime(), TraceRef: tracing.TraceRef{Kind: "http_top", Stream: profileStream("http_top")}} + latest, err := read.Top(context.Background(), ref, read.TopQuery{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(latest.Frame.Tick).To(Equal(int64(3))) + g.Expect(latest.Frame.Retained).To(BeZero()) + g.Expect(latest.Rows).To(BeEmpty()) + }).WithTimeout(10 * time.Second).Should(Succeed()) + }, Entry("request-sized queue", "requests"), Entry("item-sized queue", "items")) +}) diff --git a/internal/otelreceiver/profiles_ingest.go b/internal/otelreceiver/profiles_ingest.go new file mode 100644 index 0000000..a4de3bb --- /dev/null +++ b/internal/otelreceiver/profiles_ingest.go @@ -0,0 +1,98 @@ +package otelreceiver + +import ( + "context" + "errors" + "fmt" + "net/http" + "time" + + "github.com/flanksource/telltale/pkg/otlp/profiles" + "go.opentelemetry.io/collector/pdata/pprofile/pprofileotlp" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type profilesGRPC struct { + pprofileotlp.UnimplementedGRPCServer + receiver *Receiver +} + +func (g *profilesGRPC) Export(ctx context.Context, request pprofileotlp.ExportRequest) (pprofileotlp.ExportResponse, error) { + if err := g.receiver.acquire(); err != nil { + return pprofileotlp.ExportResponse{}, err + } + defer g.receiver.release() + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + return g.receiver.ingestProfiles(ctx, request) +} + +func (r *Receiver) exportProfilesHTTP(request *http.Request, format string) ([]byte, error) { + if err := pprofileotlp.ValidateDevelopmentVersion(request.Header.Values(pprofileotlp.DevelopmentVersionHeader)); err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + if err := r.acquire(); err != nil { + return nil, err + } + defer r.release() + body, err := r.readBody(request) + if err != nil { + return nil, err + } + decoded := pprofileotlp.NewExportRequest() + if format == jsonType { + err = decoded.UnmarshalJSON(body) + } else { + err = decoded.UnmarshalProto(body) + } + if err != nil { + return nil, status.Error(codes.InvalidArgument, "malformed OTLP Profiles request") + } + ctx, cancel := context.WithTimeout(request.Context(), 30*time.Second) + defer cancel() + response, err := r.ingestProfiles(ctx, decoded) + if err != nil { + return nil, err + } + if format == jsonType { + return response.MarshalJSON() + } + return response.MarshalProto() +} + +func (r *Receiver) ingestProfiles(ctx context.Context, request pprofileotlp.ExportRequest) (pprofileotlp.ExportResponse, error) { + response := pprofileotlp.NewExportResponse() + data, err := request.MarshalProto() + if err != nil { + return response, status.Error(codes.InvalidArgument, "invalid OTLP Profiles request") + } + if len(data) > r.config.Options.MaxRequestBytes { + return response, status.Error(codes.ResourceExhausted, "OTLP Profiles request exceeds its byte limit") + } + if request.Profiles().ProfileCount() == 0 { + return response, nil + } + snapshots, err := profiles.Decode(data, profiles.Options{MaxBytes: r.config.Options.MaxRequestBytes, MaxSamples: r.config.Options.MaxSpans}) + if err != nil { + if errors.Is(err, profiles.ErrLimit) { + return response, status.Error(codes.ResourceExhausted, err.Error()) + } + return response, status.Error(codes.InvalidArgument, err.Error()) + } + outcome, err := r.config.Store.PersistProfiles(ctx, snapshots) + if err != nil { + if errors.Is(err, errInvalidProfile) { + return response, status.Error(codes.InvalidArgument, err.Error()) + } + if ctx.Err() != nil { + return response, status.FromContextError(ctx.Err()).Err() + } + return response, status.Error(codes.Unavailable, fmt.Sprintf("OTLP Profiles storage unavailable: %v", err)) + } + if outcome.Rejected > 0 { + response.PartialSuccess().SetRejectedProfiles(outcome.Rejected) + response.PartialSuccess().SetErrorMessage(outcome.Reason) + } + return response, nil +} diff --git a/internal/otelreceiver/profiles_protocol_test.go b/internal/otelreceiver/profiles_protocol_test.go new file mode 100644 index 0000000..42f0158 --- /dev/null +++ b/internal/otelreceiver/profiles_protocol_test.go @@ -0,0 +1,115 @@ +package otelreceiver + +import ( + "bytes" + "compress/gzip" + "context" + "net/http" + "net/http/httptest" + "os" + "time" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/credentials/apitoken" + "github.com/flanksource/telltale/pkg/datadir" + "github.com/flanksource/telltale/pkg/otlp/profiles" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.opentelemetry.io/collector/pdata/pprofile/pprofileotlp" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +func javaTopRequest(name string) pprofileotlp.ExportRequest { + GinkgoHelper() + data, err := os.ReadFile("../../pkg/otlp/profiles/testdata/java-http-top-" + name + ".pb") + Expect(err).NotTo(HaveOccurred()) + request := pprofileotlp.NewExportRequest() + Expect(request.UnmarshalProto(data)).To(Succeed()) + return request +} + +var _ = Describe("Profiles protocol boundaries", func() { + It("reports invalid operation attributes as a permanent input error", func() { + receiver, store := testReceiver(DefaultOptions()) + snapshot := topSnapshot(1) + delete(snapshot.Rows[0].Attributes, "startedAt") + data, err := profiles.Encode([]profiles.Snapshot{snapshot}) + Expect(err).NotTo(HaveOccurred()) + request := pprofileotlp.NewExportRequest() + Expect(request.UnmarshalProto(data)).To(Succeed()) + _, err = receiver.ingestProfiles(context.Background(), request) + Expect(status.Code(err)).To(Equal(codes.InvalidArgument)) + stored, err := store.ReadProfiles(context.Background(), ProfileReadOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(stored).To(BeEmpty()) + }) + DescribeTable("accepts Java active, updated and empty profiles using authenticated protobuf, JSON and gRPC", func(format string) { + receiver, store := testReceiver(DefaultOptions()) + tokens := agentclient.NewTokenStore([]datadir.ConfigDir{{Name: datadir.ConfigGlobal, Dir: store.options.DataDir}}) + api, err := tokens.Mint("profile-exporter", apitoken.ScopeAPI) + Expect(err).NotTo(HaveOccurred()) + connection, err := grpc.NewClient(serveGRPC(receiver, true), grpc.WithTransportCredentials(insecure.NewCredentials())) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(connection.Close) + client := pprofileotlp.NewGRPCClient(connection) + for _, name := range []string{"active", "updated", "empty"} { + request := javaTopRequest(name) + if format == "grpc" { + ctx, cancel := context.WithTimeout(metadata.AppendToOutgoingContext(context.Background(), "authorization", "Bearer "+api.Secret.Value()), 5*time.Second) + _, err = client.Export(ctx, request) + cancel() + Expect(err).NotTo(HaveOccurred()) + continue + } + var data []byte + if format == jsonType { + data, err = request.MarshalJSON() + } else { + data, err = request.MarshalProto() + } + Expect(err).NotTo(HaveOccurred()) + var compressed bytes.Buffer + writer := gzip.NewWriter(&compressed) + _, err = writer.Write(data) + Expect(err).NotTo(HaveOccurred()) + Expect(writer.Close()).To(Succeed()) + post := httptest.NewRequest(http.MethodPost, "/v1/profiles", &compressed) + post.Header.Set("Content-Type", format) + post.Header.Set("Content-Encoding", "gzip") + post.Header.Set("Authorization", "Bearer "+api.Secret.Value()) + post.Header.Set("OTLP-Profiles-Development-Version", profiles.DevelopmentVersion) + response := httptest.NewRecorder() + receiver.HTTPHandler(true).ServeHTTP(response, post) + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + Expect(response.Header().Get("OTLP-Profiles-Development-Version")).To(Equal(profiles.DevelopmentVersion)) + } + stored, err := store.ReadProfiles(context.Background(), ProfileReadOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(stored).To(HaveLen(3)) + Expect(stored[0].Rows[0].Attributes).To(HaveKeyWithValue("threadId", int64(9007199254740993))) + Expect(stored[1].Rows[0].ElapsedNS).To(Equal(int64(500000000))) + Expect(stored[2].Rows).To(BeEmpty()) + }, Entry("HTTP protobuf + gzip", protobufType), Entry("HTTP JSON + gzip", jsonType), Entry("gRPC", "grpc")) + It("rejects missing and agent-scoped gRPC credentials before writing", func() { + receiver, store := testReceiver(DefaultOptions()) + tokens := agentclient.NewTokenStore([]datadir.ConfigDir{{Name: datadir.ConfigGlobal, Dir: store.options.DataDir}}) + agent, err := tokens.Mint("agent", apitoken.ScopeAgent) + Expect(err).NotTo(HaveOccurred()) + connection, err := grpc.NewClient(serveGRPC(receiver, true), grpc.WithTransportCredentials(insecure.NewCredentials())) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(connection.Close) + client := pprofileotlp.NewGRPCClient(connection) + _, err = client.Export(context.Background(), javaTopRequest("active")) + Expect(status.Code(err)).To(Equal(codes.Unauthenticated)) + ctx := metadata.AppendToOutgoingContext(context.Background(), "authorization", "Bearer "+agent.Secret.Value()) + _, err = client.Export(ctx, javaTopRequest("active")) + Expect(status.Code(err)).To(Equal(codes.PermissionDenied)) + stored, err := store.ReadProfiles(context.Background(), ProfileReadOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(stored).To(BeEmpty()) + }) +}) diff --git a/internal/otelreceiver/profiles_read.go b/internal/otelreceiver/profiles_read.go new file mode 100644 index 0000000..565dbf9 --- /dev/null +++ b/internal/otelreceiver/profiles_read.go @@ -0,0 +1,78 @@ +package otelreceiver + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/flanksource/telltale/pkg/otlp/profiles" +) + +type ProfileReadOptions struct { + Kind, CaptureID, Source string + Limit int +} + +func (s *Store) ReadProfiles(ctx context.Context, options ProfileReadOptions) ([]profiles.Snapshot, error) { + s.lifecycle.RLock() + defer s.lifecycle.RUnlock() + if s.closed { + return nil, fmt.Errorf("OTLP store is closed") + } + if options.Kind != "" && options.Kind != "http_top" && options.Kind != "sql_top" && options.Kind != "jms_top" { + return nil, fmt.Errorf("unsupported profile kind %q", options.Kind) + } + if options.Limit < 0 { + return nil, fmt.Errorf("profile read limit must be nonnegative") + } + if options.Limit == 0 { + options.Limit = 10000 + } + kinds := []string{options.Kind} + if options.Kind == "" { + kinds = []string{"http_top", "sql_top", "jms_top"} + } + result := []profiles.Snapshot{} + for _, kind := range kinds { + err := s.results.Backend.Scan(ctx, profileStream(kind)+"-frames", 0, func(_ int64, row recordstore.Row) error { + encoded, err := json.Marshal(row) + if err != nil { + return err + } + var frame sampleframe.Frame + if err = json.Unmarshal(encoded, &frame); err != nil { + return err + } + native, ok := frame.Summary["otlp"].(string) + if !ok { + return fmt.Errorf("received profile frame has no native payload") + } + data, err := base64.StdEncoding.DecodeString(native) + if err != nil { + return err + } + snapshots, err := profiles.Decode(data, profiles.Options{}) + if err != nil { + return err + } + for _, snapshot := range snapshots { + if options.CaptureID != "" && snapshot.CaptureID != options.CaptureID || options.Source != "" && snapshot.Source != options.Source { + continue + } + result = append(result, snapshot) + if len(result) > options.Limit { + return fmt.Errorf("profile read exceeds %d snapshots; narrow the selection", options.Limit) + } + } + return nil + }) + if err != nil && !errors.Is(err, recordstore.ErrNotFound) { + return nil, err + } + } + return result, nil +} diff --git a/internal/otelreceiver/profiles_store.go b/internal/otelreceiver/profiles_store.go new file mode 100644 index 0000000..054aa04 --- /dev/null +++ b/internal/otelreceiver/profiles_store.go @@ -0,0 +1,144 @@ +package otelreceiver + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "time" + + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/flanksource/telltale/pkg/otlp/profilepublication" + "github.com/flanksource/telltale/pkg/otlp/profiles" + "github.com/flanksource/telltale/pkg/tracing/httptop" + "github.com/flanksource/telltale/pkg/tracing/jmstop" + "github.com/flanksource/telltale/pkg/tracing/sqltop" +) + +const profileIdentityStream = profilepublication.IdentityStream + +var errInvalidProfile = errors.New("invalid active operation profile") + +func profileIdentityResult() recordresults.ResultType[profilepublication.Identity] { + return profilepublication.ResultType() +} + +func profileStream(kind string) string { + if kind == jmstop.Kind { + return "received-jms-top" + } + if kind == "sql_top" { + return "received-sql-top-active" + } + return "received-http-top" +} + +func profileStorageKind(kind string) string { + if kind == jmstop.Kind { + return jmstop.Kind + } + if kind == "sql_top" { + return "sql_top_active" + } + return httptop.Kind +} + +func (s *Store) PersistProfiles(ctx context.Context, snapshots []profiles.Snapshot) (Outcome, error) { + s.lifecycle.RLock() + defer s.lifecycle.RUnlock() + if s.closed || !s.options.Writable { + return Outcome{}, fmt.Errorf("OTLP store is closed or read-only") + } + s.write.Lock() + defer s.write.Unlock() + if err := s.trimProfiles(ctx); err != nil { + return Outcome{}, err + } + outcome := Outcome{} + for _, snapshot := range snapshots { + if len(snapshot.Native) == 0 || len(snapshot.Digest) != sha256.Size*2 || (snapshot.Kind != "http_top" && snapshot.Kind != "sql_top" && snapshot.Kind != jmstop.Kind) { + return Outcome{}, fmt.Errorf("%w: profile requires validated native payload, digest and supported kind", errInvalidProfile) + } + published, err := s.appendProfile(ctx, snapshot) + if errors.Is(err, profilepublication.ErrConflict) { + outcome.Rejected++ + outcome.Reason = err.Error() + continue + } + if err != nil { + return Outcome{}, err + } + if published { + outcome.Accepted++ + } else { + outcome.Duplicates++ + } + } + return outcome, nil +} + +func (s *Store) trimProfiles(ctx context.Context) error { + cutoff := time.Now().Add(-s.options.Retention) + for _, stream := range []string{profileStream("http_top"), profileStream("http_top") + "-frames", profileStream("sql_top"), profileStream("sql_top") + "-frames", profileStream(jmstop.Kind), profileStream(jmstop.Kind) + "-frames", profileIdentityStream} { + _, err := s.results.Backend.Trim(ctx, stream, cutoff) + if err != nil && !errors.Is(err, recordstore.ErrNotFound) { + return err + } + } + return nil +} + +func (s *Store) appendProfile(ctx context.Context, snapshot profiles.Snapshot) (bool, error) { + rows, schema, err := profileRows(snapshot) + if err != nil { + return false, fmt.Errorf("%w: %v", errInvalidProfile, err) + } + stream := profileStream(snapshot.Kind) + frames, err := sampleframe.Schema(profileStorageKind(snapshot.Kind) + "_frames") + if err != nil { + return false, err + } + return profilepublication.Append(ctx, s.results.Backend, profilepublication.Publication{Kind: snapshot.Kind, Stream: stream, FrameStream: stream + "-frames", Schema: schema, FrameSchema: frames, Frame: snapshot.Frame(), Rows: rows}) +} + +func profileRows(snapshot profiles.Snapshot) ([]recordstore.Row, recordstore.KindSchema, error) { + if snapshot.Kind == jmstop.Kind { + samples, err := jmstop.FromSnapshot(snapshot) + if err != nil { + return nil, recordstore.KindSchema{}, err + } + rows, err := profileSampleRows(samples) + schema, schemaErr := jmstop.ResultType().KindSchema() + return rows, schema, errors.Join(err, schemaErr) + } + if snapshot.Kind == "http_top" { + samples, err := httptop.FromSnapshot(snapshot) + if err != nil { + return nil, recordstore.KindSchema{}, err + } + rows, err := profileSampleRows(samples) + schema, schemaErr := httptop.ResultType().KindSchema() + return rows, schema, errors.Join(err, schemaErr) + } + samples, err := sqltop.FromSnapshot(snapshot) + if err != nil { + return nil, recordstore.KindSchema{}, err + } + rows, err := profileSampleRows(samples) + schema, schemaErr := sqltop.ActiveResult().KindSchema() + return rows, schema, errors.Join(err, schemaErr) +} + +func profileSampleRows[T any](samples []T) ([]recordstore.Row, error) { + rows := make([]recordstore.Row, len(samples)) + for i, sample := range samples { + row, err := recordstore.EncodeRow(sample) + if err != nil { + return nil, err + } + rows[i] = row + } + return rows, nil +} diff --git a/internal/otelreceiver/profiles_store_test.go b/internal/otelreceiver/profiles_store_test.go new file mode 100644 index 0000000..2bcc034 --- /dev/null +++ b/internal/otelreceiver/profiles_store_test.go @@ -0,0 +1,157 @@ +package otelreceiver + +import ( + "context" + "time" + + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/pkg/otlp/profiles" + "github.com/flanksource/telltale/pkg/traces/read" + "github.com/flanksource/telltale/pkg/tracing" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.opentelemetry.io/collector/pdata/pprofile" +) + +func decodedTop(snapshot profiles.Snapshot) profiles.Snapshot { + GinkgoHelper() + data, err := profiles.Encode([]profiles.Snapshot{snapshot}) + Expect(err).NotTo(HaveOccurred()) + decoded, err := profiles.Decode(data, profiles.Options{}) + Expect(err).NotTo(HaveOccurred()) + return decoded[0] +} + +var _ = Describe("durable Profiles snapshots", func() { + It("replays a JMS-only receiver without advertising a nonexistent spans stream", func() { + _, store := testReceiver(DefaultOptions()) + snapshot := topSnapshot(1) + snapshot.Kind = "jms_top" + snapshot.Rows[0].Attributes = map[string]any{"destination": "orders", "handler": "OrderListener.accept", "state": "idle", "activeCount": 0} + outcome, err := store.PersistProfiles(context.Background(), []profiles.Snapshot{decodedTop(snapshot)}) + Expect(err).NotTo(HaveOccurred()) + Expect(outcome.Accepted).To(Equal(int64(1))) + selected, err := read.ResolveRuns(read.Scope{DataDir: store.options.DataDir, Runs: []string{SourceID}}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(selected.Close) + page, err := read.FeedPage(context.Background(), selected.Refs, read.FeedQuery{}) + Expect(err).NotTo(HaveOccurred()) + Expect(page.Rows).To(ContainElement(And(HaveField("Kind", "jms_top"), HaveField("Result", "")))) + Expect(selected.Refs).NotTo(ContainElement(HaveField("Kind", "spans"))) + latest, err := read.Top(context.Background(), read.Ref{Runtime: store.Runtime(), TraceRef: tracing.TraceRef{Kind: "jms_top", Stream: profileStream("jms_top")}}, read.TopQuery{}) + Expect(err).NotTo(HaveOccurred()) + Expect(latest.Rows).To(ConsistOf(And(HaveKeyWithValue("handler", "OrderListener.accept"), HaveKeyWithValue("state", "idle")))) + Expect(latest.Display.RowDetail).To(ConsistOf(Not(BeNil()))) + Expect(latest.Display.Rows[0]["activeCount"].String()).To(Equal("0")) + }) + It("rejects unsealed snapshots without leaving partial rows or frames", func() { + _, store := testReceiver(DefaultOptions()) + _, err := store.PersistProfiles(context.Background(), []profiles.Snapshot{topSnapshot(1)}) + Expect(err).To(MatchError(ContainSubstring("native"))) + stored, err := store.ReadProfiles(context.Background(), ProfileReadOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(stored).To(BeEmpty()) + }) + It("recovers committed identities after restart and rejects profile or sequence conflicts", func() { + ctx := context.Background() + options := StoreOptions{DataDir: GinkgoT().TempDir(), Retention: time.Hour, Writable: true} + store, err := OpenStore(options) + Expect(err).NotTo(HaveOccurred()) + original := decodedTop(topSnapshot(1)) + outcome, err := store.PersistProfiles(ctx, []profiles.Snapshot{original}) + Expect(err).NotTo(HaveOccurred()) + Expect(outcome).To(Equal(Outcome{Accepted: 1})) + Expect(store.Close()).To(Succeed()) + store, err = OpenStore(options) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(store.Close) + outcome, err = store.PersistProfiles(ctx, []profiles.Snapshot{original}) + Expect(err).NotTo(HaveOccurred()) + Expect(outcome).To(Equal(Outcome{Duplicates: 1})) + native, err := (&pprofile.ProtoUnmarshaler{}).UnmarshalProfiles(original.Native) + Expect(err).NotTo(HaveOccurred()) + native.ResourceProfiles().At(0).ScopeProfiles().At(0).Profiles().At(0).SetPeriod(77) + wire, err := (&pprofile.ProtoMarshaler{}).MarshalProfiles(native) + Expect(err).NotTo(HaveOccurred()) + metadataChange, err := profiles.Decode(wire, profiles.Options{}) + Expect(err).NotTo(HaveOccurred()) + outcome, err = store.PersistProfiles(ctx, metadataChange) + Expect(err).NotTo(HaveOccurred()) + Expect(outcome.Rejected).To(Equal(int64(1))) + changed := topSnapshot(1) + changed.Rows[0].ElapsedNS++ + outcome, err = store.PersistProfiles(ctx, []profiles.Snapshot{decodedTop(changed)}) + Expect(err).NotTo(HaveOccurred()) + Expect(outcome.Rejected).To(Equal(int64(1))) + changed.ProfileID = topSnapshot(2).ProfileID + outcome, err = store.PersistProfiles(ctx, []profiles.Snapshot{decodedTop(changed)}) + Expect(err).NotTo(HaveOccurred()) + Expect(outcome.Rejected).To(Equal(int64(1))) + stored, err := store.ReadProfiles(ctx, ProfileReadOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(stored).To(ConsistOf(original)) + identities := 0 + Expect(store.results.Backend.Scan(ctx, profileIdentityStream, 0, func(_ int64, _ recordstore.Row) error { + identities++ + return nil + })).To(Succeed()) + Expect(identities).To(Equal(2)) + }) + DescribeTable("projects SQL active operations using the shared Top and standing receiver seams", func(api string) { + _, store := testReceiver(DefaultOptions()) + snapshot := topSnapshot(1) + snapshot.Kind = "sql_top" + snapshot.Rows[0].Attributes["api"] = api + snapshot.Rows[0].Attributes["sql"] = "select * from orders" + outcome, err := store.PersistProfiles(context.Background(), []profiles.Snapshot{decodedTop(snapshot)}) + Expect(err).NotTo(HaveOccurred()) + Expect(outcome.Accepted).To(Equal(int64(1))) + ref := read.Ref{Runtime: store.Runtime(), TraceRef: tracing.TraceRef{Kind: "sql_top_active", Stream: profileStream("sql_top")}} + latest, err := read.Top(context.Background(), ref, read.TopQuery{}) + Expect(err).NotTo(HaveOccurred()) + Expect(latest.Rows).To(ConsistOf(And(HaveKeyWithValue("operationId", "request-a"), HaveKeyWithValue("api", api), HaveKeyWithValue("sql", "select * from orders")))) + info, err := telltale.Get(telltale.GetOptions{DataDir: store.options.DataDir, RunID: SourceID}) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Captures).To(ContainElement(HaveField("Kind", "sql_top_active"))) + }, Entry("JDBC", "jdbc"), Entry("R2DBC", "r2dbc")) + It("keeps an empty latest frame despite late older delivery and exposes retained history", func() { + _, store := testReceiver(DefaultOptions()) + ctx := context.Background() + empty := topSnapshot(3) + empty.Rows, empty.Observed = nil, 0 + outcome, err := store.PersistProfiles(ctx, []profiles.Snapshot{decodedTop(empty), decodedTop(topSnapshot(1)), decodedTop(topSnapshot(2))}) + Expect(err).NotTo(HaveOccurred()) + Expect(outcome.Accepted).To(Equal(int64(3))) + ref := read.Ref{Runtime: store.Runtime(), TraceRef: tracing.TraceRef{Kind: "http_top", Stream: profileStream("http_top")}} + latest, err := read.Top(ctx, ref, read.TopQuery{}) + Expect(err).NotTo(HaveOccurred()) + Expect(latest.Rows).To(BeEmpty()) + Expect(latest.Frame.Tick).To(Equal(int64(3))) + Expect(latest.Frame.Summary).To(HaveKeyWithValue("complete", true)) + history, err := read.Top(ctx, ref, read.TopQuery{Frame: topSnapshot(1).ProfileID}) + Expect(err).NotTo(HaveOccurred()) + Expect(history.Rows).To(ConsistOf(HaveKeyWithValue("requestId", "request-a"))) + info, err := telltale.Get(telltale.GetOptions{DataDir: store.options.DataDir, RunID: SourceID}) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Captures).To(ContainElement(HaveField("Kind", "http_top"))) + }) + It("expires identity metadata and native frames together by receive time", func() { + options := StoreOptions{DataDir: GinkgoT().TempDir(), Retention: 150 * time.Millisecond, Writable: true} + store, err := OpenStore(options) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(store.Close) + original := decodedTop(topSnapshot(1)) + outcome, err := store.PersistProfiles(context.Background(), []profiles.Snapshot{original}) + Expect(err).NotTo(HaveOccurred()) + Expect(outcome.Accepted).To(Equal(int64(1))) + expiry := time.Now().Add(2 * options.Retention) + Eventually(func() bool { return time.Now().After(expiry) }).Should(BeTrue()) + outcome, err = store.PersistProfiles(context.Background(), []profiles.Snapshot{original}) + Expect(err).NotTo(HaveOccurred()) + Expect(outcome).To(Equal(Outcome{Accepted: 1})) + stored, err := store.ReadProfiles(context.Background(), ProfileReadOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(stored).To(HaveLen(1)) + }) +}) diff --git a/internal/otelreceiver/profiles_test.go b/internal/otelreceiver/profiles_test.go new file mode 100644 index 0000000..3b1a70e --- /dev/null +++ b/internal/otelreceiver/profiles_test.go @@ -0,0 +1,67 @@ +package otelreceiver + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "time" + + "github.com/flanksource/telltale/pkg/otlp/profiles" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func topSnapshot(sequence int64) profiles.Snapshot { + at := time.Date(2026, 11, 14, 12, 0, 0, 0, time.UTC).Add(time.Duration(sequence) * time.Second) + return profiles.Snapshot{Kind: "http_top", CaptureID: "capture-a", Source: "jvm-a", ProfileID: []string{"10f5dd68-2656-4c07-abd7-bc884147e020", "20f5dd68-2656-4c07-abd7-bc884147e020", "30f5dd68-2656-4c07-abd7-bc884147e020"}[sequence-1], Sequence: sequence, + StartedAt: at, CompletedAt: at.Add(time.Millisecond), Observed: 1, Limit: 50, Complete: true, PollIntervalMS: 5000, + Rows: []profiles.Row{{OperationID: "request-a", ElapsedNS: sequence * int64(time.Second), SampledAt: at, Attributes: map[string]any{"method": "GET", "url": "http://example.test/orders", "direction": "incoming", "startedAt": at.Add(-time.Duration(sequence) * time.Second).Format(time.RFC3339Nano)}}}} +} + +var _ = Describe("OTLP Profiles receiver", func() { + It("commits active, updated and empty snapshots once before acknowledging", func() { + receiver, store := testReceiver(DefaultOptions()) + for sequence := int64(1); sequence <= 3; sequence++ { + snapshot := topSnapshot(sequence) + if sequence == 3 { + snapshot.Rows = nil + snapshot.Observed = 0 + } + body, err := profiles.Encode([]profiles.Snapshot{snapshot}) + Expect(err).NotTo(HaveOccurred()) + for retry := 0; retry < 2; retry++ { + request := httptest.NewRequest(http.MethodPost, "/v1/profiles", bytes.NewReader(body)) + request.Header.Set("Content-Type", protobufType) + request.Header.Set("OTLP-Profiles-Development-Version", "1") + response := httptest.NewRecorder() + receiver.HTTPHandler(false).ServeHTTP(response, request) + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + } + } + stored, err := store.ReadProfiles(context.Background(), ProfileReadOptions{Kind: "http_top"}) + Expect(err).NotTo(HaveOccurred()) + Expect(stored).To(HaveLen(3)) + Expect(stored[0].Rows[0].OperationID).To(Equal("request-a")) + Expect(stored[1].Rows[0].ElapsedNS).To(Equal(int64(2 * time.Second))) + Expect(stored[2].Rows).To(BeEmpty()) + }) + DescribeTable("rejects unsupported schema metadata before decoding", func(version string) { + receiver, _ := testReceiver(DefaultOptions()) + request := httptest.NewRequest(http.MethodPost, "/v1/profiles", bytes.NewBufferString("bad protobuf")) + request.Header.Set("Content-Type", protobufType) + request.Header.Set("OTLP-Profiles-Development-Version", version) + response := httptest.NewRecorder() + receiver.HTTPHandler(false).ServeHTTP(response, request) + Expect(response.Code).To(Equal(http.StatusBadRequest)) + Expect(response.Body.String()).To(ContainSubstring("version")) + }, Entry("future version", "2"), Entry("malformed version", "01")) + It("requires authentication for Profiles just as for traces", func() { + receiver, _ := testReceiver(DefaultOptions()) + request := httptest.NewRequest(http.MethodPost, "/v1/profiles", bytes.NewBufferString("{}")) + request.Header.Set("Content-Type", jsonType) + response := httptest.NewRecorder() + receiver.HTTPHandler(true).ServeHTTP(response, request) + Expect(response.Code).To(Equal(http.StatusUnauthorized)) + }) +}) diff --git a/internal/otelreceiver/registry.go b/internal/otelreceiver/registry.go index 4ebf472..44998fb 100644 --- a/internal/otelreceiver/registry.go +++ b/internal/otelreceiver/registry.go @@ -1,6 +1,7 @@ package otelreceiver import ( + "context" "errors" "fmt" "os" @@ -8,6 +9,7 @@ import ( "sync" "syscall" + "github.com/flanksource/commons-db/recordstore" "github.com/flanksource/commons-db/recordstore/owner" "github.com/flanksource/telltale" "github.com/flanksource/telltale/pkg/tracing" @@ -59,12 +61,45 @@ func standingInfo(dataDir string) (telltale.RunInfo, bool, error) { if running { status = telltale.StatusRunning } + captures, err := standingCaptures(dataDir) + if err != nil { + return telltale.RunInfo{}, false, err + } return telltale.RunInfo{ Dir: filepath.Dir(lockPath(dataDir)), Status: status, - Captures: []telltale.CaptureResult{{Kind: spans.Kind, Ref: tracing.TraceRef{Kind: spans.Kind, Stream: stream}}}, + Captures: captures, }, true, nil } +func standingCaptures(dataDir string) ([]telltale.CaptureResult, error) { + runtime, closeStore, err := openStanding(dataDir) + if err != nil { + return nil, err + } + var captures []telltale.CaptureResult + _, err = runtime.Results.Backend.Meta(context.Background(), stream) + if err != nil && !errors.Is(err, recordstore.ErrNotFound) { + return nil, errors.Join(err, closeStore()) + } + if err == nil { + captures = append(captures, telltale.CaptureResult{Kind: spans.Kind, Ref: tracing.TraceRef{Kind: spans.Kind, Stream: stream}}) + } + for _, kind := range []string{"http_top", "sql_top", "jms_top"} { + meta, err := runtime.Results.Backend.Meta(context.Background(), profileStream(kind)+"-frames") + if errors.Is(err, recordstore.ErrNotFound) { + continue + } + if err != nil { + return nil, errors.Join(err, closeStore()) + } + if meta.HighSeq > 0 { + storageKind := profileStorageKind(kind) + captures = append(captures, telltale.CaptureResult{Kind: storageKind, Ref: tracing.TraceRef{Kind: storageKind, Stream: profileStream(kind)}}) + } + } + return captures, closeStore() +} + // receiving reports a receiver holding dataDir's store: this process, or a // live process on this host that published itself as the lock's owner. func receiving(dataDir string) (bool, error) { diff --git a/internal/otelreceiver/store.go b/internal/otelreceiver/store.go index 6f52f65..2da487a 100644 --- a/internal/otelreceiver/store.go +++ b/internal/otelreceiver/store.go @@ -12,9 +12,13 @@ import ( "github.com/flanksource/commons-db/recordstore" "github.com/flanksource/commons-db/recordstore/owner" "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/commons-db/recordstore/sampleframe" "github.com/flanksource/commons-db/recordstore/sqlite" "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/httptop" + "github.com/flanksource/telltale/pkg/tracing/jmstop" "github.com/flanksource/telltale/pkg/tracing/spans" + "github.com/flanksource/telltale/pkg/tracing/sqltop" ) type StoreOptions struct { @@ -96,16 +100,45 @@ func OpenStore(options StoreOptions) (_ *Store, err error) { Register: func(registry *recordresults.Registry) error { resultType := spans.ResultType() resultType.Retention = recordstore.RetainRows - return recordresults.RegisterResultType(registry, resultType) + if err := recordresults.RegisterResultType(registry, resultType); err != nil { + return err + } + if err := recordresults.RegisterResultType(registry, httptop.ResultType()); err != nil { + return err + } + if err := recordresults.RegisterResultType(registry, sqltop.ActiveResult()); err != nil { + return err + } + if err := recordresults.RegisterResultType(registry, profileIdentityResult()); err != nil { + return err + } + if err := recordresults.RegisterResultType(registry, jmstop.ResultType()); err != nil { + return err + } + for _, kind := range []string{"http_top_frames", "sql_top_active_frames", "jms_top_frames"} { + if err := recordresults.RegisterResultType(registry, recordresults.ResultType[sampleframe.Frame]{Kind: kind, Title: "Completed profile snapshots", TimeColumn: "completedAt", Follow: true}); err != nil { + return err + } + } + return nil }, }) if err != nil { return nil, err } kinds := tracing.NewKinds() + if err := kinds.RegisterKind(jmstop.Kind, jmstop.NewKind(nil)); err != nil { + return nil, err + } if err := kinds.RegisterKind(spans.Kind, spans.NewKind(nil)); err != nil { return nil, err } + if err := kinds.RegisterKind(httptop.Kind, httptop.NewKind(nil)); err != nil { + return nil, err + } + if err := kinds.RegisterKind("sql_top_active", sqltop.NewActiveKind()); err != nil { + return nil, err + } store.runtime = &tracing.Runtime{ Results: store.results, Kinds: kinds, Sessions: query.NewSessionRegistry(query.RegistryOptions{BeforeRead: store.results.Registry.BeforeRead}), diff --git a/internal/otelreceiver/types.go b/internal/otelreceiver/types.go index 0849fe2..9532621 100644 --- a/internal/otelreceiver/types.go +++ b/internal/otelreceiver/types.go @@ -3,18 +3,18 @@ package otelreceiver import ( "encoding/base64" "encoding/json" - "errors" "fmt" "math" "time" + otlpspans "github.com/flanksource/telltale/pkg/otlp/spans" sdk "github.com/flanksource/telltale/sdk" "go.opentelemetry.io/collector/pdata/ptrace/ptraceotlp" ) const SourceID = "receiver" -var ErrLimit = errors.New("OTLP request limit exceeded") +var ErrLimit = otlpspans.ErrLimit type Options struct { Enabled bool diff --git a/internal/protocol/awswire/decode.go b/internal/protocol/awswire/decode.go new file mode 100644 index 0000000..e38403e --- /dev/null +++ b/internal/protocol/awswire/decode.go @@ -0,0 +1,119 @@ +package awswire + +import ( + "bytes" + "encoding/json" + "encoding/xml" + "fmt" + "io" + "mime" + "net/http" + "net/url" + "strings" +) + +type Metadata struct { + Service string `json:"service,omitempty"` + Region string `json:"region,omitempty"` + Operation string `json:"operation,omitempty"` + RequestID string `json:"requestId,omitempty"` + ErrorCode string `json:"errorCode,omitempty"` +} + +func Decode(req *http.Request, response *http.Response, requestBody, responseBody []byte) (Metadata, error) { + if req == nil || req.URL == nil { + return Metadata{}, fmt.Errorf("AWS metadata requires a request URL") + } + m := Metadata{} + query, err := url.ParseQuery(req.URL.RawQuery) + if err != nil { + return m, fmt.Errorf("AWS request query: %w", err) + } + credential := query.Get("X-Amz-Credential") + if authorization := req.Header.Get("Authorization"); strings.HasPrefix(authorization, "AWS4-") { + _, rest, found := strings.Cut(authorization, "Credential=") + if !found { + return m, fmt.Errorf("AWS authorization has no credential scope") + } + credential, _, _ = strings.Cut(rest, ",") + } + if credential != "" { + parts := strings.Split(strings.TrimSpace(credential), "/") + if len(parts) != 5 || parts[0] == "" || len(parts[1]) != 8 || parts[2] == "" || parts[3] == "" || parts[4] != "aws4_request" { + return m, fmt.Errorf("invalid AWS credential scope") + } + m.Region, m.Service = parts[2], parts[3] + } + m.Operation = query.Get("Action") + if target := req.Header.Get("X-Amz-Target"); target != "" { + _, operation, ok := strings.Cut(target, ".") + if !ok || operation == "" { + return m, fmt.Errorf("invalid AWS operation target") + } + m.Operation = operation + } + mediaType, _, _ := mime.ParseMediaType(req.Header.Get("Content-Type")) + if mediaType == "application/x-www-form-urlencoded" && len(requestBody) > 0 { + form, err := url.ParseQuery(string(requestBody)) + if err != nil { + return m, fmt.Errorf("AWS query payload: %w", err) + } + m.Operation = form.Get("Action") + } + if response == nil { + return m, nil + } + m.RequestID = response.Header.Get("X-Amzn-Requestid") + if m.RequestID == "" { + m.RequestID = response.Header.Get("X-Amz-Request-Id") + } + if response.StatusCode >= 400 && len(responseBody) > 0 { + if err := m.decodeError(responseBody); err != nil { + return m, err + } + } + return m, nil +} + +func (m *Metadata) decodeError(body []byte) error { + body = bytes.TrimSpace(body) + if len(body) == 0 { + return nil + } + if body[0] == '{' { + var fields struct { + Type string `json:"__type"` + Code string `json:"code"` + } + if err := json.Unmarshal(body, &fields); err != nil { + return fmt.Errorf("AWS JSON error: %w", err) + } + m.ErrorCode = fields.Code + if m.ErrorCode == "" { + _, m.ErrorCode, _ = strings.Cut(fields.Type, "#") + if m.ErrorCode == "" { + m.ErrorCode = fields.Type + } + } + return nil + } + if body[0] != '<' { + return nil + } + d := xml.NewDecoder(bytes.NewReader(body)) + for { + token, err := d.Token() + if err == io.EOF { + return nil + } + if err != nil { + return fmt.Errorf("AWS XML error: %w", err) + } + start, ok := token.(xml.StartElement) + if ok && start.Name.Local == "Code" { + if err := d.DecodeElement(&m.ErrorCode, &start); err != nil { + return fmt.Errorf("AWS XML error code: %w", err) + } + } + } +} diff --git a/internal/protocol/awswire/decode_test.go b/internal/protocol/awswire/decode_test.go new file mode 100644 index 0000000..7c997a4 --- /dev/null +++ b/internal/protocol/awswire/decode_test.go @@ -0,0 +1,57 @@ +package awswire + +import ( + "net/http" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestAWS(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "AWS metadata") } + +var _ = Describe("AWS metadata", func() { + It("reads signing scope and JSON operation without modifying signed requests", func() { + req, err := http.NewRequest("POST", "https://service.example.com/path%2Fkey?x=1", nil) + Expect(err).NotTo(HaveOccurred()) + req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=client/20261102/eu-west-1/dynamodb/aws4_request, SignedHeaders=host, Signature=secret") + req.Header.Set("X-Amz-Target", "DynamoDB_20120810.GetItem") + before := req.Clone(req.Context()) + got, err := Decode(req, nil, nil, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(Equal(Metadata{Service: "dynamodb", Region: "eu-west-1", Operation: "GetItem"})) + Expect(req).To(Equal(before)) + }) + It("reads presigned query scope and query operations", func() { + req, err := http.NewRequest("GET", "https://service.example.com/?X-Amz-Credential=client%2F20261102%2Fus-east-1%2Fsts%2Faws4_request&Action=GetCallerIdentity", nil) + Expect(err).NotTo(HaveOccurred()) + got, err := Decode(req, nil, nil, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(Equal(Metadata{Service: "sts", Region: "us-east-1", Operation: "GetCallerIdentity"})) + }) + It("reads form operations and XML errors", func() { + req, err := http.NewRequest("POST", "https://service.example.com/", nil) + Expect(err).NotTo(HaveOccurred()) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8") + resp := &http.Response{StatusCode: 403, Header: http.Header{"X-Amzn-Requestid": []string{"request-1"}}} + got, err := Decode(req, resp, []byte("Action=DescribeInstances&Version=2016-11-15"), []byte("Denied")) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(Equal(Metadata{Operation: "DescribeInstances", RequestID: "request-1", ErrorCode: "Denied"})) + }) + It("does not invent REST operation names", func() { + req, err := http.NewRequest("GET", "https://bucket.s3.us-east-1.amazonaws.com/object", nil) + Expect(err).NotTo(HaveOccurred()) + got, err := Decode(req, nil, nil, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(got.Operation).To(BeEmpty()) + }) + It("reports malformed signing scopes and structured payloads", func() { + req, err := http.NewRequest("GET", "https://service.example.com/?X-Amz-Credential=broken", nil) + Expect(err).NotTo(HaveOccurred()) + _, err = Decode(req, nil, nil, nil) + Expect(err).To(HaveOccurred()) + req.URL.RawQuery = "" + _, err = Decode(req, &http.Response{StatusCode: 400, Header: http.Header{"Content-Type": []string{"application/json"}}}, nil, []byte("{")) + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/internal/protocol/gitwire/decoder.go b/internal/protocol/gitwire/decoder.go new file mode 100644 index 0000000..c0ea1c0 --- /dev/null +++ b/internal/protocol/gitwire/decoder.go @@ -0,0 +1,161 @@ +package gitwire + +import ( + "errors" + "fmt" + "strconv" + "strings" +) + +var ErrIncomplete = errors.New("incomplete git packet") + +type Event struct { + Kind string `json:"kind"` + Text string `json:"text,omitempty"` + Bytes int `json:"bytes,omitempty"` +} + +type Decoder struct { + Emit func(Event) + buf []byte + pack bool + err error +} + +func (d *Decoder) Write(p []byte) (int, error) { + if d.err != nil { + return 0, d.err + } + for consumed := 0; consumed < len(p); { + if d.pack { + d.emit(Event{Kind: "pack", Bytes: len(p) - consumed}) + return len(p), nil + } + need := 4 + if len(d.buf) >= 4 { + n, err := strconv.ParseUint(string(d.buf[:4]), 16, 16) + if err != nil || n < 4 || n > 65520 { + d.err = fmt.Errorf("invalid git packet length %q", d.buf[:4]) + return consumed, d.err + } + need = int(n) + } + take := min(need-len(d.buf), len(p)-consumed) + d.buf = append(d.buf, p[consumed:consumed+take]...) + consumed += take + if len(d.buf) < need { + continue + } + if need == 4 { + switch string(d.buf) { + case "PACK": + d.pack = true + d.emit(Event{Kind: "pack", Bytes: 4}) + d.buf = nil + case "0000": + d.emit(Event{Kind: "flush"}) + d.buf = nil + case "0001": + d.emit(Event{Kind: "delimiter"}) + d.buf = nil + case "0002": + d.emit(Event{Kind: "response-end"}) + d.buf = nil + default: + n, err := strconv.ParseUint(string(d.buf), 16, 16) + if err != nil || n < 4 || n > 65520 { + d.err = fmt.Errorf("invalid git packet length %q", d.buf) + return consumed, d.err + } + if n == 4 { + d.emit(Event{Kind: "data"}) + d.buf = nil + } + } + continue + } + d.decode(d.buf[4:]) + d.buf = nil + } + return len(p), nil +} + +func (d *Decoder) Close() error { + if d.err != nil { + return d.err + } + if len(d.buf) != 0 { + return ErrIncomplete + } + return nil +} + +func (d *Decoder) emit(e Event) { + if d.Emit != nil { + d.Emit(e) + } +} + +func (d *Decoder) decode(p []byte) { + if len(p) == 0 { + d.emit(Event{Kind: "data"}) + return + } + if p[0] >= 1 && p[0] <= 3 { + if p[0] == 1 { + d.emit(Event{Kind: "pack", Bytes: len(p) - 1}) + return + } + kind := "progress" + if p[0] == 3 { + kind = "error" + } + d.emit(Event{Kind: kind, Text: strings.TrimSuffix(string(p[1:]), "\n")}) + return + } + text := strings.TrimSuffix(string(p), "\n") + line, caps, hasCaps := strings.Cut(text, "\x00") + kind := "data" + switch { + case strings.HasPrefix(line, "version "): + kind, line = "version", strings.TrimPrefix(line, "version ") + case strings.HasPrefix(line, "command="): + kind, line = "command", strings.TrimPrefix(line, "command=") + case strings.HasPrefix(line, "ERR "): + kind, line = "error", strings.TrimPrefix(line, "ERR ") + case strings.HasPrefix(line, "want "): + kind = "want" + case strings.HasPrefix(line, "have "): + kind = "have" + case strings.HasPrefix(line, "ACK ") || line == "NAK": + kind = "ack" + case strings.HasPrefix(line, "shallow ") || strings.HasPrefix(line, "unshallow "): + kind = "shallow" + case strings.HasPrefix(line, "ok ") || strings.HasPrefix(line, "ng ") || strings.HasPrefix(line, "unpack "): + kind = "status" + default: + fields := strings.Fields(line) + if len(fields) >= 2 && objectID(fields[0]) { + kind = "ref" + if objectID(fields[1]) { + kind = "update" + } + } + } + d.emit(Event{Kind: kind, Text: line}) + if hasCaps { + d.emit(Event{Kind: "capabilities", Text: caps}) + } +} + +func objectID(s string) bool { + if len(s) != 40 && len(s) != 64 { + return false + } + for _, c := range s { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} diff --git a/internal/protocol/gitwire/decoder_test.go b/internal/protocol/gitwire/decoder_test.go new file mode 100644 index 0000000..8fcf714 --- /dev/null +++ b/internal/protocol/gitwire/decoder_test.go @@ -0,0 +1,67 @@ +package gitwire + +import ( + "fmt" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestDecoder(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Git wire decoder") } + +func packet(payload string) string { return fmt.Sprintf("%04x%s", len(payload)+4, payload) } + +var _ = Describe("Git packet decoding", func() { + It("decodes fragmented v2 packets and controls", func() { + var events []Event + d := Decoder{Emit: func(e Event) { events = append(events, e) }} + wire := packet("version 2\n") + packet("command=ls-refs\n") + "00010000" + for _, b := range []byte(wire) { + _, err := d.Write([]byte{b}) + Expect(err).NotTo(HaveOccurred()) + } + Expect(d.Close()).To(Succeed()) + Expect(events).To(Equal([]Event{{Kind: "version", Text: "2"}, {Kind: "command", Text: "ls-refs"}, {Kind: "delimiter"}, {Kind: "flush"}})) + }) + It("keeps pack payloads opaque while retaining progress and errors", func() { + var events []Event + d := Decoder{Emit: func(e Event) { events = append(events, e) }} + _, err := d.Write([]byte(packet("\x01PACK-private-object") + packet("\x02counting objects\n") + packet("\x03rejected\n"))) + Expect(err).NotTo(HaveOccurred()) + Expect(events).To(Equal([]Event{{Kind: "pack", Bytes: 19}, {Kind: "progress", Text: "counting objects"}, {Kind: "error", Text: "rejected"}})) + }) + It("decodes ref capabilities and push updates", func() { + var events []Event + d := Decoder{Emit: func(e Event) { events = append(events, e) }} + oid := "0123456789012345678901234567890123456789" + _, err := d.Write([]byte(packet(oid+" refs/heads/main\x00side-band-64k\n") + packet(oid+" "+oid+" refs/heads/main\n"))) + Expect(err).NotTo(HaveOccurred()) + Expect(events).To(Equal([]Event{{Kind: "ref", Text: oid + " refs/heads/main"}, {Kind: "capabilities", Text: "side-band-64k"}, {Kind: "update", Text: oid + " " + oid + " refs/heads/main"}})) + }) + It("rejects malformed and incomplete frames", func() { + d := Decoder{} + _, err := d.Write([]byte("0003")) + Expect(err).To(HaveOccurred()) + d = Decoder{} + _, err = d.Write([]byte("0008ab")) + Expect(err).NotTo(HaveOccurred()) + Expect(d.Close()).To(MatchError(ErrIncomplete)) + }) + It("counts unframed pack bytes without interpreting objects", func() { + var events []Event + d := Decoder{Emit: func(e Event) { events = append(events, e) }} + _, err := d.Write([]byte("PACKopaque")) + Expect(err).NotTo(HaveOccurred()) + _, err = d.Write([]byte("more")) + Expect(err).NotTo(HaveOccurred()) + Expect(d.Close()).To(Succeed()) + total := 0 + for _, event := range events { + Expect(event.Kind).To(Equal("pack")) + Expect(event.Text).To(BeEmpty()) + total += event.Bytes + } + Expect(total).To(Equal(14)) + }) +}) diff --git a/internal/protocol/sftpwire/decoder.go b/internal/protocol/sftpwire/decoder.go new file mode 100644 index 0000000..4320cdf --- /dev/null +++ b/internal/protocol/sftpwire/decoder.go @@ -0,0 +1,156 @@ +package sftpwire + +import ( + "encoding/binary" + "fmt" + "sync" +) + +type Event struct { + RequestID uint32 `json:"requestId"` + Operation string `json:"operation"` + Path string `json:"path,omitempty"` + Offset uint64 `json:"offset,omitempty"` + Bytes uint32 `json:"bytes,omitempty"` + Status *uint32 `json:"status,omitempty"` +} + +type Decoder struct { + Emit func(Event) + mu sync.Mutex + pending map[uint32]pending + handles map[string]string +} + +type pending struct { + event Event + handle string +} + +var operations = map[byte]string{3: "open", 4: "close", 5: "read", 6: "write", 7: "lstat", 8: "fstat", 9: "setstat", 10: "fsetstat", 11: "opendir", 12: "readdir", 13: "remove", 14: "mkdir", 15: "rmdir", 16: "realpath", 17: "stat", 18: "rename", 19: "readlink", 20: "symlink"} + +func (d *Decoder) Request(packet []byte) error { + d.mu.Lock() + defer d.mu.Unlock() + if len(packet) < 5 { + return fmt.Errorf("incomplete SFTP request") + } + if packet[0] == 1 { + if binary.BigEndian.Uint32(packet[1:]) != 3 { + return fmt.Errorf("SFTP decoder supports version 3") + } + return nil + } + id := binary.BigEndian.Uint32(packet[1:5]) + payload := packet[5:] + if d.pending == nil { + d.pending = map[uint32]pending{} + d.handles = map[string]string{} + } + if _, exists := d.pending[id]; exists { + return fmt.Errorf("duplicate SFTP request %d", id) + } + if len(d.pending) >= 4096 { + return fmt.Errorf("SFTP outstanding request limit exceeded") + } + value, err := takeString(&payload) + if err != nil { + return err + } + p := pending{event: Event{RequestID: id, Operation: operations[packet[0]], Path: value}} + switch packet[0] { + case 4, 5, 6, 8, 10, 12: + p.handle = value + path, ok := d.handles[value] + if !ok { + return fmt.Errorf("unknown SFTP handle") + } + p.event.Path = path + if packet[0] == 5 || packet[0] == 6 { + if len(payload) < 12 { + return fmt.Errorf("incomplete SFTP transfer request") + } + p.event.Offset = binary.BigEndian.Uint64(payload) + p.event.Bytes = binary.BigEndian.Uint32(payload[8:]) + if packet[0] == 6 && uint64(p.event.Bytes) != uint64(len(payload)-12) { + return fmt.Errorf("invalid SFTP write length") + } + } + case 200: + p.event.Operation = "extension:" + value + p.event.Path = "" + default: + if p.event.Operation == "" { + return fmt.Errorf("unsupported SFTP request type %d", packet[0]) + } + } + d.pending[id] = p + return nil +} + +func (d *Decoder) Response(packet []byte) error { + d.mu.Lock() + defer d.mu.Unlock() + if len(packet) < 5 { + return fmt.Errorf("incomplete SFTP response") + } + if packet[0] == 2 { + if binary.BigEndian.Uint32(packet[1:]) != 3 { + return fmt.Errorf("SFTP decoder supports version 3") + } + return nil + } + id := binary.BigEndian.Uint32(packet[1:5]) + payload := packet[5:] + p, ok := d.pending[id] + if !ok { + return fmt.Errorf("unmatched SFTP response %d", id) + } + switch packet[0] { + case 101: + if len(payload) < 4 { + return fmt.Errorf("incomplete SFTP status") + } + status := binary.BigEndian.Uint32(payload) + p.event.Status = &status + if status == 0 && p.event.Operation == "close" { + delete(d.handles, p.handle) + } + case 102: + handle, err := takeString(&payload) + if err != nil { + return err + } + if len(d.handles) >= 4096 { + return fmt.Errorf("SFTP open handle limit exceeded") + } + d.handles[handle] = p.event.Path + case 103: + if len(payload) < 4 || uint64(binary.BigEndian.Uint32(payload)) != uint64(len(payload)-4) { + return fmt.Errorf("invalid SFTP data length") + } + p.event.Bytes = binary.BigEndian.Uint32(payload) + case 104, 105, 201: + default: + return fmt.Errorf("unsupported SFTP response type %d", packet[0]) + } + delete(d.pending, id) + if d.Emit != nil { + d.Emit(p.event) + } + return nil +} + +func takeString(payload *[]byte) (string, error) { + if len(*payload) < 4 { + return "", fmt.Errorf("incomplete SFTP string length") + } + n := uint64(binary.BigEndian.Uint32(*payload)) + rest := (*payload)[4:] + if n > uint64(len(rest)) { + return "", fmt.Errorf("incomplete SFTP string") + } + value := string(rest[:int(n)]) + *payload = rest[int(n):] + return value, nil +} diff --git a/internal/protocol/sftpwire/decoder_test.go b/internal/protocol/sftpwire/decoder_test.go new file mode 100644 index 0000000..3390798 --- /dev/null +++ b/internal/protocol/sftpwire/decoder_test.go @@ -0,0 +1,40 @@ +package sftpwire + +import ( + "encoding/binary" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestSFTP(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "SFTP operations") } +func u32(n uint32) []byte { return binary.BigEndian.AppendUint32(nil, n) } +func text(s string) []byte { return append(u32(uint32(len(s))), s...) } +func message(kind byte, id uint32, payload []byte) []byte { + return append(append([]byte{kind}, u32(id)...), payload...) +} + +var _ = Describe("SFTP decoding", func() { + It("correlates out-of-order replies and keeps file contents opaque", func() { + var events []Event + d := Decoder{Emit: func(e Event) { events = append(events, e) }} + open := append(text("/data/report.txt"), u32(1)...) + open = append(open, u32(0)...) + Expect(d.Request(message(3, 7, open))).To(Succeed()) + Expect(d.Response(message(102, 7, text("handle-1")))).To(Succeed()) + read := append(text("handle-1"), binary.BigEndian.AppendUint64(nil, 128)...) + read = append(read, u32(4)...) + Expect(d.Request(message(5, 8, read))).To(Succeed()) + Expect(d.Request(message(17, 9, text("/data/report.txt")))).To(Succeed()) + Expect(d.Response(message(101, 9, append(append(u32(2), text("missing")...), text("")...)))).To(Succeed()) + Expect(d.Response(message(103, 8, text("data")))).To(Succeed()) + status := uint32(2) + Expect(events).To(Equal([]Event{{RequestID: 7, Operation: "open", Path: "/data/report.txt"}, {RequestID: 9, Operation: "stat", Path: "/data/report.txt", Status: &status}, {RequestID: 8, Operation: "read", Path: "/data/report.txt", Offset: 128, Bytes: 4}})) + }) + It("rejects malformed and unmatched responses", func() { + d := Decoder{} + Expect(d.Request([]byte{3})).NotTo(Succeed()) + Expect(d.Response(message(101, 99, u32(0)))).NotTo(Succeed()) + }) +}) diff --git a/internal/server/connections.go b/internal/server/connections.go index f71f917..755057c 100644 --- a/internal/server/connections.go +++ b/internal/server/connections.go @@ -19,7 +19,7 @@ import ( ) // managedProvider is the connection store's provider with the connections -// telltale manages in its data dir, such as the managed Prometheus of the +// telltale manages in its data dir, such as the VictoriaMetrics instance of the // prometheus trace kind, listed and resolved as virtual, read-only // connections the service refuses to edit. Writes go to the store. type managedProvider struct { diff --git a/internal/server/connections_openapi.go b/internal/server/connections_openapi.go index 50406a1..6c5d790 100644 --- a/internal/server/connections_openapi.go +++ b/internal/server/connections_openapi.go @@ -5,9 +5,12 @@ import ( "strconv" "strings" + "github.com/flanksource/clicky/entity" "github.com/flanksource/clicky/rpc" "github.com/flanksource/commons-db/models" - "github.com/flanksource/telltale/sdk/protocol" + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/internal/jvm" + "github.com/flanksource/telltale/pkg/agentclient" ) func addConnectionPaths(spec *rpc.OpenAPISpec) { @@ -22,6 +25,9 @@ func addConnectionPaths(spec *rpc.OpenAPISpec) { "/api/v1/connection/{id}/browser/query": {"POST"}, "/api/v1/connection/{id}/browser/filters/values": {"POST"}, "/api/agents": {"GET"}, + "/api/agents/{agent}/catalog": {"GET"}, + "/api/agents/{agent}/call": {"POST"}, + "/api/traces": {"GET", "POST"}, "/api/traces/{run}/traces/{kind}/records": {"GET"}, } { item := rpc.OpenAPIPath{} @@ -29,7 +35,27 @@ func addConnectionPaths(spec *rpc.OpenAPISpec) { status := http.StatusOK response := &rpc.OpenAPISchema{Type: "object"} if path == "/api/agents" { - response = &rpc.OpenAPISchema{Type: "array", Items: rpc.SchemaForStruct(protocol.Hello{})} + item := rpc.SchemaForStruct(agentclient.Info{}) + item.Required = []string{"agentId", "resource", "connected", "group"} + response = &rpc.OpenAPISchema{Type: "array", Items: item} + } + if strings.HasSuffix(path, "/catalog") { + response = &rpc.OpenAPISchema{Type: "array", Items: &rpc.OpenAPISchema{Type: "object", AdditionalProperties: &rpc.OpenAPISchema{}}} + } + if path == "/api/agents/{agent}/call" { + response = &rpc.OpenAPISchema{} + } + if path == "/api/traces" { + response = rpc.SchemaForStruct(telltale.RunsList{}) + if method == "POST" { + status = http.StatusCreated + response = rpc.SchemaForStruct(telltale.MountedRun{}) + } + } + if path == "/api/v1/connection" && method == "GET" { + response = &rpc.OpenAPISchema{OneOf: []*rpc.OpenAPISchema{ + {Type: "array", Items: rpc.SchemaForStruct(models.Connection{})}, rpc.SchemaForStruct(entity.LookupResponse{}), + }} } if path == "/api/v1/connection" && method == "POST" { status = http.StatusCreated @@ -41,7 +67,12 @@ func addConnectionPaths(spec *rpc.OpenAPISpec) { strconv.Itoa(status): {Description: "Success", Content: map[string]rpc.OpenAPIMediaType{"application/json": {Schema: response}}}, "400": {Description: "Invalid request"}, "404": {Description: "Resource not found"}, "405": {Description: "Read-only resource"}, }} - for _, name := range []string{"id", "run", "kind"} { + if path == "/api/v1/connection" && method == "GET" { + for _, name := range []string{"type", "types", "__lookup", "__lookup_filter", "__lookup_q"} { + op.Parameters = append(op.Parameters, rpc.OpenAPIParameter{Name: name, In: "query", Schema: &rpc.OpenAPISchema{Type: "string"}}) + } + } + for _, name := range []string{"id", "run", "kind", "agent"} { if strings.Contains(path, "{"+name+"}") { op.Parameters = append(op.Parameters, rpc.OpenAPIParameter{Name: name, In: "path", Required: true, Schema: &rpc.OpenAPISchema{Type: "string"}}) } @@ -51,6 +82,15 @@ func addConnectionPaths(spec *rpc.OpenAPISpec) { if path == "/api/v1/connection" || path == "/api/v1/connection/{id}" { body = rpc.SchemaForStruct(models.Connection{}) } + if path == "/api/agents/{agent}/call" { + body = rpc.SchemaForStruct(jvm.Request{}) + delete(body.Properties, "agent") + body.Properties["params"] = &rpc.OpenAPISchema{Type: "object", AdditionalProperties: &rpc.OpenAPISchema{}} + body.Required = []string{"operation"} + } + if path == "/api/traces" { + body = rpc.SchemaForStruct(telltale.StartRequest{}) + } op.RequestBody = &rpc.OpenAPIRequestBody{Required: true, Content: map[string]rpc.OpenAPIMediaType{"application/json": {Schema: body}}} } item[strings.ToLower(method)] = *op diff --git a/internal/server/connections_test.go b/internal/server/connections_test.go index c6be635..9404363 100644 --- a/internal/server/connections_test.go +++ b/internal/server/connections_test.go @@ -11,7 +11,6 @@ import ( "github.com/flanksource/commons-db/connection/store" "github.com/flanksource/commons-db/models" - "github.com/flanksource/deps/start/state" "github.com/flanksource/telltale/internal/server" "github.com/flanksource/telltale/pkg/tracing/prometheus" . "github.com/onsi/ginkgo/v2" @@ -71,14 +70,15 @@ var _ = Describe("Connection workspace API", func() { Expect(request("GET", "/api/v1/connection/"+connection.ID, "").Code).To(Equal(404)) }) - It("lists the data dir's managed Prometheus as a read-only connection and refuses to edit it", func() { + It("lists the data dir's managed VictoriaMetrics as a read-only connection and refuses to edit it", func() { dir := GinkgoT().TempDir() connections, err := store.New(store.Layer{Name: store.LayerCWD, Dir: filepath.Join(dir, ".connections")}) Expect(err).NotTo(HaveOccurred()) - managed := &state.State{Name: "prometheus", Runtime: "binary", Connection: models.Connection{Name: "prometheus", Type: models.ConnectionTypePrometheus, URL: "http://localhost:19090"}} paths, err := prometheus.Paths(dir) Expect(err).NotTo(HaveOccurred()) - Expect(managed.Save(paths.State)).To(Succeed()) + stateDir := filepath.Join(paths.State, "victoriametrics") + Expect(os.MkdirAll(stateDir, 0o700)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(stateDir, "state.json"), []byte(`{"port":19090,"pid":1234,"startedAt":"2026-11-19T00:00:00Z"}`), 0o600)).To(Succeed()) handler, err := server.NewHandler(server.HandlerOptions{ Root: &cobra.Command{Use: "telltale"}, Connections: connections, DataDir: dir, Assets: fstest.MapFS{"index.html": &fstest.MapFile{Data: []byte("ok")}}, diff --git a/internal/server/feed_openapi.go b/internal/server/feed_openapi.go new file mode 100644 index 0000000..c760062 --- /dev/null +++ b/internal/server/feed_openapi.go @@ -0,0 +1,34 @@ +package server + +import ( + "github.com/flanksource/clicky/rpc" + "github.com/flanksource/telltale/pkg/traces/read" + "github.com/flanksource/telltale/pkg/tracing/processcapture" +) + +func addFeedPaths(spec *rpc.OpenAPISpec) { + if spec.Paths == nil { + spec.Paths = map[string]rpc.OpenAPIPath{} + } + addRecordsPath(spec) + pageSchema := rpc.SchemaForStruct(read.FeedResult{}) + pageSchema.Properties["rows"].Items.Properties["ansi"] = &rpc.OpenAPISchema{Type: "string", Description: "Styled summary, included when presentation=ansi"} + operation := rpc.OpenAPIOperation{Summary: "Read or follow a run's events", Description: "Top samples and their named results are excluded from the event feed. Read them through the capture's records endpoint with view=top.", Tags: []string{"Traces"}, Responses: map[string]rpc.OpenAPIResponse{ + "200": {Description: "Paged records, or a resumable live stream", Content: map[string]rpc.OpenAPIMediaType{ + "application/json": {Schema: pageSchema}, + "application/x-ndjson": {Schema: &rpc.OpenAPISchema{Type: "string"}}, + "text/event-stream": {Schema: &rpc.OpenAPISchema{Type: "string"}}, + }}, "400": {Description: "Invalid selection, ordering or cursor"}, + }} + operation.Parameters = append(operation.Parameters, rpc.OpenAPIParameter{Name: "run", In: "path", Required: true, Schema: &rpc.OpenAPISchema{Type: "string"}}) + for _, name := range []string{"kind", "excludeKind", "presentation", "stream", "sort", "limit", "cursor", "follow", "format", "resume", "from", "until"} { + operation.Parameters = append(operation.Parameters, rpc.OpenAPIParameter{Name: name, In: "query", Schema: &rpc.OpenAPISchema{Type: "string"}}) + } + spec.Paths[TracesMount+"/{run}/records"] = rpc.OpenAPIPath{"get": operation} + spec.Paths[TracesMount+"/{run}/processes"] = rpc.OpenAPIPath{"get": rpc.OpenAPIOperation{OperationID: "traceProcesses", Summary: "Latest measurements of each recorded process", Parameters: []rpc.OpenAPIParameter{{Name: "run", In: "path", Required: true, Schema: &rpc.OpenAPISchema{Type: "string"}}}, Responses: map[string]rpc.OpenAPIResponse{"200": {Description: "Process identities and latest recorded measurements", Content: map[string]rpc.OpenAPIMediaType{"application/json": {Schema: rpc.SchemaForStruct(read.ProcessSnapshot{})}}}, "400": {Description: "Invalid run or query"}, "500": {Description: "Process snapshot could not be read"}}}} + spec.Paths[TracesMount+"/{run}/process/groups"] = rpc.OpenAPIPath{"get": rpc.OpenAPIOperation{OperationID: "recordedProcessGroups", Summary: "Derive complete process group frames from numeric observations", Parameters: []rpc.OpenAPIParameter{ + {Name: "run", In: "path", Required: true, Schema: &rpc.OpenAPISchema{Type: "string"}}, + {Name: "stream", In: "query", Required: true, Schema: &rpc.OpenAPISchema{Type: "string"}}, + {Name: "afterSeq", In: "query", Schema: &rpc.OpenAPISchema{Type: "integer"}}, + }, Responses: map[string]rpc.OpenAPIResponse{"200": {Description: "Group frames derived from one atomic process metrics stream", Content: map[string]rpc.OpenAPIMediaType{"application/json": {Schema: rpc.SchemaForStruct(processcapture.Frames{})}}}, "400": {Description: "Invalid process capture or sequence"}}}} +} diff --git a/internal/server/handler.go b/internal/server/handler.go index 14cce0e..b275520 100644 --- a/internal/server/handler.go +++ b/internal/server/handler.go @@ -63,18 +63,23 @@ func routes(options HandlerOptions, ui http.Handler) (http.Handler, error) { } api := http.NewServeMux() rpc.NewSwaggerServer(&rpc.ServeConfig{Version: options.Version}, options.Root, - &rpc.OpenAPIConfig{Title: "Telltale", Version: options.Version, Extensions: []func(*rpc.OpenAPISpec){addRulesPaths, connections.AddConnectionsOpenAPI, addConnectionPaths, addOtelPaths, addReceiverPath}}).RegisterRoutes(route.NewRouter(api)) + &rpc.OpenAPIConfig{Title: "Telltale", Version: options.Version, Extensions: []func(*rpc.OpenAPISpec){addRulesPaths, connections.AddConnectionsOpenAPI, addConnectionPaths, addOtelPaths, addReceiverPath, addFeedPaths, addJVMSnapshotPath, addJVMTopPath}}).RegisterRoutes(route.NewRouter(api)) root := http.NewServeMux() if options.OTLP == nil { options.OTLP = http.NotFoundHandler() } root.Handle("/v1/traces", options.OTLP) + root.Handle("/v1/profiles", options.OTLP) installer := http.NewServeMux() rpc.NewSwaggerServer(&rpc.ServeConfig{Version: options.Version, Executor: &rpc.ExecutorConfig{Enabled: true}}, options.Root, nil). RegisterExecutionRoutes(route.NewRouter(installer)) root.Handle("/api/v1/agents/install/java", http.NewCrossOriginProtection().Handler(installer)) + root.Handle("/api/v1/jvm/", http.NewCrossOriginProtection().Handler(installer)) + root.Handle("POST /api/v1/jvm/snapshot", http.NewCrossOriginProtection().Handler(http.HandlerFunc(jvmSnapshotHandler))) hub := agentclient.NewHub(agentclient.Default, agentclient.NewTokenStore(options.ConfigDirs)) root.HandleFunc("/api/agents", hub.List) + root.HandleFunc("GET /api/agents/{agent}/catalog", jvmHandler) + root.Handle("POST /api/agents/{agent}/call", http.NewCrossOriginProtection().Handler(http.HandlerFunc(jvmHandler))) otel := otelHandler(tracequery.Service{DataDir: options.DataDir, Connections: options.Connections}) root.Handle("/api/otel", otel) root.Handle("/api/otel/", otel) @@ -85,6 +90,9 @@ func routes(options HandlerOptions, ui http.Handler) (http.Handler, error) { root.Handle(TracesMount, options.Traces) root.Handle(TracesMount+"/", options.Traces) root.Handle("GET "+TracesMount+"/{run}/traces/{kind}/records", records.Handler(options.DataDir)) + root.Handle("GET "+TracesMount+"/{run}/records", records.FeedHandler(options.DataDir)) + root.Handle("GET "+TracesMount+"/{run}/processes", records.ProcessesHandler(options.DataDir)) + root.Handle("GET "+TracesMount+"/{run}/process/groups", records.ProcessGroupsHandler(options.DataDir)) root.Handle(TopMount, options.Top) root.Handle(TopProcessesPath, options.Top) root.Handle(TopFacetsPath, options.Top) @@ -127,7 +135,7 @@ func assetHandler(assets fs.FS) http.Handler { info, err := fs.Stat(assets, path) // A client route is the page: any path without an extension, and the // /top/ pages, whose program names may hold dots. - if errors.Is(err, fs.ErrNotExist) && (!strings.Contains(path, ".") || strings.HasPrefix(path, "top/") || strings.HasPrefix(path, "arm/") || strings.HasPrefix(path, "records/")) && !strings.HasPrefix(path, "assets/") { + if errors.Is(err, fs.ErrNotExist) && (!strings.Contains(path, ".") || strings.HasPrefix(path, "top/") || strings.HasPrefix(path, "arm/") || strings.HasPrefix(path, "records/") || strings.HasPrefix(path, "jvm/")) && !strings.HasPrefix(path, "assets/") { path = "index.html" } else if err != nil || info.IsDir() { http.NotFound(w, r) diff --git a/internal/server/handler_test.go b/internal/server/handler_test.go index 97dcc1e..4556b88 100644 --- a/internal/server/handler_test.go +++ b/internal/server/handler_test.go @@ -3,6 +3,7 @@ package server_test import ( "net/http" "net/http/httptest" + "strings" "testing/fstest" "github.com/flanksource/telltale/internal/server" @@ -72,6 +73,7 @@ var _ = Describe("HTTP routing", func() { Entry("index", "GET", "/", 200, "no-cache"), Entry("deep link", "GET", "/settings", 200, "no-cache"), Entry("a program's top page, its name dotted", "GET", "/top/com.apple.WebKit.Networking", 200, "no-cache"), + Entry("a JVM diagnostic operation, its name dotted", "GET", "/jvm/java%2Ffixture/jvm.groovy.eval", 200, "no-cache"), Entry("a missing file beside the page", "GET", "/favicon.ico", 404, ""), Entry("hashed asset", "GET", "/assets/app-ab12.js", 200, "public, max-age=31536000, immutable"), Entry("missing asset", "GET", "/assets/missing.js", 404, ""), @@ -81,6 +83,13 @@ var _ = Describe("HTTP routing", func() { Entry("unsupported UI method", "POST", "/settings", 405, ""), ) + DescribeTable("requires an operation for a diagnostic call", func(body string) { + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodPost, "/api/agents/missing/call", strings.NewReader(body))) + Expect(response.Code).To(Equal(http.StatusBadRequest)) + Expect(response.Body.String()).To(ContainSubstring("an operation is required")) + }, Entry("empty object", `{}`), Entry("null", `null`), Entry("parameters only", `{"params":{}}`)) + It("serves Clicky's health and generated OpenAPI", func() { for _, path := range []string{"/health", "/api/openapi.json"} { response := httptest.NewRecorder() diff --git a/internal/server/jvm.go b/internal/server/jvm.go new file mode 100644 index 0000000..fdf87b8 --- /dev/null +++ b/internal/server/jvm.go @@ -0,0 +1,37 @@ +package server + +import ( + "encoding/json" + "io" + "net/http" + + "github.com/flanksource/telltale/internal/jvm" +) + +func jvmHandler(w http.ResponseWriter, r *http.Request) { + request := jvm.Request{Agent: r.PathValue("agent")} + if r.Method == http.MethodPost { + decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 24<<20)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&request); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if decoder.Decode(new(any)) != io.EOF { + http.Error(w, "expected one request object", http.StatusBadRequest) + return + } + request.Agent = r.PathValue("agent") + } else { + request.Operation = "jvm.catalog" + } + value, err := jvm.Call(r.Context(), request) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + if _, err := w.Write(value); err != nil { + panic(err) + } +} diff --git a/internal/server/jvm_snapshot.go b/internal/server/jvm_snapshot.go new file mode 100644 index 0000000..b0e6e45 --- /dev/null +++ b/internal/server/jvm_snapshot.go @@ -0,0 +1,66 @@ +package server + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/flanksource/clicky/rpc" + "github.com/flanksource/telltale/internal/jvm/snapshot" +) + +func jvmSnapshotHandler(w http.ResponseWriter, r *http.Request) { + var options snapshot.Options + decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&options); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if decoder.Decode(new(any)) != io.EOF || options.Agent == "" { + http.Error(w, "one object with agent is required", http.StatusBadRequest) + return + } + if err := options.Validate(); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + result, err := snapshot.Collect(r.Context(), options) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + data, err := result.Zip() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + status := "complete" + if !result.Manifest.Complete { + status = "partial" + } + w.Header().Set("Content-Type", "application/zip") + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"telltale-jvm-%s.zip\"", time.Now().UTC().Format("20060102T150405Z"))) + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("X-Telltale-Snapshot-Status", status) + if _, err := w.Write(data); err != nil { + return + } +} + +func addJVMSnapshotPath(spec *rpc.OpenAPISpec) { + // The binary download overrides the ordinary command executor at this path. + if spec.Paths == nil { + spec.Paths = map[string]rpc.OpenAPIPath{} + } + spec.Paths["/api/v1/jvm/snapshot"] = rpc.OpenAPIPath{"post": rpc.OpenAPIOperation{ + Summary: "Download a JVM support ZIP", OperationID: "jvmSnapshot", + RequestBody: &rpc.OpenAPIRequestBody{Required: true, Content: map[string]rpc.OpenAPIMediaType{"application/json": {Schema: &rpc.OpenAPISchema{Type: "object", Required: []string{"agent"}, Properties: map[string]*rpc.OpenAPISchema{ + "agent": {Type: "string"}, "jfr": {Type: "string", Description: "Positive duration, at most 5m; omitted disables profiling"}, "heapHistogram": {Type: "boolean"}, + "workload": {Type: "string", Description: "Duration from 5s to 5m to sample into workload.html; omitted disables it"}, + }}}}}, + Responses: map[string]rpc.OpenAPIResponse{"200": {Description: "ZIP; X-Telltale-Snapshot-Status is complete or partial", Content: map[string]rpc.OpenAPIMediaType{"application/zip": {Schema: &rpc.OpenAPISchema{Type: "string", Format: "binary"}}}}, "400": {Description: "Invalid request or unavailable Java target"}}, + }} +} diff --git a/internal/server/jvm_top.go b/internal/server/jvm_top.go new file mode 100644 index 0000000..8d2a773 --- /dev/null +++ b/internal/server/jvm_top.go @@ -0,0 +1,24 @@ +package server + +import ( + "github.com/flanksource/clicky/rpc" + "github.com/flanksource/telltale/internal/jvm/top" +) + +func addJVMTopPath(spec *rpc.OpenAPISpec) { + path := spec.Paths["/api/v1/jvm/top"] + operation, ok := path["post"] + if !ok { + return + } + operation.Summary = "Read one live JVM Top snapshot" + operation.Responses["200"] = rpc.OpenAPIResponse{Description: "Clicky command response; output contains the JSON JVM Top frame", Content: map[string]rpc.OpenAPIMediaType{"application/json": {Schema: &rpc.OpenAPISchema{Type: "object", Properties: map[string]*rpc.OpenAPISchema{"success": {Type: "boolean"}, "output": {Type: "string", Description: "JSON-encoded JVM Top frame"}, "error": {Type: "string"}}}}}} + if spec.Components == nil { + spec.Components = &rpc.OpenAPIComponents{} + } + if spec.Components.Schemas == nil { + spec.Components.Schemas = map[string]*rpc.OpenAPISchema{} + } + spec.Components.Schemas["JVMTopFrame"] = rpc.SchemaForStruct(top.Frame{}) + path["post"] = operation +} diff --git a/internal/server/otlpendpoints.go b/internal/server/otlpendpoints.go index 098b01c..600745a 100644 --- a/internal/server/otlpendpoints.go +++ b/internal/server/otlpendpoints.go @@ -5,13 +5,15 @@ import ( "net" "strconv" "strings" + + "github.com/flanksource/telltale/pkg/otlp/profiles" ) // otlpEndpoints is what serve prints at startup so an OpenTelemetry SDK can be // pointed at the receiver: both transports and the exporter variables for each. type otlpEndpoints struct { GRPC net.Addr - // API is the HTTP listener; it serves OTLP/HTTP at /v1/traces. + // API is the HTTP listener for traces and Profiles. API net.Addr // Host replaces a wildcard bind address, which no exporter can dial. Host string @@ -31,7 +33,8 @@ func (e otlpEndpoints) dialable(addr net.Addr) string { func (e otlpEndpoints) String() string { grpc := e.dialable(e.GRPC) - httpURL := "http://" + e.dialable(e.API) + "/v1/traces" + httpBase := "http://" + e.dialable(e.API) + httpURL := httpBase + "/v1/traces" grpcNote, httpNote := "", "" if e.GRPCAuth { grpcNote = " (API token required)" @@ -40,9 +43,10 @@ func (e otlpEndpoints) String() string { httpNote = "; API token required" } var b strings.Builder - b.WriteString("Telltale OTLP traces receivers:\n") + b.WriteString("Telltale OTLP traces and Profiles receivers:\n") fmt.Fprintf(&b, " gRPC %s%s\n", grpc, grpcNote) fmt.Fprintf(&b, " HTTP %s (application/x-protobuf or application/json%s)\n", httpURL, httpNote) + fmt.Fprintf(&b, " Profiles %s/v1/profiles (OTLP-Profiles-Development-Version: %s%s)\n", httpBase, profiles.DevelopmentVersion, httpNote) if e.GRPCAuth || e.HTTPAuth { b.WriteString("Create an API token with: telltale agents token create --scope api\n") } diff --git a/internal/server/otlpendpoints_internal_test.go b/internal/server/otlpendpoints_internal_test.go index 0aa26ce..a68715c 100644 --- a/internal/server/otlpendpoints_internal_test.go +++ b/internal/server/otlpendpoints_internal_test.go @@ -13,9 +13,10 @@ var _ = Describe("OTLP startup endpoints", func() { GRPC: &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 4317}, API: &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 8080}, } - Expect(endpoints.String()).To(Equal(`Telltale OTLP traces receivers: + Expect(endpoints.String()).To(Equal(`Telltale OTLP traces and Profiles receivers: gRPC 127.0.0.1:4317 HTTP http://127.0.0.1:8080/v1/traces (application/x-protobuf or application/json) + Profiles http://127.0.0.1:8080/v1/profiles (OTLP-Profiles-Development-Version: 1) Export over gRPC: export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://127.0.0.1:4317 @@ -32,9 +33,10 @@ Export over HTTP: Host: "build-01", HTTPAuth: true, } - Expect(endpoints.String()).To(Equal(`Telltale OTLP traces receivers: + Expect(endpoints.String()).To(Equal(`Telltale OTLP traces and Profiles receivers: gRPC 127.0.0.1:4317 HTTP http://build-01:8080/v1/traces (application/x-protobuf or application/json; API token required) + Profiles http://build-01:8080/v1/profiles (OTLP-Profiles-Development-Version: 1; API token required) Create an API token with: telltale agents token create --scope api Export over gRPC: export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc diff --git a/internal/server/profiles_route_test.go b/internal/server/profiles_route_test.go new file mode 100644 index 0000000..c40e972 --- /dev/null +++ b/internal/server/profiles_route_test.go @@ -0,0 +1,32 @@ +package server_test + +import ( + "net/http" + "net/http/httptest" + "testing/fstest" + + "github.com/flanksource/telltale/internal/server" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/spf13/cobra" +) + +var _ = Describe("Profiles receiver routing", func() { + It("dispatches Profiles exports to the configured OTLP receiver without serving the SPA", func() { + var received string + handler, err := server.NewHandler(server.HandlerOptions{ + Root: &cobra.Command{Use: "telltale"}, DataDir: GinkgoT().TempDir(), + Assets: fstest.MapFS{"index.html": &fstest.MapFile{Data: []byte("application")}}, + Traces: http.NotFoundHandler(), Top: http.NotFoundHandler(), + OTLP: http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + received = request.URL.Path + w.WriteHeader(http.StatusAccepted) + }), + }) + Expect(err).NotTo(HaveOccurred()) + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodPost, "/v1/profiles", nil)) + Expect(response.Code).To(Equal(http.StatusAccepted)) + Expect(received).To(Equal("/v1/profiles")) + }) +}) diff --git a/internal/server/records_openapi.go b/internal/server/records_openapi.go new file mode 100644 index 0000000..8c006d4 --- /dev/null +++ b/internal/server/records_openapi.go @@ -0,0 +1,23 @@ +package server + +import ( + "github.com/flanksource/clicky/rpc" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/flanksource/telltale/pkg/traces/read" +) + +func addRecordsPath(spec *rpc.OpenAPISpec) { + operation := rpc.OpenAPIOperation{OperationID: "traceSampleRecords", Summary: "Read native history, completed samples, or a Top selection", Tags: []string{"Traces"}, Responses: map[string]rpc.OpenAPIResponse{ + "200": {Description: "Native history page, completed sample metadata, or Top response according to view", Content: map[string]rpc.OpenAPIMediaType{ + "application/json": {Schema: &rpc.OpenAPISchema{OneOf: []*rpc.OpenAPISchema{rpc.SchemaForStruct(read.TopResult{}), {Type: "array", Items: rpc.SchemaForStruct(sampleframe.Frame{})}, {Type: "object", Properties: map[string]*rpc.OpenAPISchema{"rows": {Type: "array", Items: &rpc.OpenAPISchema{Type: "object"}}, "nextCursor": {Type: "string"}}}}}}, + "application/schema+json": {Schema: &rpc.OpenAPISchema{Type: "object", Description: "Options schema and native columns resolved for the selected view"}}, + }}, "400": {Description: "Invalid selection, filter, sort or cursor"}, "409": {Description: "Recording has no provable complete sample boundaries"}, + }} + for _, name := range []string{"run", "kind"} { + operation.Parameters = append(operation.Parameters, rpc.OpenAPIParameter{Name: name, In: "path", Required: true, Schema: &rpc.OpenAPISchema{Type: "string"}}) + } + for _, name := range []string{"stream", "result", "view", "frame", "source", "metric", "q", "filter", "where", "sort", "desc", "limit", "cursor", "all", "from", "until", "afterSeq", "toSeq", "format", "follow", "stats", "groupBy"} { + operation.Parameters = append(operation.Parameters, rpc.OpenAPIParameter{Name: name, In: "query", Schema: &rpc.OpenAPISchema{Type: "string"}}) + } + spec.Paths[TracesMount+"/{run}/traces/{kind}/records"] = rpc.OpenAPIPath{"get": operation} +} diff --git a/internal/server/run.go b/internal/server/run.go index 7e0b20b..2fc5830 100644 --- a/internal/server/run.go +++ b/internal/server/run.go @@ -19,6 +19,7 @@ import ( "github.com/flanksource/telltale/pkg/datadir" "github.com/flanksource/telltale/pkg/definition" "github.com/flanksource/telltale/pkg/traces" + "github.com/flanksource/telltale/pkg/tracing/prometheus" "github.com/flanksource/telltale/webapp" "github.com/spf13/cobra" ) @@ -40,6 +41,7 @@ type Options struct { EnforceRules bool OTLP otelreceiver.Options SelfTrace bool + Prometheus bool } func Run(ctx context.Context, options Options) (err error) { @@ -81,6 +83,15 @@ func Run(ctx context.Context, options Options) (err error) { return err } defer func() { err = errors.Join(err, telemetry.close()) }() + var metrics prometheus.Server + if options.Prometheus { + managed := prometheus.NewManaged(options.DataDir) + defer func() { err = errors.Join(err, managed.Release(ctx)) }() + metrics, err = managed.Server(ctx) + if err != nil { + return fmt.Errorf("serve --prometheus: %w", err) + } + } connections, err := telltale.OpenConnections() if err != nil { return err @@ -146,6 +157,11 @@ func Run(ctx context.Context, options Options) (err error) { if _, err := fmt.Fprintf(options.Root.OutOrStdout(), "Telltale API: %s\n", apiURL); err != nil { return fmt.Errorf("write server URL: %w", err) } + if options.Prometheus { + if _, err := fmt.Fprintf(options.Root.OutOrStdout(), "VictoriaMetrics: %s\n", metrics.URL); err != nil { + return fmt.Errorf("write VictoriaMetrics URL: %w", err) + } + } err = serve(ctx, httpOptions{listener: listener, handler: handler}) if cause := context.Cause(ctx); cause != nil && !errors.Is(cause, context.Canceled) { err = errors.Join(err, cause) diff --git a/internal/server/telemetry_openapi.go b/internal/server/telemetry_openapi.go index bac1849..09cd804 100644 --- a/internal/server/telemetry_openapi.go +++ b/internal/server/telemetry_openapi.go @@ -1,6 +1,10 @@ package server -import "github.com/flanksource/clicky/rpc" +import ( + "maps" + + "github.com/flanksource/clicky/rpc" +) func addReceiverPath(spec *rpc.OpenAPISpec) { if spec.Paths == nil { @@ -19,4 +23,21 @@ func addReceiverPath(spec *rpc.OpenAPISpec) { Description: "Continuously persists traces without arming. Non-loopback HTTP listeners require an API-scoped bearer token. Supports gzip. Responses use the request encoding; errors are google.rpc.Status.", RequestBody: &rpc.OpenAPIRequestBody{Required: true, Content: content}, Responses: responses, }} + profileContent := map[string]rpc.OpenAPIMediaType{ + "application/x-protobuf": {Schema: &rpc.OpenAPISchema{Type: "string", Format: "binary"}}, + "application/json": {Schema: &rpc.OpenAPISchema{Type: "object", Description: "OTLP ExportProfilesServiceRequest with resource profiles and dictionary references"}}, + } + profileResponses := maps.Clone(responses) + for code, response := range profileResponses { + response.Content = profileContent + profileResponses[code] = response + } + profileResponses["200"] = rpc.OpenAPIResponse{Description: "Atomically committed profile snapshots; partialSuccess reports rejected samples", Content: profileContent} + profileResponses["413"] = rpc.OpenAPIResponse{Description: "Profile request or sample limit exceeded", Content: profileContent} + spec.Paths["/v1/profiles"] = rpc.OpenAPIPath{"post": { + Summary: "Receive OTLP active-operation Profiles (serve --otlp)", Tags: []string{"OpenTelemetry"}, + Description: "Stores complete HTTP, SQL and JMS handler top snapshots in the receiver source. Capture attributes do not authorize writes into active runs. Supports gzip and requires the compiled Profiles development version.", + Parameters: []rpc.OpenAPIParameter{{Name: "OTLP-Profiles-Development-Version", In: "header", Required: true, Schema: &rpc.OpenAPISchema{Type: "string", Description: "Profiles development version; currently 1"}}}, + RequestBody: &rpc.OpenAPIRequestBody{Required: true, Content: profileContent}, Responses: profileResponses, + }} } diff --git a/internal/server/viewer.go b/internal/server/viewer.go new file mode 100644 index 0000000..f5e1e3e --- /dev/null +++ b/internal/server/viewer.go @@ -0,0 +1,128 @@ +package server + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "net" + "net/http" + "net/url" + "sync" + "time" + + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/pkg/traceserve/records" + "github.com/flanksource/telltale/webapp" +) + +type Viewer struct { + closeOnce sync.Once + closeErr error + URL string + listener net.Listener + server *http.Server + done chan error +} + +type ViewerOptions struct { + DataDir string + RunIDs []string +} + +// StartViewer exposes only the selected runs' records and the shared frontend. +func StartViewer(options ViewerOptions) (*Viewer, error) { + if len(options.RunIDs) == 0 { + return nil, fmt.Errorf("trace viewer requires at least one run ID") + } + allowed := map[string]bool{} + for _, id := range options.RunIDs { + if allowed[id] { + return nil, fmt.Errorf("run %q selected twice", id) + } + info, err := telltale.Get(telltale.GetOptions{DataDir: options.DataDir, RunID: id}) + if err != nil { + return nil, err + } + if info.Unavailable != "" { + return nil, fmt.Errorf("view run %s: %s", id, info.Unavailable) + } + allowed[id] = true + } + assets, err := webapp.Assets() + if err != nil { + return nil, err + } + if _, err := fs.ReadFile(assets, "index.html"); err != nil { + return nil, err + } + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, err + } + mux := http.NewServeMux() + feed := records.FeedHandler(options.DataDir) + mux.HandleFunc("GET /api/traces/{run}/records", func(w http.ResponseWriter, r *http.Request) { + if !allowed[r.PathValue("run")] { + http.NotFound(w, r) + return + } + feed.ServeHTTP(w, r) + }) + processes := records.ProcessesHandler(options.DataDir) + mux.HandleFunc("GET /api/traces/{run}/processes", func(w http.ResponseWriter, r *http.Request) { + if !allowed[r.PathValue("run")] { + http.NotFound(w, r) + return + } + processes.ServeHTTP(w, r) + }) + native := records.Handler(options.DataDir) + mux.Handle("GET /api/traces/{run}/api/v1/traces/{kind}/{operation}", viewerNative(options.DataDir, allowed)) + mux.HandleFunc("GET /api/traces/{run}/traces/{kind}/records", func(w http.ResponseWriter, r *http.Request) { + if !allowed[r.PathValue("run")] { + http.NotFound(w, r) + return + } + native.ServeHTTP(w, r) + }) + mux.HandleFunc("GET /api/traces", func(w http.ResponseWriter, r *http.Request) { + runs := []telltale.MountedRun{} + for _, id := range options.RunIDs { + info, err := telltale.Get(telltale.GetOptions{DataDir: options.DataDir, RunID: id}) + if err != nil { + http.Error(w, err.Error(), 500) + return + } + runs = append(runs, telltale.MountedRun{RunInfo: info, Mount: TracesMount + "/" + id}) + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(telltale.RunsList{Runs: runs}); err != nil { + panic(http.ErrAbortHandler) + } + }) + mux.HandleFunc("/api/", http.NotFound) + mux.Handle("/", assetHandler(assets)) + params := url.Values{"viewer": {"1"}, "run": {options.RunIDs[0]}} + viewer := &Viewer{URL: "http://" + listener.Addr().String() + "/?" + params.Encode(), listener: listener, server: &http.Server{Handler: mux, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 30 * time.Second, IdleTimeout: time.Minute}, done: make(chan error, 1)} + go func() { + err := viewer.server.Serve(listener) + if errors.Is(err, http.ErrServerClosed) { + err = nil + } + viewer.done <- err + }() + return viewer, nil +} + +func (v *Viewer) Close(ctx context.Context) error { + v.closeOnce.Do(func() { + v.closeErr = v.server.Shutdown(ctx) + if v.closeErr != nil { + v.closeErr = errors.Join(v.closeErr, v.server.Close()) + } + v.closeErr = errors.Join(v.closeErr, <-v.done) + }) + return v.closeErr +} diff --git a/internal/server/viewer_native.go b/internal/server/viewer_native.go new file mode 100644 index 0000000..82819f1 --- /dev/null +++ b/internal/server/viewer_native.go @@ -0,0 +1,37 @@ +package server + +import ( + "net/http" + "strings" + + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/pkg/traceserve" +) + +func viewerNative(dataDir string, allowed map[string]bool) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + id, operation := r.PathValue("run"), r.PathValue("operation") + if !allowed[id] || (operation != "seq" && operation != "export") { + http.NotFound(w, r) + return + } + runtime, closeStore, err := telltale.Reopen(dataDir, id) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer func() { + if closeStore() != nil { + panic(http.ErrAbortHandler) + } + }() + handler, err := traceserve.Handler(runtime) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + request := r.Clone(r.Context()) + request.URL.Path = strings.TrimPrefix(r.URL.Path, TracesMount+"/"+id) + handler.ServeHTTP(w, request) + }) +} diff --git a/internal/server/viewer_test.go b/internal/server/viewer_test.go new file mode 100644 index 0000000..f732681 --- /dev/null +++ b/internal/server/viewer_test.go @@ -0,0 +1,59 @@ +package server_test + +import ( + "context" + "encoding/json" + "net/http" + "net/url" + "os" + "path/filepath" + "time" + + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/internal/server" + "github.com/flanksource/telltale/pkg/traces" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("trace replay viewer", func() { + It("lists and serves only the selected runs", func() { + dataDir := GinkgoT().TempDir() + ids := []string{"first", "second", "unselected"} + for _, id := range ids { + dir := traces.StoreDir(dataDir, id) + Expect(os.MkdirAll(dir, 0o755)).To(Succeed()) + data, err := json.Marshal(telltale.Result{RunID: id, Dir: dir, StartedAt: time.Now(), EndedAt: time.Now()}) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(dir, telltale.ManifestFile), data, 0o600)).To(Succeed()) + } + viewer, err := server.StartViewer(server.ViewerOptions{DataDir: dataDir, RunIDs: ids[:2]}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { Expect(viewer.Close(context.Background())).To(Succeed()) }) + address, err := url.Parse(viewer.URL) + Expect(err).NotTo(HaveOccurred()) + base := address.Scheme + "://" + address.Host + response, err := http.Get(base + "/api/traces") + Expect(err).NotTo(HaveOccurred()) + var listed telltale.RunsList + Expect(json.NewDecoder(response.Body).Decode(&listed)).To(Succeed()) + Expect(response.Body.Close()).To(Succeed()) + Expect(listed.Runs).To(HaveExactElements(HaveField("RunID", ids[0]), HaveField("RunID", ids[1]))) + for _, path := range []string{"/api/traces/unselected/records", "/api/traces/unselected/processes", "/api/traces/unselected/api/v1/traces/process/export", "/api/traces/first/api/v1/traces/process/start", "/api/traces/start", "/api/v1/trace/run"} { + response, err := http.Get(base + path) + Expect(err).NotTo(HaveOccurred()) + Expect(response.StatusCode).To(Equal(http.StatusNotFound), path) + Expect(response.Body.Close()).To(Succeed()) + } + for _, id := range ids[:2] { + response, err := http.Get(base + "/api/traces/" + id + "/processes") + Expect(err).NotTo(HaveOccurred()) + Expect(response.StatusCode).To(Equal(http.StatusOK)) + Expect(response.Body.Close()).To(Succeed()) + response, err = http.Get(base + "/api/traces/" + id + "/records") + Expect(err).NotTo(HaveOccurred()) + Expect(response.StatusCode).To(Equal(http.StatusOK)) + Expect(response.Body.Close()).To(Succeed()) + } + }) +}) diff --git a/internal/shutdown/shutdown.go b/internal/shutdown/shutdown.go new file mode 100644 index 0000000..37c27c6 --- /dev/null +++ b/internal/shutdown/shutdown.go @@ -0,0 +1,120 @@ +package shutdown + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "sync" + "sync/atomic" + "syscall" + "time" +) + +const GracePeriod = 30 * time.Second + +type Options struct { + GracePeriod time.Duration + Signals <-chan os.Signal + Output io.Writer + ForceExit func(int) + Restore func(bool) +} + +type contextKey struct{} + +type Controller struct { + ctx context.Context + cancel context.CancelFunc + options Options + request chan struct{} + done chan struct{} + stopped chan struct{} + closeOnce sync.Once + requestOnce sync.Once + terminal atomic.Bool +} + +func New(parent context.Context, options Options) (*Controller, error) { + if options.GracePeriod <= 0 || options.Signals == nil || options.Output == nil || options.ForceExit == nil || options.Restore == nil { + return nil, errors.New("shutdown requires a positive grace period, signal channel, output, exit and terminal restore functions") + } + ctx, cancel := context.WithCancel(parent) + c := &Controller{cancel: cancel, options: options, request: make(chan struct{}, 1), done: make(chan struct{}), stopped: make(chan struct{})} + c.ctx = context.WithValue(ctx, contextKey{}, c) + go c.watch(parent) + return c, nil +} + +func (c *Controller) Context() context.Context { return c.ctx } + +func (c *Controller) Close() { + c.closeOnce.Do(func() { close(c.done); c.cancel() }) + <-c.stopped +} + +func (c *Controller) watch(parent context.Context) { + defer close(c.stopped) + select { + case <-c.done: + return + case _, open := <-c.options.Signals: + if !open { + c.force(1, "shutdown signal channel closed unexpectedly") + return + } + case <-c.request: + case <-parent.Done(): + } + c.cancel() + timer := time.NewTimer(c.options.GracePeriod) + defer timer.Stop() + _, _ = fmt.Fprintln(c.options.Output, "Shutting down; press Ctrl+C again to force exit.") + select { + case <-c.done: + case sig, open := <-c.options.Signals: + if !open { + c.force(1, "shutdown signal channel closed unexpectedly") + return + } + code := 1 + if value, ok := sig.(syscall.Signal); ok { + code = 128 + int(value) + } + c.force(code, "Forced exit.") + case <-timer.C: + c.force(1, fmt.Sprintf("Shutdown exceeded %s; exiting with incomplete cleanup.", c.options.GracePeriod)) + } +} + +func (c *Controller) force(code int, message string) { + select { + case <-c.done: + return + default: + } + c.options.Restore(c.terminal.Load()) + _, _ = fmt.Fprintln(c.options.Output, message) + c.options.ForceExit(code) +} + +// Request starts the same graceful shutdown used by an OS signal. +func Request(ctx context.Context) error { + c, ok := ctx.Value(contextKey{}).(*Controller) + if !ok { + return errors.New("command shutdown controller is missing") + } + c.requestOnce.Do(func() { c.request <- struct{}{} }) + return nil +} + +// Terminal tracks an alternate-screen view until its Run returns. +func Terminal(ctx context.Context) (func(), error) { + c, ok := ctx.Value(contextKey{}).(*Controller) + if !ok { + return nil, errors.New("command shutdown controller is missing") + } + c.terminal.Store(true) + return func() { c.terminal.Store(false) }, nil +} diff --git a/internal/shutdown/shutdown_test.go b/internal/shutdown/shutdown_test.go new file mode 100644 index 0000000..a10f645 --- /dev/null +++ b/internal/shutdown/shutdown_test.go @@ -0,0 +1,71 @@ +package shutdown_test + +import ( + "context" + "io" + "os" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/flanksource/telltale/internal/shutdown" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestShutdown(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Command shutdown") } + +var _ = Describe("Command lifecycle", func() { + It("cancels on the first signal and forces exit on the second after restoring the terminal", func() { + signals, exits := make(chan os.Signal, 2), make(chan int, 1) + var restored atomic.Bool + controller, err := shutdown.New(context.Background(), shutdown.Options{ + GracePeriod: time.Hour, Signals: signals, Output: io.Discard, + Restore: func(bool) { restored.Store(true) }, + ForceExit: func(code int) { + if !restored.Load() { + code = -1 + } + exits <- code + }, + }) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(controller.Close) + signals <- os.Interrupt + Eventually(controller.Context().Done()).Should(BeClosed()) + Expect(exits).NotTo(Receive()) + signals <- syscall.SIGTERM + Eventually(exits).Should(Receive(Equal(143))) + }) + + It("starts the same deadline for a terminal quit request and exits with an error on expiry", func() { + exits := make(chan int, 1) + controller, err := shutdown.New(context.Background(), shutdown.Options{ + GracePeriod: 40 * time.Millisecond, Signals: make(chan os.Signal), Output: io.Discard, + Restore: func(bool) {}, ForceExit: func(code int) { exits <- code }, + }) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(controller.Close) + Expect(shutdown.Request(controller.Context())).To(Succeed()) + Eventually(controller.Context().Done()).Should(BeClosed()) + Eventually(exits).Should(Receive(Equal(1))) + }) + + It("disarms the watchdog after graceful completion", func() { + signals, exits := make(chan os.Signal, 2), make(chan int, 1) + controller, err := shutdown.New(context.Background(), shutdown.Options{ + GracePeriod: 40 * time.Millisecond, Signals: signals, Output: io.Discard, + Restore: func(bool) {}, ForceExit: func(code int) { exits <- code }, + }) + Expect(err).NotTo(HaveOccurred()) + signals <- os.Interrupt + Eventually(controller.Context().Done()).Should(BeClosed()) + controller.Close() + Consistently(exits, 100*time.Millisecond).ShouldNot(Receive()) + }) + + It("requires an installed controller for a terminal quit request", func() { + Expect(shutdown.Request(context.Background())).To(MatchError(ContainSubstring("shutdown controller"))) + }) +}) diff --git a/internal/terminalui/render.go b/internal/terminalui/render.go new file mode 100644 index 0000000..c02453d --- /dev/null +++ b/internal/terminalui/render.go @@ -0,0 +1,42 @@ +package terminalui + +import ( + "charm.land/lipgloss/v2" + "github.com/charmbracelet/x/ansi" + "strings" +) + +type Share struct { + Percent float64 + Style lipgloss.Style +} + +func Bar(width int, shares []Share) string { + width = max(0, width) + var b strings.Builder + used := 0 + for _, part := range shares { + cells := min(width-used, int(part.Percent/100*float64(width)+0.5)) + if cells > 0 { + b.WriteString(part.Style.Render(strings.Repeat("█", cells))) + used += cells + } + } + return "[" + b.String() + Muted.Render(strings.Repeat("░", width-used)) + "]" +} + +func Fit(text string, width int) string { + width = max(0, width) + text = ansi.Truncate(text, width, "…") + return text + strings.Repeat(" ", max(0, width-ansi.StringWidth(text))) +} + +func Clip(text string, width int) string { return ansi.Truncate(text, max(0, width), "") } + +func PadLines(text string, height int) []string { + lines := strings.Split(text, "\n") + for len(lines) < height { + lines = append(lines, "") + } + return lines[:max(0, height)] +} diff --git a/internal/terminalui/render_test.go b/internal/terminalui/render_test.go new file mode 100644 index 0000000..88fd471 --- /dev/null +++ b/internal/terminalui/render_test.go @@ -0,0 +1,27 @@ +package terminalui + +import ( + "github.com/charmbracelet/x/ansi" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "strings" + "testing" +) + +func TestTerminalUI(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Terminal rendering") } + +var _ = Describe("shared rendering", func() { + It("fits Unicode and ANSI without splitting cells or leaking beyond the viewport", func() { + for _, width := range []int{0, 1, 5, 20} { + styled := "\x1b[31m数据库 worker\x1b[0m" + Expect(ansi.StringWidth(Fit(styled, width))).To(Equal(width)) + Expect(ansi.StringWidth(Clip(styled, width))).To(BeNumerically("<=", width)) + } + }) + It("bounds stacked meters and fills the remaining cells", func() { + Expect(ansi.Strip(Bar(10, []Share{{Percent: 25, Style: CPU}, {Percent: 200, Style: Memory}}))).To(Equal("[" + strings.Repeat("█", 10) + "]")) + Expect(ansi.Strip(Bar(4, nil))).To(Equal("[░░░░]")) + Expect(PadLines("one\ntwo", 1)).To(Equal([]string{"one"})) + Expect(PadLines("one", 3)).To(Equal([]string{"one", "", ""})) + }) +}) diff --git a/internal/terminalui/styles.go b/internal/terminalui/styles.go new file mode 100644 index 0000000..1a5216e --- /dev/null +++ b/internal/terminalui/styles.go @@ -0,0 +1,14 @@ +// Package terminalui holds the common visual language of Telltale's terminal monitors. +package terminalui + +import "charm.land/lipgloss/v2" + +var Title = lipgloss.NewStyle().Bold(true) +var Muted = lipgloss.NewStyle().Foreground(lipgloss.Color("244")) +var Selected = lipgloss.NewStyle().Reverse(true) +var Heading = lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("244")) +var Status = lipgloss.NewStyle().Foreground(lipgloss.Color("208")) +var CPU = lipgloss.NewStyle().Foreground(lipgloss.Color("39")) +var Kernel = lipgloss.NewStyle().Foreground(lipgloss.Color("203")) +var Memory = lipgloss.NewStyle().Foreground(lipgloss.Color("78")) +var Swap = lipgloss.NewStyle().Foreground(lipgloss.Color("214")) diff --git a/internal/topui/actions.go b/internal/topui/actions.go index b2999a7..221c2c6 100644 --- a/internal/topui/actions.go +++ b/internal/topui/actions.go @@ -23,6 +23,19 @@ func (m Model) cpuHog(cpu *float64) bool { return cpu != nil && m.frame != nil && m.frame.Host != nil && m.frame.Host.Cores > 0 && *cpu > 25*float64(m.frame.Host.Cores) } +func (m Model) hogLabel(usage groups.Usage) string { + if m.frame == nil { + return "" + } + if label := usage.Hogs.Label(m.frame.SampledAt); label != "" { + return label + } + if m.cpuHog(usage.CPUPercent) { + return "CPU hog" + } + return "" +} + func (m *Model) cycleProcess(key string) { processes, _ := m.selectedProcesses() if len(processes) == 0 { diff --git a/internal/topui/grouping_test.go b/internal/topui/grouping_test.go index 2929d12..21ab431 100644 --- a/internal/topui/grouping_test.go +++ b/internal/topui/grouping_test.go @@ -41,6 +41,9 @@ var _ = Describe("Model grouping", func() { m = press(m, "left") Expect(shown(m)).NotTo(ContainElement("agent-a")) m = press(m, "right", "b", "b", "b") + Expect(m.groupBy).To(Equal(groups.GroupSlots)) + m = press(m, "b") + Expect(m.groupBy).To(Equal(groups.GroupTree)) Expect(m.frame.Groups).To(HaveLen(len(fixture.Frame.Groups))) Expect(m.frame.Groups).NotTo(ContainElement(HaveField("Kind", groups.KindBucket))) }) diff --git a/internal/topui/memory_test.go b/internal/topui/memory_test.go new file mode 100644 index 0000000..1955688 --- /dev/null +++ b/internal/topui/memory_test.go @@ -0,0 +1,48 @@ +package topui + +import ( + "context" + "time" + + "github.com/flanksource/telltale/pkg/tracing/groups" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("memory columns and pins", func() { + It("shows compressed memory separately from unavailable swap and pins a quiet recent hog", func() { + frame := testFrame + hogs := groups.Hogs{"memory": frame.SampledAt.Add(groups.HogRetention)} + frame.Groups = []groups.GroupSample{ + sample("busy", "", "busy", groups.KindTool, 100, 100<<20), + {Key: "hog", Name: "worker", Kind: groups.KindTool, Total: groups.Usage{Processes: 1, CompressedBytes: new(uint64(512 << 20)), Hogs: hogs}}, + } + m := New(context.Background(), Options{Source: &fakeSource{}, Every: time.Second, Order: groups.Order{By: groups.SortCPU}}) + m.width, m.height = 200, 40 + next, _ := m.Update(frameMsg{frame: frame, ok: true}) + m = next.(Model) + Expect(m.rows()[0].group.Key).To(Equal("hog")) + Expect(m.render()).To(SatisfyAll(ContainSubstring("CMPRS"), ContainSubstring("SWAP"), ContainSubstring("512.0M"), ContainSubstring("memory hog"))) + m = press(m, "S") + Expect(m.rows()[0].group.Key).To(Equal("hog")) + frame.SampledAt = frame.SampledAt.Add(120 * time.Second) + next, _ = m.Update(frameMsg{frame: frame, ok: true}) + m = press(next.(Model), "S") + Expect(m.rows()[0].group.Key).To(Equal("busy")) + }) + + It("orders the selected group's recent hog before a busier process", func() { + m := New(context.Background(), Options{Every: time.Second, Order: groups.Order{By: groups.SortCPU}}) + m.width, m.height = 200, 40 + m.source = &fakeSource{keeps: true, processes: map[string][]groups.ProcessSample{ + "10": { + {Key: "busy@1", Usage: groups.Usage{CPUPercent: percent(100)}}, + {Key: "hog@1", Usage: groups.Usage{Hogs: groups.Hogs{"swap": testFrame.SampledAt.Add(groups.HogRetention)}}}, + }, + }} + next, _ := m.Update(frameMsg{frame: testFrame, ok: true}) + processes, kept := next.(Model).selectedProcesses() + Expect(kept).To(BeTrue()) + Expect(processes[0].Key).To(Equal("hog@1")) + }) +}) diff --git a/internal/topui/model.go b/internal/topui/model.go index f5456af..0506dc0 100644 --- a/internal/topui/model.go +++ b/internal/topui/model.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "github.com/flanksource/clicky/tui/table" "slices" "strings" "time" @@ -41,6 +42,9 @@ type Options struct { // Model is the view's state: the latest frame, the selected group and how // the tree is folded, filtered and sorted. type Model struct { + inspection *table.Model + paused bool + help bool ctx context.Context options Options source Source @@ -55,6 +59,8 @@ type Model struct { selected string offset int folded map[string]bool + // expanded are the groups whose member processes the tree lists. + expanded map[string]bool query string typing bool detail bool @@ -74,7 +80,7 @@ func New(ctx context.Context, options Options) Model { return Model{ ctx: ctx, options: options, source: options.Source, toggles: options.Toggles, order: options.Order, groupBy: options.GroupBy, - folded: map[string]bool{}, + folded: map[string]bool{}, expanded: map[string]bool{}, } } @@ -119,7 +125,7 @@ func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { case frameMsg: return m.received(msg) case tickMsg: - if msg.gen == m.gen { + if msg.gen == m.gen && !m.paused { return m, m.read() } case doneMsg: @@ -138,6 +144,9 @@ func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { // received takes a frame, or a failed read: the first failure is shown, // the second in a row ends the view. func (m Model) received(msg frameMsg) (tea.Model, tea.Cmd) { + if m.paused { + return m, nil + } if msg.gen != m.gen { return m, nil } @@ -173,6 +182,22 @@ func (m Model) pressed(msg tea.KeyPressMsg) (tea.Model, tea.Cmd) { if msg.Keystroke() == "ctrl+c" { return m, tea.Quit } + if m.inspection != nil { + switch msg.Keystroke() { + case "q": + return m, tea.Quit + case "esc", "d": + m.inspection = nil + return m, nil + case "space", "P": + m.inspection = nil + m.paused = false + return m, m.read() + default: + m.inspection.Key(msg.Keystroke()) + return m, nil + } + } if m.typing { return m.typed(msg), nil } @@ -203,7 +228,9 @@ func (m Model) pressed(msg tea.KeyPressMsg) (tea.Model, tea.Cmd) { m.fold(rows, index, !rows[index].folded) } case tea.KeyEnter: - m.detail = !m.detail + if index >= 0 { + m.toggleExpanded(rows[index]) + } case tea.KeyEscape: m.detail, m.query = false, "" default: @@ -216,8 +243,22 @@ func (m Model) pressed(msg tea.KeyPressMsg) (tea.Model, tea.Cmd) { // command runs a key that names one: sort, filter, toggles, quit. func (m Model) command(key string) (tea.Model, tea.Cmd) { switch key { + case "d": + view := m.processTable() + view.Detail = true + m.inspection = &view + m.paused = true + case "P": + m.paused = !m.paused + if !m.paused { + return m, m.read() + } + case "?": + m.help = !m.help case "q": return m, tea.Quit + case "f": + m.detail = !m.detail case "[", "]": m.cycleProcess(key) case "x", "r": @@ -260,7 +301,7 @@ func (m Model) typed(msg tea.KeyPressMsg) Model { m.query += msg.Text } if rows := m.rows(); m.selectedIndex(rows) < 0 && len(rows) > 0 { - m.selected = rows[0].group.Key + m.selected = rows[0].key } m.scroll() return m @@ -275,14 +316,14 @@ func (m *Model) resort() { m.err, m.status = err, err.Error() return } - sorted, err := projected.Sorted(m.order) + sorted, err := projected.WithGavelTasks().Sorted(m.order) if err != nil { m.err, m.status = err, err.Error() return } m.frame = &sorted if rows := m.rows(); m.selectedIndex(rows) < 0 && len(rows) > 0 { - m.selected = rows[0].group.Key + m.selected = rows[0].key } } @@ -308,14 +349,23 @@ func (m Model) reopen(toggles Toggles) (tea.Model, tea.Cmd) { return m, m.read() } -// row is a group as the tree shows it. +// row is a group, or one of an expanded group's processes, as the tree +// shows it. key and parentKey are the tree's: a group's own, or +// proc: under the row its process nests in. type row struct { - group groups.GroupSample - depth int - parent bool - folded bool + group groups.GroupSample + // process is set on a process row; group is then the group it belongs to. + process *groups.ProcessSample + // detached marks a process whose parent is outside its group. + detached bool + key, parentKey string + depth int + parent bool + folded bool } +const processPrefix = "proc:" + func (m Model) matchingGroups() []groups.GroupSample { if m.frame == nil { return nil @@ -360,6 +410,7 @@ func (m Model) rows() []row { for _, group := range displayed { parents[group.ParentKey] = true } + under := m.processesUnder(displayed) var rows []row depth := map[string]int{} hidden := map[string]bool{} @@ -369,25 +420,138 @@ func (m Model) rows() []row { level++ } depth[group.Key] = level - folded := m.isFolded(group, parents[group.Key]) if hidden[group.ParentKey] { hidden[group.Key] = true continue } + processes := under[group.Key] + parent := parents[group.Key] || len(processes) > 0 + folded := m.isFolded(group.Key, group, parent) hidden[group.Key] = folded - rows = append(rows, row{group: group, depth: level, parent: parents[group.Key], folded: folded}) + rows = append(rows, row{group: group, key: group.Key, parentKey: group.ParentKey, depth: level, parent: parent, folded: folded}) + if !folded { + for _, process := range m.foldProcesses(processes) { + process.depth += level + 1 + rows = append(rows, process) + } + } } return rows } -func (m Model) isFolded(group groups.GroupSample, parent bool) bool { +// foldProcesses drops the process rows under a folded process row. +func (m Model) foldProcesses(processes []row) []row { + var shown []row + hidden := map[string]bool{} + for _, r := range processes { + if hidden[r.parentKey] { + hidden[r.key] = true + continue + } + r.folded = m.isFolded(r.key, r.group, r.parent) + hidden[r.key] = r.folded + shown = append(shown, r) + } + return shown +} + +func (m Model) isFolded(key string, group groups.GroupSample, parent bool) bool { if !parent || m.query != "" { return false } - if folded, ok := m.folded[group.Key]; ok { + if folded, ok := m.folded[key]; ok { return folded } - return group.Kind == groups.KindOther + return group.Kind == groups.KindOther && !group.Total.Hogs.Pinned(m.frame.SampledAt) +} + +// toggleExpanded lists or hides the member processes of r's group, or of +// the group a process row belongs to. +func (m *Model) toggleExpanded(r row) { + key := r.group.Key + if r.group.Task != nil || r.group.Kind == groups.KindBucket { + m.status = "enter lists a group's processes; f shows them all below" + return + } + if _, kept := m.source.Processes(groups.ProcessFilter{GroupKey: key}); !kept { + m.status = "A recording keeps each group's use, not its processes." + return + } + m.expanded[key] = !m.expanded[key] + if !m.expanded[key] && r.process != nil { + m.selected = key + } +} + +// processesUnder are the expanded groups' member processes as rows, by the +// key of the row each subtree nests under, their depths counted from it. +// A group's anchor is the group's own row; a member a gavel task runs, and +// the members under it, nest under that task's row. +func (m Model) processesUnder(displayed []groups.GroupSample) map[string][]row { + under := map[string][]row{} + for _, group := range displayed { + if !m.expanded[group.Key] || group.Task != nil || group.Kind == groups.KindBucket { + continue + } + members, _ := m.source.Processes(groups.ProcessFilter{GroupKey: group.Key}) + claims := map[int]string{} + for _, task := range displayed { + if task.Task != nil && task.Task.Item != nil && task.Task.Item.PID > 0 && strings.HasPrefix(task.Key, group.Key+"/") { + claims[task.Task.Item.PID] = task.Key + } + } + for key, rows := range processRows(group, members, claims) { + under[key] = append(under[key], rows...) + } + } + return under +} + +// processRows nests members by parent pid: the anchor's children under the +// group, a claimed process (and those under it) under its task's row, and a +// process whose parent is outside the group directly under it, detached. +// Siblings come busiest first. +func processRows(group groups.GroupSample, members []groups.ProcessSample, claims map[int]string) map[string][]row { + byPID := map[int]groups.ProcessSample{} + for _, p := range members { + if p.PID != group.PID { + byPID[p.PID] = p + } + } + children := map[int][]groups.ProcessSample{} + var roots []groups.ProcessSample + for _, p := range byPID { + if _, ok := byPID[p.PPID]; ok && claims[p.PID] == "" { + children[p.PPID] = append(children[p.PPID], p) + } else { + roots = append(roots, p) + } + } + busiest := func(processes []groups.ProcessSample) { + slices.SortFunc(processes, func(a, b groups.ProcessSample) int { + return cmp.Or(cmp.Compare(cpuOf(b.Usage), cpuOf(a.Usage)), cmp.Compare(a.PID, b.PID)) + }) + } + under := map[string][]row{} + var walk func(owner string, p groups.ProcessSample, parentKey string, depth int, detached bool) + walk = func(owner string, p groups.ProcessSample, parentKey string, depth int, detached bool) { + process := p + key := processPrefix + p.Key + under[owner] = append(under[owner], row{group: group, process: &process, detached: detached, key: key, parentKey: parentKey, depth: depth, parent: len(children[p.PID]) > 0}) + busiest(children[p.PID]) + for _, child := range children[p.PID] { + walk(owner, child, key, depth+1, false) + } + } + busiest(roots) + for _, p := range roots { + if task := claims[p.PID]; task != "" { + walk(task, p, task, 0, false) + continue + } + walk(group.Key, p, group.Key, 0, p.PPID != group.PID) + } + return under } func matches(group groups.GroupSample, query string) bool { @@ -400,14 +564,20 @@ func matches(group groups.GroupSample, query string) bool { } func (m Model) selectedIndex(rows []row) int { - return slices.IndexFunc(rows, func(r row) bool { return r.group.Key == m.selected }) + return slices.IndexFunc(rows, func(r row) bool { return r.key == m.selected }) } +// selectAt selects the row at index; a process row also chooses its +// process for x and r. func (m *Model) selectAt(rows []row, index int) { if len(rows) == 0 { return } - m.selected = rows[max(0, min(index, len(rows)-1))].group.Key + r := rows[max(0, min(index, len(rows)-1))] + m.selected = r.key + if r.process != nil { + m.processKey = r.process.Key + } } // fold folds or unfolds the selected group; folding a group that holds @@ -418,11 +588,11 @@ func (m *Model) fold(rows []row, index int, fold bool) { } current := rows[index] if current.parent && current.folded != fold { - m.folded[current.group.Key] = fold + m.folded[current.key] = fold return } - if fold && current.group.ParentKey != "" && slices.ContainsFunc(rows, func(r row) bool { return r.group.Key == current.group.ParentKey }) { - m.selected = current.group.ParentKey + if fold && current.parentKey != "" && slices.ContainsFunc(rows, func(r row) bool { return r.key == current.parentKey }) { + m.selected = current.parentKey } } @@ -446,6 +616,13 @@ func (m Model) selectedProcesses() ([]groups.ProcessSample, bool) { if m.frame == nil || m.selected == "" { return nil, true } + if strings.HasPrefix(m.selected, processPrefix) { + rows := m.rows() + if index := m.selectedIndex(rows); index >= 0 { + return []groups.ProcessSample{*rows[index].process}, true + } + return nil, true + } under := map[string]bool{m.selected: true} var processes []groups.ProcessSample for _, group := range m.matchingGroups() { @@ -462,7 +639,15 @@ func (m Model) selectedProcesses() ([]groups.ProcessSample, bool) { } processes = append(processes, read...) } - slices.SortStableFunc(processes, func(a, b groups.ProcessSample) int { return cmp.Compare(cpuOf(b.Usage), cpuOf(a.Usage)) }) + slices.SortStableFunc(processes, func(a, b groups.ProcessSample) int { + if a.Hogs.Pinned(m.frame.SampledAt) != b.Hogs.Pinned(m.frame.SampledAt) { + if a.Hogs.Pinned(m.frame.SampledAt) { + return -1 + } + return 1 + } + return cmp.Compare(cpuOf(b.Usage), cpuOf(a.Usage)) + }) return processes, true } diff --git a/internal/topui/model_test.go b/internal/topui/model_test.go index 9acb721..69e6fbb 100644 --- a/internal/topui/model_test.go +++ b/internal/topui/model_test.go @@ -3,11 +3,14 @@ package topui import ( "context" "errors" + "fmt" + "slices" "strings" "time" tea "charm.land/bubbletea/v2" "charm.land/lipgloss/v2" + "github.com/flanksource/gavel/sdk/status" "github.com/flanksource/telltale/pkg/tracing/groups" "github.com/flanksource/telltale/pkg/tracing/process" . "github.com/onsi/ginkgo/v2" @@ -123,8 +126,8 @@ var _ = Describe("Model", func() { newModel := func(width, height int) Model { GinkgoHelper() source = &fakeSource{keeps: true, processes: map[string][]groups.ProcessSample{ - "11": {{PID: 1101, Name: "gavel", Command: "gavel test ./...", Usage: groups.Usage{Processes: 1, CPUPercent: percent(30)}}}, - "12": {{PID: 1201, Name: "node", Command: "node vitest", Usage: groups.Usage{Processes: 1, CPUPercent: percent(5)}}}, + "11": {{Key: "1101@1", PID: 1101, Name: "gavel", Command: "gavel test ./...", Usage: groups.Usage{Processes: 1, CPUPercent: percent(30)}}}, + "12": {{Key: "1201@1", PID: 1201, Name: "node", Command: "node vitest", Usage: groups.Usage{Processes: 1, CPUPercent: percent(5)}}}, }} reopened = nil m := New(context.Background(), Options{ @@ -184,6 +187,19 @@ var _ = Describe("Model", func() { Expect(m.order).To(Equal(groups.Order{By: groups.SortMemory, Reverse: true})) Expect(shown(m)[0]).To(Equal("other:system")) }) + It("freezes inspection across arriving frames until live updates are explicitly resumed", func() { + m := press(newModel(80, 24), "d") + Expect(m.paused).To(BeTrue()) + Expect(m.inspection).NotTo(BeNil()) + next, _ := m.Update(frameMsg{frame: groups.Frame{Tick: 99}, ok: true}) + m = next.(Model) + Expect(m.frame.Tick).To(Equal(int64(7))) + m = press(m, "esc") + Expect(m.inspection).To(BeNil()) + Expect(m.paused).To(BeTrue()) + m = press(m, "P") + Expect(m.paused).To(BeFalse()) + }) It("filters to the groups that match and their ancestors, unfolding them, and clears with esc", func() { m := press(newModel(120, 30), "/", "l", "o", "g", "enter") @@ -195,14 +211,110 @@ var _ = Describe("Model", func() { Expect(shown(m)).To(HaveLen(5)) }) - It("lists the processes of the selected group and those under it, busiest first", func() { - m := press(newModel(160, 40), "enter") + It("lists the processes of the selected group and those under it, busiest first, in the pane f opens", func() { + m := press(newModel(160, 40), "f") processes, kept := m.selectedProcesses() Expect(kept).To(BeTrue()) Expect(processes).To(HaveLen(2)) Expect(processes[0].PID).To(Equal(1101)) - Expect(m.render()).To(ContainSubstring("gavel test ./...")) + screen := m.render() + Expect(screen).To(ContainSubstring("gavel test ./...")) + Expect(screen).To(ContainSubstring("── claude · 2 processes ─")) + Expect(screen).NotTo(ContainSubstring("─…"), "the pane's rule fits the width") + }) + + It("lists a group's member processes in the tree on enter, nested by parent, a detached one under the group", func() { + m := newModel(200, 40) + frame := *m.raw + frame.Groups = append([]groups.GroupSample(nil), frame.Groups...) + frame.Groups[2].PID = 1200 + source.processes["12"] = []groups.ProcessSample{ + {Key: "1200@1", PID: 1200, PPID: 1100, Command: "/usr/local/bin/vitest run", Usage: groups.Usage{Processes: 1}}, + {Key: "1201@1", PID: 1201, PPID: 1200, Command: "/opt/node/bin/node worker.js", Usage: groups.Usage{Processes: 1, CPUPercent: percent(4)}}, + {Key: "1202@1", PID: 1202, PPID: 1201, Command: "/opt/node/bin/node esbuild", Usage: groups.Usage{Processes: 1}}, + {Key: "1203@1", PID: 1203, PPID: 1, Command: "/Applications/Acme.app/Contents/MacOS/Acme Helper (Renderer) --type=renderer", Usage: groups.Usage{Processes: 1}}, + } + next, _ := m.Update(frameMsg{frame: frame, ok: true}) + m = press(next.(Model), "down", "down", "enter") + Expect(m.selected).To(Equal("12")) + + var tree []string + for _, r := range m.rows() { + if r.process != nil { + tree = append(tree, fmt.Sprintf("%d:%d:%t", r.process.PID, r.depth, r.detached)) + } + } + Expect(tree).To(Equal([]string{"1201:2:false", "1202:3:false", "1203:2:true"}), "the anchor is the group's own row") + Expect(m.render()).To(SatisfyAll( + ContainSubstring("1201 node worker.js"), + ContainSubstring("↰ 1203 Acme Helper (Renderer) --type=renderer"), + )) + Expect(m.detail).To(BeFalse(), "enter lists processes in the tree, not the pane") + + m = press(m, "enter") + Expect(slices.ContainsFunc(m.rows(), func(r row) bool { return r.process != nil })).To(BeFalse()) + }) + + It("chooses a process row's process for x", func() { + m := press(newModel(200, 40), "down", "enter", "down") + Expect(m.rows()[m.selectedIndex(m.rows())].process).NotTo(BeNil()) + Expect(m.processKey).To(Equal("1101@1"), "selecting the row chooses its process") + processes, _ := m.selectedProcesses() + Expect(processes).To(ConsistOf(HaveField("PID", 1101))) + }) + + It("nests a gavel's task groups and running tasks under it, and the process a task runs under its task", func() { + frame := testFrame + frame.Groups = append([]groups.GroupSample(nil), testFrame.Groups...) + frame.Groups[1].SkipAncestry, frame.Groups[1].PID = true, 1100 + frame.Groups[1].Gavel = &status.Status{ + Groups: []status.Group{{Name: "Running linters", Kind: status.GroupLint, Summary: status.Summary{Total: 7, Passed: 2, Failed: 1, Running: 4}}}, + Running: []status.Item{{Name: "golangci-lint run ./...", Group: "Running linters", PID: 1102, Started: frame.SampledAt.Add(-42 * time.Second)}}, + Slots: []status.Lease{{Pool: "global", Limit: 4, Index: 1, State: status.LeaseHeld, Since: frame.SampledAt.Add(-time.Minute), Command: "golangci-lint run ./..."}}, + } + frame.Slots = []groups.SlotPool{{Name: "global", Limit: 4, Held: 1}} + m := newModel(200, 40) + source.processes["11"] = append(source.processes["11"], + groups.ProcessSample{Key: "1102@1", PID: 1102, PPID: 1101, Name: "golangci-lint", Command: "/opt/bin/golangci-lint run ./...", Usage: groups.Usage{Processes: 1, CPUPercent: percent(80)}}) + next, _ := m.Update(frameMsg{frame: frame, ok: true}) + m = next.(Model) + + Expect(strings.Split(m.render(), "\n")[headerLines-1]).To(ContainSubstring("SLT gavel global [██░░░░░░] 1/4")) + depths := map[string]int{} + for _, r := range m.rows() { + depths[r.key] = r.depth + } + Expect(depths).To(HaveKeyWithValue("11", 1)) + Expect(depths).To(HaveKeyWithValue("11/gavel/lint", 2)) + Expect(depths).To(HaveKeyWithValue("11/gavel/lint/0", 3)) + // 2 passed → 2 cells █, 1 failed → 1 ▓, 4 running → the other 5 ▒. + Expect(m.render()).To(SatisfyAll( + ContainSubstring("gavel ●global"), + ContainSubstring(" ▾ lint [██▓▒▒▒▒▒] 3/7 ✗1"), + ContainSubstring(" ▶ golangci-lint run ./... 42s"), + )) + + m = press(m, "down", "enter") + Expect(m.selected).To(Equal("11")) + Expect(depths).NotTo(HaveKey("proc:1102@1")) + for _, r := range m.rows() { + depths[r.key] = r.depth + } + Expect(depths).To(HaveKeyWithValue("proc:1102@1", 4), "the task's process nests under its task row") + Expect(depths).To(HaveKeyWithValue("proc:1101@1", 2)) + Expect(m.render()).NotTo(ContainSubstring("3/7 done"), "the pane no longer repeats the progress") + + m.selected = "11/gavel/lint" + m = press(m, "left") + Expect(slices.ContainsFunc(m.rows(), func(r row) bool { return r.key == "11/gavel/lint/0" })).To(BeFalse(), "← folds the lint row's tasks") + }) + + It("refuses to list processes in the tree from a recording", func() { + m := newModel(160, 40) + source.keeps = false + m = press(m, "enter") + Expect(m.status).To(ContainSubstring("A recording keeps each group's use")) }) It("reads from a new source with a flipped toggle, closing the old one", func() { diff --git a/internal/topui/source.go b/internal/topui/source.go index 68f2af6..8fbf818 100644 --- a/internal/topui/source.go +++ b/internal/topui/source.go @@ -7,6 +7,7 @@ import ( "context" "github.com/flanksource/telltale/pkg/tracing/groups" + "github.com/flanksource/telltale/pkg/tracing/processcapture" ) // Source reads the frames a Model shows. @@ -38,13 +39,15 @@ func (l Live) Close() { l.Top.Close() } // Recorded follows a recording's committed frames, which keep no processes. type Recorded struct { - Follower *groups.Follower + Follower *processcapture.Follower } func (r Recorded) Next(ctx context.Context) (groups.Frame, bool, error) { return r.Follower.Frame(ctx) } -func (Recorded) Processes(groups.ProcessFilter) ([]groups.ProcessSample, bool) { return nil, false } +func (r Recorded) Processes(filter groups.ProcessFilter) ([]groups.ProcessSample, bool) { + return r.Follower.Processes(filter), true +} func (Recorded) Close() {} diff --git a/internal/topui/table.go b/internal/topui/table.go new file mode 100644 index 0000000..03dae08 --- /dev/null +++ b/internal/topui/table.go @@ -0,0 +1,74 @@ +package topui + +import ( + "fmt" + "strings" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" + "github.com/flanksource/clicky/tui/table" + "github.com/flanksource/telltale/pkg/tracing/groups" +) + +type processTableRow struct { + command string + name string + usage groups.Usage + origin, context string +} + +func (r processTableRow) RowDetail() api.Textable { + return clicky.Map(map[string]any{"Name": r.name, "Command": r.command, "Usage": r.usage, "Origin": r.origin, "Context": r.context}) +} + +func (processTableRow) Columns() []api.ColumnDef { + return []api.ColumnDef{api.Column("name").Label("GROUP / PROCESS").Build(), api.Column("cpu").Label("CPU%").Build(), api.Column("rss").Label("RSS").Build(), api.Column("processes").Label("PROCS").Build(), api.Column("compressed").Label("CMPRS").Build(), api.Column("swap").Label("SWAP").Build(), api.Column("read").Label("DISK R/s").Build(), api.Column("write").Label("DISK W/s").Build(), api.Column("netRx").Label("NET RX/s").Build(), api.Column("netTx").Label("NET TX/s").Build(), api.Column("sockets").Label("SOCK").Build(), api.Column("connections").Label("CONN").Build(), api.Column("origin").Label("ORIGIN").Build(), api.Column("context").Label("CONTEXT").Build()} +} +func (r processTableRow) Row() map[string]any { + return map[string]any{"name": r.name, "cpu": api.TableCell{Value: groups.Percent(r.usage.CPUPercent), FilterValue: r.usage.CPUPercent}, "rss": api.TableCell{Value: groups.Size(float64(r.usage.RSSBytes)), FilterValue: r.usage.RSSBytes}, "processes": r.usage.Processes, "compressed": groups.MemorySize(r.usage.CompressedBytes), "swap": groups.MemorySize(r.usage.SwapBytes), "read": groups.Rate(r.usage.DiskRead), "write": groups.Rate(r.usage.DiskWrite), "netRx": groups.Rate(r.usage.NetRx), "netTx": groups.Rate(r.usage.NetTx), "sockets": r.usage.Sockets, "connections": r.usage.Connections, "origin": r.origin, "context": r.context} +} + +func (m Model) processTable() table.Model { + rows, ids := []processTableRow{}, []string{} + for _, r := range m.rows() { + usage, label := r.group.Total, groups.Label(r.group) + if r.process != nil { + usage, label = r.process.Usage, fmt.Sprintf("%d %s", r.process.PID, groups.ShortCommand(r.process.Command)) + } + if r.group.Task != nil { + label = groups.TaskText(r.group, m.frame.SampledAt) + } + if r.process == nil && r.group.Task == nil { + if slots := groups.SlotGlyphs(r.group.Gavel); slots != "" { + label += " " + slots + } + if r.group.GavelError != "" { + label += " · status unreadable" + } + } + glyph := " " + if r.parent { + glyph = "▾ " + if r.folded { + glyph = "▸ " + } + } + if r.detached { + glyph = "↰ " + } + if hog := m.hogLabel(usage); hog != "" { + label = "! " + hog + " · " + label + } + command := r.group.Command + if r.process != nil { + command = r.process.Command + } + rows = append(rows, processTableRow{command: command, name: strings.Repeat(" ", r.depth) + glyph + label, usage: usage, origin: groups.OriginText(r.group.Origin), context: groups.ContextText(r.group.Context)}) + ids = append(ids, r.key) + } + model := table.Model{Selected: m.selected} + if err := model.Set(api.NewTableFrom(rows), ids); err != nil { + panic(err) + } + return model +} diff --git a/internal/topui/view.go b/internal/topui/view.go index 61f22a9..c7259f3 100644 --- a/internal/topui/view.go +++ b/internal/topui/view.go @@ -6,30 +6,24 @@ import ( tea "charm.land/bubbletea/v2" "charm.land/lipgloss/v2" + "github.com/flanksource/telltale/internal/terminalui" "github.com/flanksource/telltale/pkg/tracing/groups" ) -// headerLines are the title, CPU, memory, swap and disk lines above the -// table. -const headerLines = 5 +// headerLines are the title, CPU, memory, swap, disk and gavel slots lines +// above the table. +const headerLines = 6 var ( - titleStyle = lipgloss.NewStyle().Bold(true) - headStyle = lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("244")) - mutedStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("244")) - selectedStyle = lipgloss.NewStyle().Reverse(true) - statusStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("208")) - userStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("39")) - kernelStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("203")) - memoryStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("78")) - swapStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("214")) - kindStyles = map[string]lipgloss.Style{ - groups.KindAgent: lipgloss.NewStyle().Foreground(lipgloss.Color("39")), - groups.KindTest: lipgloss.NewStyle().Foreground(lipgloss.Color("78")), - groups.KindBrowser: lipgloss.NewStyle().Foreground(lipgloss.Color("170")), - groups.KindService: lipgloss.NewStyle().Foreground(lipgloss.Color("214")), - groups.KindOther: mutedStyle, - } + titleStyle = terminalui.Title + headStyle = terminalui.Heading + mutedStyle = terminalui.Muted + selectedStyle = terminalui.Selected + statusStyle = terminalui.Status + userStyle = terminalui.CPU + kernelStyle = terminalui.Kernel + memoryStyle = terminalui.Memory + swapStyle = terminalui.Swap ) func (m Model) View() tea.View { @@ -42,17 +36,14 @@ func (m Model) render() string { if m.width <= 0 || m.height <= 0 { return "Reading process groups…" } - lines := m.header() - columns := layout(m.width) - lines = append(lines, headStyle.Render(fit(columns.head(), m.width))) - rows := m.rows() - for index := m.offset; index < m.offset+m.bodyHeight(); index++ { - if index >= len(rows) { - lines = append(lines, "") - continue - } - lines = append(lines, m.renderRow(columns, rows[index])) + if m.inspection != nil { + return "Recorded process detail · paused\n" + m.inspection.View(m.width, max(1, m.height-2)) + "\n↑↓ scroll · esc close · space resume live" } + if m.help { + return "Process Top\n↑↓ select · ←→ fold · enter list processes\ns sort · S reverse · / filter · b group\nd inspect and freeze · P pause/resume\nf process pane · x terminate · r nice (confirmation required)\n? close help · q quit" + } + lines := m.header() + lines = append(lines, terminalui.PadLines(m.processTable().View(m.width, max(1, m.bodyHeight()+1)), max(1, m.bodyHeight()+1))...) lines = append(lines, m.renderDetail()...) lines = append(lines, m.footer()) return strings.Join(lines[:min(len(lines), m.height)], "\n") @@ -71,7 +62,7 @@ func (m Model) bodyHeight() int { func (m Model) header() []string { if m.frame == nil { - return padLines(titleStyle.Render(fit("telltale top · "+m.options.Title, m.width))+"\n"+mutedStyle.Render("Reading process groups…"), headerLines) + return terminalui.PadLines(titleStyle.Render(terminalui.Fit("telltale top · "+m.options.Title, m.width))+"\n"+mutedStyle.Render("Reading process groups…"), headerLines) } processes := 0 present := map[string]bool{} @@ -89,6 +80,9 @@ func (m Model) header() []string { } title := fmt.Sprintf("telltale top · %s · %s · tick %d · %d groups · %d processes · by %s · sort %s%s", m.options.Title, m.frame.SampledAt.Local().Format("15:04:05"), m.frame.Tick, len(m.frame.Groups), processes, m.groupBy, m.order.By, direction) + if m.paused { + title += " · paused" + } if overhead := m.frame.Overhead; overhead != nil { title += fmt.Sprintf(" · overhead %s%% CPU %.0fms/tick", groups.Percent(overhead.CPUPercent), overhead.TickMillis) } @@ -100,23 +94,27 @@ func (m Model) header() []string { } host := m.frame.Host if host == nil { - return padLines(titleStyle.Render(fit(title, m.width))+"\n"+mutedStyle.Render(fit("The host's CPU, memory, swap and disks are read live, not recorded.", m.width)), headerLines) + return terminalui.PadLines(titleStyle.Render(terminalui.Fit(title, m.width))+"\n"+mutedStyle.Render(terminalui.Fit("The host's CPU, memory, swap and disks are read live, not recorded.", m.width)), headerLines) } barWidth := max(10, min(60, m.width/3)) - cpu := "CPU " + bar(barWidth, []share{{percentOf(host.CPUUserPercent), userStyle}, {percentOf(host.CPUSystemPercent), kernelStyle}}) + + cpu := "CPU " + terminalui.Bar(barWidth, []terminalui.Share{{Percent: percentOf(host.CPUUserPercent), Style: userStyle}, {Percent: percentOf(host.CPUSystemPercent), Style: kernelStyle}}) + fmt.Sprintf(" %s%% user %s%% kernel %s%% idle · %d cores · load %.2f %.2f %.2f", groups.Percent(host.CPUUserPercent), groups.Percent(host.CPUSystemPercent), groups.Percent(host.CPUIdlePercent), host.Cores, host.Load1, host.Load5, host.Load15) - memory := "MEM " + bar(barWidth, []share{{ratio(host.MemoryUsedBytes, host.MemoryTotalBytes), memoryStyle}}) + + memory := "MEM " + terminalui.Bar(barWidth, []terminalui.Share{{Percent: ratio(host.MemoryUsedBytes, host.MemoryTotalBytes), Style: memoryStyle}}) + fmt.Sprintf(" %s / %s", groups.Size(float64(host.MemoryUsedBytes)), groups.Size(float64(host.MemoryTotalBytes))) - swap := "SWP " + bar(barWidth, []share{{ratio(host.SwapUsedBytes, host.SwapTotalBytes), swapStyle}}) + + swap := "SWP " + terminalui.Bar(barWidth, []terminalui.Share{{Percent: ratio(host.SwapUsedBytes, host.SwapTotalBytes), Style: swapStyle}}) + fmt.Sprintf(" %s / %s", groups.Size(float64(host.SwapUsedBytes)), groups.Size(float64(host.SwapTotalBytes))) if host.SwapTotalBytes == 0 { swap = "SWP " + mutedStyle.Render("no swap") } disks := fmt.Sprintf("DSK read %s IOPS %s/s · write %s IOPS %s/s", ops(host.ReadOps), groups.Rate(host.ReadBps), ops(host.WriteOps), groups.Rate(host.WriteBps)) - return []string{titleStyle.Render(fit(title, m.width)), clip(cpu, m.width), clip(memory, m.width), clip(swap, m.width), clip(disks, m.width)} + slots := "SLT " + mutedStyle.Render("no gavel slot held or awaited") + if len(m.frame.Slots) > 0 { + slots = "SLT gavel " + groups.SlotsText(m.frame.Slots) + } + return []string{titleStyle.Render(terminalui.Fit(title, m.width)), terminalui.Clip(cpu, m.width), terminalui.Clip(memory, m.width), terminalui.Clip(swap, m.width), terminalui.Clip(disks, m.width), terminalui.Clip(slots, m.width)} } // ops is a known operations rate, in thousands from a thousand, or "-". @@ -130,25 +128,6 @@ func ops(value *float64) string { return fmt.Sprintf("%.0f", *value) } -// share is a part of a bar, in percent, and how it is drawn. -type share struct { - percent float64 - style lipgloss.Style -} - -func bar(width int, shares []share) string { - var b strings.Builder - used := 0 - for _, part := range shares { - cells := min(width-used, int(part.percent/100*float64(width)+0.5)) - if cells > 0 { - b.WriteString(part.style.Render(strings.Repeat("█", cells))) - used += cells - } - } - return "[" + b.String() + mutedStyle.Render(strings.Repeat("░", width-used)) + "]" -} - func percentOf(value *float64) float64 { if value == nil { return 0 @@ -163,129 +142,67 @@ func ratio(part, whole uint64) float64 { return 100 * float64(part) / float64(whole) } -// columns are the table's columns at a width: the name takes what the -// fixed ones leave, and origin and context show only where they fit. -type columns struct { - name int - origin, context int -} - -const fixedWidth = 6 + 7 + 8 + 9 + 9 + 9 + 9 + 6 - -func layout(width int) columns { - c := columns{} - if width >= 140 { - c.origin = 26 - } - if width >= 190 { - c.context = 34 - } - c.name = max(16, width-fixedWidth-c.origin-c.context) - return c -} - -func (c columns) head() string { - return c.line("GROUP", "PROCS", "CPU%", "RSS", "DISK R/s", "DISK W/s", "NET IN/s", "NET OUT/s", "SOCK", "ORIGIN", "CONTEXT") -} - -func (c columns) line(name, procs, cpu, rss, diskRead, diskWrite, netIn, netOut, sockets, origin, context string) string { - text := fmt.Sprintf("%-*s%6s%7s%8s%9s%9s%9s%9s%6s", c.name, groups.Truncate(name, c.name-1), procs, cpu, rss, diskRead, diskWrite, netIn, netOut, sockets) - if c.origin > 0 { - text += fmt.Sprintf(" %-*s", c.origin-2, groups.Truncate(origin, c.origin-2)) - } - if c.context > 0 { - text += " " + groups.Truncate(context, c.context-2) - } - return text -} - -func (m Model) renderRow(c columns, r row) string { - glyph := " " - if r.parent { - glyph = "▾ " - if r.folded { - glyph = "▸ " - } - } - usage := r.group.Total - label := groups.Label(r.group) - if m.cpuHog(usage.CPUPercent) { - label = "! CPU hog · " + label - } - text := fit(c.line(strings.Repeat(" ", r.depth)+glyph+label, fmt.Sprint(usage.Processes), groups.Percent(usage.CPUPercent), - groups.Size(float64(usage.RSSBytes)), groups.Rate(usage.DiskRead), groups.Rate(usage.DiskWrite), groups.Rate(usage.NetRx), - groups.Rate(usage.NetTx), groups.Count(usage.Sockets), groups.OriginText(r.group.Origin), groups.ContextText(r.group.Context)), m.width) - if r.group.Key == m.selected { - return selectedStyle.Render(text) - } - if m.cpuHog(usage.CPUPercent) { - return statusStyle.Render(text) - } - if style, ok := kindStyles[r.group.Kind]; ok { - return style.Render(text) - } - return text -} - func (m Model) renderDetail() []string { height := m.detailHeight() if height == 0 { return nil } - title := "processes of " + m.selected - if m.frame != nil { - for _, group := range m.frame.Groups { - if group.Key == m.selected { - title = "processes of " + groups.Label(group) + " and the groups under it" - } + title := m.selected + if rows := m.rows(); m.selectedIndex(rows) >= 0 { + selected := rows[m.selectedIndex(rows)] + title = groups.Label(selected.group) + if selected.process != nil { + title = fmt.Sprintf("%d %s", selected.process.PID, groups.ShortCommand(selected.process.Command)) } } - lines := []string{headStyle.Render(fit("── "+title+" "+strings.Repeat("─", m.width), m.width))} processes, kept := m.selectedProcesses() + if kept { + title += fmt.Sprintf(" · %d processes", len(processes)) + } + heading := "── " + groups.Truncate(title, max(1, m.width-4)) + " " + lines := []string{headStyle.Render(heading + strings.Repeat("─", max(0, m.width-lipgloss.Width(heading))))} + rows := height if !kept { - return append(lines, padLines(mutedStyle.Render("A recording keeps each group's use, not its processes."), height-1)...) + return append(lines, terminalui.PadLines(mutedStyle.Render("A recording keeps each group's use, not its processes."), rows-1)...) } - commandWidth := max(10, m.width-(8+12+7+8+9+9+9+9+6+24)) - lines = append(lines, headStyle.Render(fit(fmt.Sprintf("%8s %-11s%7s%8s%9s%9s%9s%9s%6s %-22s%s", - "PID", "USER", "CPU%", "RSS", "DISK R/s", "DISK W/s", "NET IN/s", "NET OUT/s", "SOCK", "ORIGIN", "COMMAND"), m.width))) + commandWidth := max(10, m.width-(8+12+7+8+8+8+9+9+9+9+6+24)) + lines = append(lines, headStyle.Render(terminalui.Fit(fmt.Sprintf("%8s %-11s%7s%8s%8s%8s%9s%9s%9s%9s%6s %-22s%s", "PID", "USER", "CPU%", "RSS", "CMPRS", "SWAP", "DISK R/s", "DISK W/s", "NET IN/s", "NET OUT/s", "SOCK", "ORIGIN", "COMMAND"), m.width))) start := 0 for index, p := range processes { if p.Key == m.processKey { - start = max(0, index-(height-3)) + start = max(0, index-(rows-3)) } } - for index := start; index < min(len(processes), start+height-2); index++ { + for index := start; index < min(len(processes), start+rows-2); index++ { p := processes[index] command := p.Command - if m.cpuHog(p.CPUPercent) { - command = "! CPU hog · " + command + if hog := m.hogLabel(p.Usage); hog != "" { + command = "! " + hog + " · " + command } - text := fit(fmt.Sprintf("%8d %-11s%7s%8s%9s%9s%9s%9s%6s %-22s%s", - p.PID, groups.Truncate(p.Origin.Owner, 10), groups.Percent(p.CPUPercent), groups.Size(float64(p.RSSBytes)), - groups.Rate(p.DiskRead), groups.Rate(p.DiskWrite), groups.Rate(p.NetRx), groups.Rate(p.NetTx), groups.Count(p.Sockets), - groups.Truncate(groups.OriginText(p.Origin), 21), groups.Truncate(command, commandWidth)), m.width) + text := terminalui.Fit(fmt.Sprintf("%8d %-11s%7s%8s%8s%8s%9s%9s%9s%9s%6s %-22s%s", p.PID, groups.Truncate(p.Origin.Owner, 10), groups.Percent(p.CPUPercent), groups.Size(float64(p.RSSBytes)), groups.MemorySize(p.CompressedBytes), groups.MemorySize(p.SwapBytes), groups.Rate(p.DiskRead), groups.Rate(p.DiskWrite), groups.Rate(p.NetRx), groups.Rate(p.NetTx), groups.Count(p.Sockets), groups.Truncate(groups.OriginText(p.Origin), 21), groups.Truncate(command, commandWidth)), m.width) + if p.Key == m.processKey || (m.processKey == "" && index == 0) { text = selectedStyle.Render(text) - } else if m.cpuHog(p.CPUPercent) { + } else if m.hogLabel(p.Usage) != "" { text = statusStyle.Render(text) } lines = append(lines, text) } - return padLines(strings.Join(lines, "\n"), height) + return terminalui.PadLines(strings.Join(lines, "\n"), height) } func (m Model) footer() string { if m.status != "" { - return statusStyle.Render(fit(m.status, m.width)) + return statusStyle.Render(terminalui.Fit(m.status, m.width)) } if m.typing { - return mutedStyle.Render(fit("type to filter · enter keeps it · esc clears it", m.width)) + return mutedStyle.Render(terminalui.Fit("type to filter · enter keeps it · esc clears it", m.width)) } if m.detail { - return mutedStyle.Render(fit("[ ] choose process x terminate r nice=10 y confirm esc cancel · ! CPU hog >25% of all cores", m.width)) + return mutedStyle.Render(terminalui.Fit("[ ] choose process x terminate r nice=10 y confirm esc cancel f close · hogs pinned 120s · CMPRS includes swapped-out pages on macOS", m.width)) } toggles := fmt.Sprintf("o other:%s n net:%s", onOff(m.toggles.All), onOff(m.toggles.Net)) - return mutedStyle.Render(fit("↑↓ move ←→ fold enter processes b group s sort S reverse / filter "+toggles+" q quit", m.width)) + return mutedStyle.Render(terminalui.Fit("↑↓ move enter processes d inspect P pause/resume / filter ? help "+toggles+" q quit", m.width)) } func onOff(on bool) string { @@ -294,25 +211,3 @@ func onOff(on bool) string { } return "off" } - -// fit pads or cuts plain text to exactly width cells. -func fit(text string, width int) string { - if lipgloss.Width(text) > width { - return groups.Truncate(text, width) - } - return text + strings.Repeat(" ", width-lipgloss.Width(text)) -} - -// clip cuts styled text to width cells. -func clip(text string, width int) string { - return lipgloss.NewStyle().MaxWidth(width).Render(text) -} - -// padLines is text as exactly height lines. -func padLines(text string, height int) []string { - lines := strings.Split(text, "\n") - for len(lines) < height { - lines = append(lines, "") - } - return lines[:height] -} diff --git a/internal/tracequery/facets.go b/internal/tracequery/facets.go deleted file mode 100644 index ab68a63..0000000 --- a/internal/tracequery/facets.go +++ /dev/null @@ -1,85 +0,0 @@ -package tracequery - -import ( - "cmp" - "fmt" - "slices" -) - -const ( - maxAttributeSuggestions = 100 - maxAttributeValueLength = 100 -) - -// facets gather what a search's traces hold, for its filters to offer: the -// traces of each root operation and the values of their root spans' attributes. -type facets struct { - byOperation map[string][]Summary - byAttribute map[string]int -} - -func newFacets() *facets { - return &facets{byOperation: map[string][]Summary{}, byAttribute: map[string]int{}} -} - -func (f *facets) add(summary Summary, root Span) { - f.byOperation[summary.Operation] = append(f.byOperation[summary.Operation], summary) - seen := map[string]bool{} - for _, values := range []map[string]any{root.Attributes, root.Resource} { - for key, value := range values { - switch value.(type) { - case string, bool, int, int32, int64, uint64, float32, float64: - default: - continue - } - text := fmt.Sprint(value) - if len(text) > maxAttributeValueLength || seen[key] { - continue - } - seen[key] = true - f.byAttribute[key+"="+text]++ - } - } -} - -// operations are the root operations, those of multi-span traces first, then the most frequent. -func (f *facets) operations() []OperationStats { - result := []OperationStats{} - for operation, traces := range f.byOperation { - durations := []float64{} - single := true - for _, trace := range traces { - durations = append(durations, trace.DurationMs) - single = single && trace.SpanCount == 1 - } - slices.Sort(durations) - result = append(result, OperationStats{Operation: operation, Count: len(traces), SingleSpan: single, P50Ms: quantile(durations, 0.5), P95Ms: quantile(durations, 0.95)}) - } - slices.SortFunc(result, func(a, b OperationStats) int { - if a.SingleSpan != b.SingleSpan { - if a.SingleSpan { - return 1 - } - return -1 - } - return cmp.Or(cmp.Compare(b.Count, a.Count), cmp.Compare(a.Operation, b.Operation)) - }) - return result -} - -// attributes are the most frequent root span attribute values, key=value. -func (f *facets) attributes() []AttributeCount { - result := []AttributeCount{} - for value, count := range f.byAttribute { - result = append(result, AttributeCount{Value: value, Count: count}) - } - slices.SortFunc(result, func(a, b AttributeCount) int { - return cmp.Or(cmp.Compare(b.Count, a.Count), cmp.Compare(a.Value, b.Value)) - }) - return result[:min(len(result), maxAttributeSuggestions)] -} - -// quantile is the q-quantile of sorted values, the nearest rank at or below it. -func quantile(sorted []float64, q float64) float64 { - return sorted[min(len(sorted)-1, int(q*float64(len(sorted))))] -} diff --git a/internal/tracequery/filter.go b/internal/tracequery/filter.go deleted file mode 100644 index a585643..0000000 --- a/internal/tracequery/filter.go +++ /dev/null @@ -1,127 +0,0 @@ -package tracequery - -import ( - "fmt" - "regexp" - "strings" -) - -// Mode includes or excludes a filter value. -type Mode string - -const ( - Include Mode = "include" - Exclude Mode = "exclude" -) - -// Modes are a filter's values, each included or excluded: a trace passes -// when it has one of the included values (or none is included) and none of -// the excluded ones. -type Modes map[string]Mode - -func (Modes) JSONSchema() map[string]any { - return map[string]any{"type": "object", "additionalProperties": map[string]any{"type": "string", "enum": []string{string(Include), string(Exclude)}}} -} - -func (m Modes) validate(name string, value func(string) error) error { - for key, mode := range m { - if mode != Include && mode != Exclude { - return fmt.Errorf("%s %q: mode must be include or exclude, not %q", name, key, mode) - } - if err := value(key); err != nil { - return fmt.Errorf("%s %q: %w", name, key, err) - } - } - return nil -} - -func (m Modes) values(mode Mode) []string { - result := []string{} - for key, value := range m { - if value == mode { - result = append(result, key) - } - } - return result -} - -// pass reports whether a trace, holding the values has reports, passes m. -func (m Modes) pass(has func(string) bool) bool { - included := m.values(Include) - matched := len(included) == 0 - for _, value := range included { - matched = matched || has(value) - } - for _, value := range m.values(Exclude) { - if has(value) { - return false - } - } - return matched -} - -// passAttributes reports whether spans pass attribute filters: included -// expressions are any-of per key and all-of across keys, and any span -// matching an excluded expression rejects the trace. -func (m Modes) passAttributes(spans []Span) bool { - byKey := map[string][]string{} - for _, expr := range m.values(Include) { - key, _, _ := strings.Cut(expr, "=") - byKey[key] = append(byKey[key], expr) - } - anySpan := func(expr string) bool { - for _, span := range spans { - if attributeMatches(span, expr) { - return true - } - } - return false - } - for _, exprs := range byKey { - matched := false - for _, expr := range exprs { - matched = matched || anySpan(expr) - } - if !matched { - return false - } - } - for _, expr := range m.values(Exclude) { - if anySpan(expr) { - return false - } - } - return true -} - -func validAttribute(expr string) error { - key, _, _ := strings.Cut(expr, "=") - if strings.TrimSpace(key) == "" { - return fmt.Errorf("an attribute filter is key, key=value or key=prefix*") - } - return nil -} - -func validStatus(status string) error { - if status != "ok" && status != "error" { - return fmt.Errorf("status must be error or ok") - } - return nil -} - -// attributeMatches reports whether span has the attribute expr names: a bare -// key matches its presence, key=value its value, with * as a wildcard. Span -// attributes win over resource attributes of the same key. -func attributeMatches(span Span, expr string) bool { - key, want, hasValue := strings.Cut(expr, "=") - key = strings.TrimSpace(key) - value, ok := span.Attributes[key] - if !ok { - value, ok = span.Resource[key] - } - if !ok || !hasValue { - return ok - } - pattern := strings.ReplaceAll(regexp.QuoteMeta(strings.TrimSpace(want)), `\*`, ".*") - return regexp.MustCompile("^" + pattern + "$").MatchString(fmt.Sprint(value)) -} diff --git a/internal/tracequery/opensearch.go b/internal/tracequery/opensearch.go deleted file mode 100644 index 10feed1..0000000 --- a/internal/tracequery/opensearch.go +++ /dev/null @@ -1,104 +0,0 @@ -package tracequery - -import ( - "context" - "fmt" - - dbcontext "github.com/flanksource/commons-db/context" - "github.com/flanksource/commons-db/models" - "github.com/flanksource/commons-db/query" - "github.com/flanksource/commons-db/query/esdsl" - "github.com/flanksource/commons-db/query/providers" - "github.com/flanksource/commons-db/types" - imported "github.com/flanksource/telltale/pkg/tracing/opensearch" -) - -func (s Service) openSearch(ctx context.Context, r Request, c *models.Connection, ids []string) ([]Span, error) { - if c.Type != models.ConnectionTypeOpenSearch && c.Type != models.ConnectionTypeOpenTelemetry { - return nil, fmt.Errorf("source %q has unsupported type %q", r.Source, c.Type) - } - queryContext := s.Connections.Attach(dbcontext.NewContext(ctx)) - reference := r.Source - if c.Type == models.ConnectionTypeOpenSearch { - reference = "connection://telltale-trace-query" - next := queryContext.ConnectionResolver() - queryContext = queryContext.WithConnectionResolver(func(ctx dbcontext.Context, ref string) (*models.Connection, error) { - if ref == reference { - return &models.Connection{Name: "telltale-trace-query", Type: models.ConnectionTypeOpenTelemetry, Properties: types.JSONStringMap{"connection": c.ID.String()}}, nil - } - return next(ctx, ref) - }) - } - options := map[string]any{} - for _, key := range []string{"index", "format", "dateField", "traceIdField", "spanIdField", "parentIdField", "parentRefType", "serviceField", "operationField"} { - if value := c.Properties[key]; value != "" { - options[key] = value - } - } - if r.Index != "" { - options["index"] = r.Index - } - if r.Format != "" { - options["format"] = string(r.Format) - } - format, _ := options["format"].(string) - if format == "jaeger" { - jaegerDefaults(options) - } - if len(ids) == 0 && r.TraceID != "" { - ids = []string{r.TraceID} - } - if len(ids) > 0 { - field := "trace_id" - if value, ok := options["traceIdField"].(string); ok { - field = value - } - values := make([]esdsl.Value, len(ids)) - for i, id := range ids { - values[i] = *esdsl.Literal(id) - } - options["search"] = &esdsl.Search{Query: &esdsl.Condition{Op: "terms", Field: field, Values: values}} - } - return readOpenSearch(queryContext, providers.OpenTelemetryRead{Connection: reference, Options: options, From: r.From, To: r.Until}) -} - -func jaegerDefaults(options map[string]any) { - for key, value := range map[string]string{"dateField": "startTimeMillis", "traceIdField": "traceID", "spanIdField": "spanID", "parentRefType": "CHILD_OF", "serviceField": "process.serviceName", "operationField": "operationName"} { - if _, ok := options[key]; !ok { - options[key] = value - } - } -} - -func readOpenSearch(ctx dbcontext.Context, request providers.OpenTelemetryRead) ([]Span, error) { - readCtx, cancel := context.WithCancel(ctx) - defer cancel() - spans := []Span{} - var readErr error - err := providers.ReadOpenTelemetrySpans(ctx.Wrap(readCtx), request, func(row query.Row) { - if readErr != nil { - return - } - span, err := imported.FromRow(row) - if err != nil { - readErr = err - cancel() - return - } - converted, err := fromSpan(span, false) - if err != nil { - readErr = err - cancel() - return - } - spans = append(spans, converted) - if len(spans) > maxSpans { - readErr = fmt.Errorf("OpenSearch read exceeds %d spans; narrow the time window or index", maxSpans) - cancel() - } - }) - if readErr != nil { - return nil, readErr - } - return spans, err -} diff --git a/internal/tracequery/opensearch_test.go b/internal/tracequery/opensearch_test.go deleted file mode 100644 index e64b9a8..0000000 --- a/internal/tracequery/opensearch_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package tracequery - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "strings" - "time" - - "github.com/flanksource/commons-db/connection/crd" - "github.com/flanksource/commons-db/connection/store" - "github.com/flanksource/commons-db/models" - "github.com/flanksource/commons-db/types" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("OpenSearch trace search", func() { - DescribeTable("reads mapped spans through the shared provider", func(format, sourceType string) { - at := time.Now().UTC().Add(-time.Minute).Truncate(time.Millisecond) - index := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer GinkgoRecover() - if r.Method == http.MethodHead { - w.WriteHeader(200) - return - } - w.Header().Set("Content-Type", "application/json") - fieldType := "date" - if format == "jaeger" { - fieldType = "long" - } - if strings.HasSuffix(r.URL.Path, "/_field_caps") { - Expect(json.NewEncoder(w).Encode(map[string]any{"fields": map[string]any{r.URL.Query().Get("fields"): map[string]any{fieldType: map[string]any{"searchable": true, "aggregatable": true}}}})).To(Succeed()) - return - } - if strings.Contains(r.URL.Path, "/_mapping/field/") { - field := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] - Expect(json.NewEncoder(w).Encode(map[string]any{"traces-example": map[string]any{"mappings": map[string]any{field: map[string]any{"full_name": field, "mapping": map[string]any{field: map[string]any{"type": fieldType}}}}}})).To(Succeed()) - return - } - var body map[string]any - Expect(json.NewDecoder(r.Body).Decode(&body)).To(Succeed()) - hits := []any{} - if body["search_after"] == nil { - for i, id := range []string{"root", "child"} { - started := at.Add(time.Duration(i) * 20 * time.Millisecond) - doc := map[string]any{"@timestamp": started.Format(time.RFC3339Nano), "trace_id": "trace-search", "span_id": id, "operation_name": id, "service_name": "checkout", "duration_ms": 100} - if i > 0 { - doc["parent_id"] = "root" - } - if format == "jaeger" { - doc = map[string]any{"startTimeMillis": started.UnixMilli(), "traceID": "trace-search", "spanID": id, "operationName": id, "process": map[string]any{"serviceName": "checkout"}, "duration": 100000} - if i > 0 { - doc["references"] = []any{map[string]any{"refType": "CHILD_OF", "spanID": "root"}} - } - } - hits = append(hits, map[string]any{"_index": "traces-example", "_id": id, "_source": doc, "sort": []any{started.UnixMilli(), id}}) - } - } - Expect(json.NewEncoder(w).Encode(map[string]any{"took": 1, "hits": map[string]any{"total": map[string]any{"value": len(hits), "relation": "eq"}, "hits": hits}})).To(Succeed()) - })) - DeferCleanup(index.Close) - catalog, err := store.New(store.Layer{Name: store.LayerCWD, Dir: GinkgoT().TempDir()}) - Expect(err).NotTo(HaveOccurred()) - search, err := crd.FromModel(models.Connection{Name: "search", Type: models.ConnectionTypeOpenSearch, URL: index.URL}) - Expect(err).NotTo(HaveOccurred()) - Expect(catalog.Save(store.LayerCWD, search)).To(Succeed()) - source := "search" - if sourceType == "opentelemetry" { - outer, err := crd.FromModel(models.Connection{Name: "traces", Type: models.ConnectionTypeOpenTelemetry, Properties: types.JSONStringMap{"connection": "connection://search"}}) - Expect(err).NotTo(HaveOccurred()) - Expect(catalog.Save(store.LayerCWD, outer)).To(Succeed()) - source = "traces" - } - service := Service{Connections: catalog} - request := Request{Source: source, Index: "traces-*", Format: DocumentFormat(format)} - result, err := service.Search(context.Background(), request) - Expect(err).NotTo(HaveOccurred()) - Expect(result.Traces).To(HaveExactElements(And(HaveField("TraceID", "trace-search"), HaveField("SpanCount", 2)))) - detail, err := service.Trace(context.Background(), request, "trace-search") - Expect(err).NotTo(HaveOccurred()) - Expect(detail.Spans[1].ParentID).To(Equal("root")) - }, Entry("flat OpenSearch", "flat", "opensearch"), Entry("Jaeger OpenTelemetry connection", "jaeger", "opentelemetry")) -}) diff --git a/internal/tracequery/search.go b/internal/tracequery/search.go deleted file mode 100644 index ca97b54..0000000 --- a/internal/tracequery/search.go +++ /dev/null @@ -1,242 +0,0 @@ -package tracequery - -import ( - "errors" - "fmt" - "math" - "reflect" - "slices" - "strings" - "time" -) - -// isError reads the canonical status every span source normalizes to. -func isError(status string) bool { return status == "error" } - -func duration(value string) (float64, error) { - if value == "" { - return 0, nil - } - d, err := time.ParseDuration(value) - if err != nil || d < 0 { - return 0, fmt.Errorf("invalid duration %q: expected a nonnegative duration such as 100ms", value) - } - return float64(d) / float64(time.Millisecond), nil -} - -func (r Request) validate() error { - min, err := duration(r.MinDuration) - if err != nil { - return err - } - max, err := duration(r.MaxDuration) - if err != nil { - return err - } - if r.MaxDuration != "" && max < min { - return fmt.Errorf("maximum duration must be at least minimum duration") - } - if r.Limit < 0 || r.Limit > 1000 { - return fmt.Errorf("trace limit must be between 1 and 1000") - } - if r.MinSpans < 0 { - return fmt.Errorf("minimum spans must not be negative") - } - anyValue := func(string) error { return nil } - if err := errors.Join(r.Services.validate("service", anyValue), r.Operations.validate("operation", anyValue), r.Statuses.validate("status", validStatus), r.Attributes.validate("attribute", validAttribute)); err != nil { - return err - } - if r.Format != "" && r.Format != "flat" && r.Format != "jaeger" { - return fmt.Errorf("format must be flat or jaeger") - } - for _, value := range []string{r.From, r.Until} { - if value != "" { - if _, err := time.Parse(time.RFC3339Nano, value); err != nil { - return fmt.Errorf("invalid timestamp %q: %w", value, err) - } - } - } - return nil -} - -func groups(spans []Span) (map[string][]Span, error) { - result := map[string][]Span{} - seen := map[string]Span{} - for _, span := range spans { - if span.TraceID == "" || span.SpanID == "" || span.StartedAt.IsZero() || math.IsNaN(span.DurationMs) || math.IsInf(span.DurationMs, 0) || span.DurationMs < 0 || span.DurationMs >= float64(math.MaxInt64)/float64(time.Millisecond) { - return nil, fmt.Errorf("invalid span %q in trace %q: identity, start time and a representable nonnegative duration are required", span.SpanID, span.TraceID) - } - key := span.TraceID + "/" + span.SpanID - if old, ok := seen[key]; ok { - if !reflect.DeepEqual(old, span) { - return nil, fmt.Errorf("conflicting span %s", key) - } - continue - } - seen[key] = span - result[span.TraceID] = append(result[span.TraceID], span) - } - return result, nil -} - -// summarize is a trace's summary and its root span: the parentless one, or the earliest. -func summarize(spans []Span) (Summary, Span) { - first := spans[0] - start, end := first.StartedAt, first.StartedAt - root := first - services := map[string]bool{} - errors := 0 - for _, span := range spans { - if span.StartedAt.Before(start) { - start = span.StartedAt - } - if finish := span.StartedAt.Add(time.Duration(span.DurationMs * float64(time.Millisecond))); finish.After(end) { - end = finish - } - if span.ParentID == "" && root.ParentID != "" { - root = span - } else if (span.ParentID == "" || root.ParentID != "") && span.StartedAt.Before(root.StartedAt) { - root = span - } - services[span.Service] = true - if isError(span.Status) { - errors++ - } - } - result := Summary{TraceID: first.TraceID, Service: root.Service, Operation: root.Operation, StartedAt: start, DurationMs: float64(end.Sub(start)) / float64(time.Millisecond), SpanCount: len(spans), Errors: errors, Services: []string{}} - for service := range services { - result.Services = append(result.Services, service) - } - slices.Sort(result.Services) - return result, root -} - -// inWindow reports whether span starts in the request's window and trace. -func inWindow(span Span, r Request) bool { - if r.From != "" { - from, _ := time.Parse(time.RFC3339Nano, r.From) - if span.StartedAt.Before(from) { - return false - } - } - if r.Until != "" { - until, _ := time.Parse(time.RFC3339Nano, r.Until) - if !span.StartedAt.Before(until) { - return false - } - } - return r.TraceID == "" || r.TraceID == span.TraceID -} - -// passes reports whether a trace passes every filter but the root operation -// and span-count ones, which the root operation counts leave out. -func (r Request) passes(summary Summary, spans []Span, min, max float64) bool { - return summary.DurationMs >= min && (r.MaxDuration == "" || summary.DurationMs <= max) && - r.Services.pass(func(service string) bool { return slices.Contains(summary.Services, service) }) && - r.Statuses.pass(func(status string) bool { return (status == "error") == (summary.Errors > 0) }) && - r.Attributes.passAttributes(spans) -} - -// Search filters whole traces with a span in the window: by their duration, -// services, status, any span's attributes, root operation and span count. -func Search(spans []Span, r Request) (SearchResult, error) { - result := SearchResult{Traces: []Summary{}, Services: []string{}, RootOperations: []OperationStats{}, Attributes: []AttributeCount{}} - if err := r.validate(); err != nil { - return result, err - } - grouped, err := groups(spans) - if err != nil { - return result, err - } - min, _ := duration(r.MinDuration) - max, _ := duration(r.MaxDuration) - services := map[string]bool{} - facets := newFacets() - for _, trace := range grouped { - if !slices.ContainsFunc(trace, func(span Span) bool { return inWindow(span, r) }) { - continue - } - summary, root := summarize(trace) - for _, service := range summary.Services { - services[service] = true - } - if !r.passes(summary, trace, min, max) { - continue - } - facets.add(summary, root) - if summary.SpanCount >= r.MinSpans && r.Operations.pass(func(operation string) bool { return operation == summary.Operation }) { - result.Traces = append(result.Traces, summary) - } - } - result.RootOperations, result.Attributes = facets.operations(), facets.attributes() - slices.SortFunc(result.Traces, func(a, b Summary) int { - if cmp := b.StartedAt.Compare(a.StartedAt); cmp != 0 { - return cmp - } - return strings.Compare(a.TraceID, b.TraceID) - }) - result.Total = len(result.Traces) - limit := r.Limit - if limit == 0 { - limit = 100 - } - if len(result.Traces) > limit { - result.Traces = result.Traces[:limit] - } - for key := range services { - result.Services = append(result.Services, key) - } - slices.Sort(result.Services) - return result, nil -} - -func Trace(spans []Span, id string) (Detail, error) { - grouped, err := groups(spans) - if err != nil { - return Detail{}, err - } - selected := grouped[id] - if len(selected) == 0 { - return Detail{}, fmt.Errorf("%w: %q", ErrTraceNotFound, id) - } - slices.SortFunc(selected, func(a, b Span) int { - if cmp := a.StartedAt.Compare(b.StartedAt); cmp != 0 { - return cmp - } - return strings.Compare(a.SpanID, b.SpanID) - }) - parents := map[string]string{} - for _, span := range selected { - parents[span.SpanID] = span.ParentID - } - missing := map[string]bool{} - finished := map[string]bool{} - for _, span := range selected { - visited := map[string]bool{} - for id := span.SpanID; id != ""; { - if finished[id] { - break - } - if visited[id] { - return Detail{}, fmt.Errorf("parent cycle in trace %s at span %s", span.TraceID, id) - } - visited[id] = true - parent, ok := parents[id] - if !ok { - missing[id] = true - break - } - id = parent - } - for id := range visited { - finished[id] = true - } - } - summary, _ := summarize(selected) - result := Detail{Summary: summary, Spans: selected, MissingParents: []string{}} - for id := range missing { - result.MissingParents = append(result.MissingParents, id) - } - slices.Sort(result.MissingParents) - return result, nil -} diff --git a/internal/tracequery/search_test.go b/internal/tracequery/search_test.go deleted file mode 100644 index d9833d2..0000000 --- a/internal/tracequery/search_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package tracequery - -import ( - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestTraceQuery(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Trace query") } - -var _ = Describe("trace search", func() { - at := time.Date(2026, 10, 29, 10, 0, 0, 0, time.UTC) - var spans []Span - BeforeEach(func() { - spans = []Span{ - {TraceID: "trace-a", SpanID: "root", Service: "checkout", Operation: "POST /orders", StartedAt: at, DurationMs: 100, Status: "ok"}, - {TraceID: "trace-a", SpanID: "child", ParentID: "root", Service: "payments", Operation: "charge", StartedAt: at.Add(10 * time.Millisecond), DurationMs: 40, Status: "error", Attributes: map[string]any{"http.status_code": 500}}, - {TraceID: "trace-b", SpanID: "other", Service: "checkout", Operation: "GET /orders", StartedAt: at.Add(time.Second), DurationMs: 10, Status: "ok"}, - } - }) - It("filters whole traces by their services, root operation, status and any span's attributes, returning whole-trace totals", func() { - result, err := Search(spans, Request{ - Services: Modes{"payments": Include}, Operations: Modes{"POST /orders": Include}, Statuses: Modes{"error": Include}, - Attributes: Modes{"http.status_code=5*": Include}, MinDuration: "90ms", - }) - Expect(err).NotTo(HaveOccurred()) - Expect(result.Traces).To(HaveExactElements(Summary{TraceID: "trace-a", Service: "checkout", Operation: "POST /orders", StartedAt: at, DurationMs: 100, SpanCount: 2, Errors: 1, Services: []string{"checkout", "payments"}})) - result, err = Search(spans, Request{Operations: Modes{"charge": Include}}) - Expect(err).NotTo(HaveOccurred()) - Expect(result.Total).To(BeZero(), "charge runs in a trace but is not its root") - }) - DescribeTable("includes are any-of, excludes reject, and attribute includes are any-of per key and all-of across keys", - func(request Request, want []string) { - result, err := Search(spans, request) - Expect(err).NotTo(HaveOccurred()) - ids := []string{} - for _, trace := range result.Traces { - ids = append(ids, trace.TraceID) - } - Expect(ids).To(Equal(want)) - }, - Entry("either root operation", Request{Operations: Modes{"POST /orders": Include, "GET /orders": Include}}, []string{"trace-b", "trace-a"}), - Entry("an excluded service", Request{Services: Modes{"payments": Exclude}}, []string{"trace-b"}), - Entry("ok traces", Request{Statuses: Modes{"ok": Include}}, []string{"trace-b"}), - Entry("a bare key matches its presence", Request{Attributes: Modes{"http.status_code": Include}}, []string{"trace-a"}), - Entry("values of one key are any-of", Request{Attributes: Modes{"http.status_code=404": Include, "http.status_code=500": Include}}, []string{"trace-a"}), - Entry("keys are all-of", Request{Attributes: Modes{"http.status_code=500": Include, "absent": Include}}, []string{}), - Entry("an excluded attribute", Request{Attributes: Modes{"http.status_code": Exclude}}, []string{"trace-b"}), - Entry("traces of at least two spans", Request{MinSpans: 2}, []string{"trace-a"}), - ) - It("counts every root operation in the window, with its percentiles, whatever the operation and span filters", func() { - result, err := Search(spans, Request{Operations: Modes{"GET /orders": Include}, MinSpans: 2}) - Expect(err).NotTo(HaveOccurred()) - Expect(result.Total).To(BeZero()) - Expect(result.RootOperations).To(Equal([]OperationStats{ - {Operation: "POST /orders", Count: 1, P50Ms: 100, P95Ms: 100}, - {Operation: "GET /orders", Count: 1, SingleSpan: true, P50Ms: 10, P95Ms: 10}, - })) - result, err = Search(spans, Request{Services: Modes{"payments": Exclude}}) - Expect(err).NotTo(HaveOccurred()) - Expect(result.RootOperations).To(HaveExactElements(HaveField("Operation", "GET /orders")), "the other filters still narrow the counts") - }) - It("suggests the root spans' attribute values, most frequent first", func() { - spans[0].Attributes = map[string]any{"http.method": "POST"} - spans[2].Attributes = map[string]any{"http.method": "GET", "payload": map[string]any{"nested": true}} - spans[2].Resource = map[string]any{"service.name": "checkout"} - spans[0].Resource = map[string]any{"service.name": "checkout"} - result, err := Search(spans, Request{}) - Expect(err).NotTo(HaveOccurred()) - Expect(result.Attributes).To(Equal([]AttributeCount{ - {Value: "service.name=checkout", Count: 2}, - {Value: "http.method=GET", Count: 1}, - {Value: "http.method=POST", Count: 1}, - })) - }) - It("caps traces after grouping and keeps deterministic newest-first order", func() { - result, err := Search(spans, Request{Limit: 1}) - Expect(err).NotTo(HaveOccurred()) - Expect(result.Total).To(Equal(2)) - Expect(result.Traces).To(HaveExactElements(HaveField("TraceID", "trace-b"))) - }) - It("reports missing parents and refuses cycles or conflicting duplicate span IDs", func() { - detail, err := Trace(spans, "trace-a") - Expect(err).NotTo(HaveOccurred()) - Expect(detail.Spans).To(HaveLen(2)) - spans[0].ParentID = "absent" - detail, err = Trace(spans, "trace-a") - Expect(err).NotTo(HaveOccurred()) - Expect(detail.MissingParents).To(Equal([]string{"absent"})) - spans[0].ParentID = "child" - _, err = Trace(spans, "trace-a") - Expect(err).To(MatchError(ContainSubstring("cycle"))) - }) - It("names a rootless trace after its earliest span regardless of input order", func() { - spans[0].ParentID = "absent" - result, err := Search([]Span{spans[1], spans[0]}, Request{}) - Expect(err).NotTo(HaveOccurred()) - Expect(result.Traces[0].Operation).To(Equal("POST /orders")) - }) - It("deduplicates identical spans and refuses conflicting or overflowing durations", func() { - result, err := Search(append(spans, spans[0]), Request{}) - Expect(err).NotTo(HaveOccurred()) - Expect(result.Traces[1].SpanCount).To(Equal(2)) - changed := spans[0] - changed.DurationMs++ - _, err = Search(append(spans, changed), Request{}) - Expect(err).To(MatchError(ContainSubstring("conflicting span"))) - spans[0].DurationMs = 1e20 - _, err = Search(spans, Request{}) - Expect(err).To(MatchError(ContainSubstring("invalid span"))) - }) - DescribeTable("refuses invalid filters", func(request Request) { _, err := Search(spans, request); Expect(err).To(HaveOccurred()) }, - Entry("negative duration", Request{MinDuration: "-1ms"}), Entry("duration order", Request{MinDuration: "2s", MaxDuration: "1s"}), Entry("limit", Request{Limit: 1001}), - Entry("status", Request{Statuses: Modes{"maybe": Include}}), Entry("mode", Request{Services: Modes{"checkout": "only"}}), - Entry("attribute key", Request{Attributes: Modes{"=500": Include}}), Entry("span count", Request{MinSpans: -1})) -}) diff --git a/internal/tracequery/service.go b/internal/tracequery/service.go index 9a78086..26485bd 100644 --- a/internal/tracequery/service.go +++ b/internal/tracequery/service.go @@ -2,130 +2,24 @@ package tracequery import ( "context" - "fmt" - "slices" - "time" - - dbcontext "github.com/flanksource/commons-db/context" - "github.com/flanksource/commons-db/models" + shared "github.com/flanksource/commons-db/query/tracequery" ) -const maxSpans = 100000 - +func (s Service) query() shared.Service { + return shared.Service{Connections: s.Connections, Read: s.load} +} func (s Service) Sources() ([]Source, error) { - if s.Connections == nil { - return nil, fmt.Errorf("trace source catalog is not configured") - } - result := []Source{{ID: "records", Name: "Recorded traces", Type: "records"}} - connections, err := s.Connections.ListConnections() - if err != nil { - return nil, err - } - for _, c := range connections { - if c.Type == models.ConnectionTypeOpenTelemetry || c.Type == models.ConnectionTypeOpenSearch { - name := c.Name - if c.Namespace != "" { - name = c.Namespace + "/" + name - } - result = append(result, Source{ID: c.ID.String(), Name: name, Type: c.Type}) - } - } - return result, nil + return (shared.Service{Connections: s.Connections}).Sources() } - func (s Service) Search(ctx context.Context, r Request) (SearchResult, error) { - if err := r.validate(); err != nil { - return SearchResult{}, err - } - var err error - r, err = r.window(time.Now()) - if err != nil { - return SearchResult{}, err - } - spans, err := s.load(ctx, r, nil) - if err != nil { - return SearchResult{}, err - } - ids := []string{} - for _, span := range spans { - if inWindow(span, r) && !slices.Contains(ids, span.TraceID) { - ids = append(ids, span.TraceID) - } - } - if len(ids) > 0 { - full := r - full.From, full.Until = "", "" - spans, err = s.load(ctx, full, ids) - if err != nil { - return SearchResult{}, err - } - } - return Search(spans, r) + return s.query().Search(ctx, r) } - func (s Service) Trace(ctx context.Context, r Request, id string) (Detail, error) { - if id == "" { - return Detail{}, fmt.Errorf("trace ID is required") - } - if err := r.validate(); err != nil { - return Detail{}, err - } - r.TraceID = id - r.From, r.Until = "", "" - spans, err := s.load(ctx, r, []string{id}) - if err != nil { - return Detail{}, err - } - return Trace(spans, id) + return s.query().Trace(ctx, r, id) } - func (s Service) load(ctx context.Context, r Request, ids []string) ([]Span, error) { if r.Source == "records" { return s.recorded(ctx, r, ids) } - if r.Source == "" { - return nil, fmt.Errorf("source is required") - } - if s.Connections == nil { - return nil, fmt.Errorf("trace source catalog is not configured") - } - c, err := s.Connections.ResolveConnection(dbcontext.NewContext(ctx), r.Source) - if err != nil { - return nil, err - } - if c == nil { - return nil, fmt.Errorf("source connection %q not found", r.Source) - } - return s.openSearch(ctx, r, c, ids) -} - -func (r Request) window(now time.Time) (Request, error) { - until := now.UTC() - var err error - if r.Until != "" { - until, err = time.Parse(time.RFC3339Nano, r.Until) - if err != nil { - return r, fmt.Errorf("until: %w", err) - } - } - lookback := r.Lookback - if lookback == "" { - lookback = "1h" - } - d, err := time.ParseDuration(lookback) - if err != nil || d <= 0 { - return r, fmt.Errorf("lookback must be a positive duration") - } - from := until.Add(-d) - if r.From != "" { - from, err = time.Parse(time.RFC3339Nano, r.From) - if err != nil { - return r, fmt.Errorf("from: %w", err) - } - } - if !from.Before(until) { - return r, fmt.Errorf("from must precede until") - } - r.From, r.Until = from.UTC().Format(time.RFC3339Nano), until.UTC().Format(time.RFC3339Nano) - return r, nil + return (shared.Service{Connections: s.Connections}).ReadSpans(ctx, r, ids) } diff --git a/internal/tracequery/suite_test.go b/internal/tracequery/suite_test.go new file mode 100644 index 0000000..f85a685 --- /dev/null +++ b/internal/tracequery/suite_test.go @@ -0,0 +1,9 @@ +package tracequery + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "testing" +) + +func TestTraceQuery(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Recorded trace query") } diff --git a/internal/tracequery/types.go b/internal/tracequery/types.go index e413e71..8483d9b 100644 --- a/internal/tracequery/types.go +++ b/internal/tracequery/types.go @@ -1,102 +1,25 @@ package tracequery import ( - "encoding/json" - "errors" - "time" - "github.com/flanksource/commons-db/connection/store" + shared "github.com/flanksource/commons-db/query/tracequery" ) -var ErrTraceNotFound = errors.New("trace not found") - -type DocumentFormat string - -func (DocumentFormat) JSONSchema() map[string]any { - return map[string]any{"type": "string", "enum": []string{"flat", "jaeger"}} -} - -type Span struct { - TraceID string `json:"traceId"` - SpanID string `json:"spanId"` - ParentID string `json:"parentId,omitempty"` - Service string `json:"service"` - Operation string `json:"operation"` - Kind string `json:"kind,omitempty"` - StartedAt time.Time `json:"startedAt"` - DurationMs float64 `json:"durationMs"` - Status string `json:"status"` - Attributes map[string]any `json:"attributes,omitempty"` - Resource map[string]any `json:"resource,omitempty"` - OTLP json.RawMessage `json:"otlp,omitempty"` -} - -type Request struct { - Source string `json:"source" clicky:"title=Source,required"` - Run string `json:"run,omitempty" clicky:"title=Run,desc=Optional recorded run ID"` - Index string `json:"index,omitempty" clicky:"title=Index,desc=OpenSearch index pattern (default otel-traces-*)"` - Format DocumentFormat `json:"format,omitempty" clicky:"title=Document format"` - Services Modes `json:"services,omitempty" clicky:"title=Services,desc=Services a trace's spans run in: each included or excluded"` - Operations Modes `json:"operations,omitempty" clicky:"title=Root operations,desc=A trace's root operation: each included or excluded"` - Statuses Modes `json:"statuses,omitempty" clicky:"title=Statuses,desc=error (a span failed) or ok: each included or excluded"` - Attributes Modes `json:"attributes,omitempty" clicky:"title=Attributes,desc=Span or resource attributes of any span: key, key=value or key=prefix*, each included or excluded"` - MinSpans int `json:"minSpans,omitempty" clicky:"title=Minimum spans,desc=Traces of at least this many spans"` - TraceID string `json:"traceId,omitempty" clicky:"title=Trace ID"` - Lookback string `json:"lookback,omitempty" clicky:"title=Lookback,desc=Duration such as 15m or 24h; used when From is empty"` - From string `json:"from,omitempty" clicky:"title=From,desc=Inclusive RFC3339 timestamp"` - Until string `json:"until,omitempty" clicky:"title=Until,desc=Exclusive RFC3339 timestamp"` - MinDuration string `json:"minDuration,omitempty" clicky:"title=Minimum trace duration,desc=Duration such as 100ms or 2s"` - MaxDuration string `json:"maxDuration,omitempty" clicky:"title=Maximum trace duration,desc=Duration such as 100ms or 2s"` - Limit int `json:"limit,omitempty" clicky:"title=Trace limit,desc=Maximum traces (1 to 1000); spans of a trace stay together"` -} - -type Source struct { - ID string `json:"id"` - Name string `json:"name"` - Type string `json:"type"` -} -type Summary struct { - TraceID string `json:"traceId"` - Operation string `json:"operation"` - Service string `json:"service"` - StartedAt time.Time `json:"startedAt"` - DurationMs float64 `json:"durationMs"` - SpanCount int `json:"spanCount"` - Errors int `json:"errors"` - Services []string `json:"services"` -} -type SearchResult struct { - Traces []Summary `json:"traces"` - Total int `json:"total"` - // Services are those of every trace in the window. - Services []string `json:"services"` - // RootOperations count the traces in the window that pass every filter - // but the operation and span-count ones, by root operation. - RootOperations []OperationStats `json:"rootOperations"` - // Attributes suggest values of those traces' root spans, most frequent first. - Attributes []AttributeCount `json:"attributes"` -} - -// OperationStats are the traces of one root operation. -type OperationStats struct { - Operation string `json:"operation"` - Count int `json:"count"` - // SingleSpan is set when every one of them is a single span. - SingleSpan bool `json:"singleSpan,omitempty"` - P50Ms float64 `json:"p50Ms"` - P95Ms float64 `json:"p95Ms"` -} - -// AttributeCount is how many traces' root spans carry key=value. -type AttributeCount struct { - Value string `json:"value"` - Count int `json:"count"` -} -type Detail struct { - Summary Summary `json:"summary"` - Spans []Span `json:"spans"` - MissingParents []string `json:"missingParents"` -} +type Span = shared.Span +type Request = shared.Request +type DocumentFormat = shared.DocumentFormat +type Source = shared.Source +type Summary = shared.Summary +type SearchResult = shared.SearchResult +type Detail = shared.Detail +type Modes = shared.Modes +type Mode = shared.Mode + +const Include = shared.Include +const Exclude = shared.Exclude +const maxSpans = 100000 + +var ErrTraceNotFound = shared.ErrTraceNotFound type Service struct { DataDir string diff --git a/internal/traceui/model.go b/internal/traceui/model.go new file mode 100644 index 0000000..b8aef2d --- /dev/null +++ b/internal/traceui/model.go @@ -0,0 +1,352 @@ +package traceui + +import ( + "context" + "slices" + "time" + + tea "charm.land/bubbletea/v2" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/pkg/traces/read" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/processcapture" +) + +type Options struct { + Refs []read.Ref + Runs []telltale.RunInfo + Sort string + Replay bool +} + +type tab struct { + top *sampledTab + name, file, cursor string + refs []read.Ref + page read.FeedResult + history []string + selected int + paused, detail bool + monitor, output string + watermarks []int64 + following bool + outputOffset int64 + browse bool +} + +type pageMsg struct { + top *read.TopResult + frames []sampleframe.Frame + topMessage string + generation int + page read.FeedResult + monitor, output string + watermarks []int64 + err error +} +type End struct { + Err error + Info *telltale.RunInfo +} +type tickMsg struct{} + +type Model struct { + ctx context.Context + options Options + tabs []tab + active, run, generation, width, height int + ended, loading bool + err error + saved map[int]runState + processes read.ProcessSnapshot + processLoading, showExited bool + processGeneration, processOffset int + processError error +} + +type runState struct { + tabs []tab + active int +} + +func New(ctx context.Context, options Options) Model { + if options.Sort == "" { + options.Sort = "timestamp" + } + m := Model{ctx: ctx, options: options, ended: options.Replay, processLoading: true, saved: map[int]runState{}} + m.buildTabs() + return m +} + +func tick() tea.Cmd { + return tea.Tick(time.Second, func(time.Time) tea.Msg { return tickMsg{} }) +} + +func (m Model) Init() tea.Cmd { return tea.Batch(m.load(), m.loadProcesses(), tick()) } + +func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch value := msg.(type) { + case tea.WindowSizeMsg: + m.width, m.height = value.Width, value.Height + case End: + m.ended = true + if value.Info != nil && len(m.options.Runs) > 0 { + m.options.Runs = slices.Clone(m.options.Runs) + m.options.Runs[m.run] = *value.Info + } + if value.Err != nil { + m.err = value.Err + return m, tea.Quit + } + m.generation++ + m.processGeneration++ + return m, tea.Batch(m.load(), m.loadProcesses()) + case tickMsg: + commands := []tea.Cmd{tick()} + if !m.processLoading && !m.ended { + m.processLoading = true + commands = append(commands, m.loadProcesses()) + } + if m.loading || m.ended || m.tabs[m.active].paused { + return m, tea.Batch(commands...) + } + m.loading = true + return m, tea.Batch(append(commands, m.load())...) + case processMsg: + if value.generation != m.processGeneration { + return m, nil + } + m.processLoading, m.processError = false, value.err + if value.err == nil { + m.processes = value.snapshot + } + case pageMsg: + if value.generation != m.generation { + return m, nil + } + m.loading = false + if value.err != nil { + m.err = value.err + return m, tea.Quit + } + t := &m.tabs[m.active] + if t.top != nil && !t.browse { + if err := t.top.receive(value); err != nil { + m.err = err + return m, tea.Quit + } + } + if !t.paused { + t.page, t.monitor, t.output, t.watermarks = value.page, value.monitor, value.output, value.watermarks + if t.file == "" { + if t.following { + t.selected = max(0, len(t.page.Rows)-1) + } else { + t.selected = min(t.selected, max(0, len(t.page.Rows)-1)) + } + } + } + case tea.KeyPressMsg: + return m.key(value.String()) + } + return m, nil +} + +func (m Model) key(key string) (tea.Model, tea.Cmd) { + t := &m.tabs[m.active] + if t.top != nil && !t.browse { + if model, cmd, handled := m.sampleKey(key); handled { + return model, cmd + } + } + if key == "h" && t.top != nil { + t.browse = false + m.generation++ + return m, m.load() + } + switch key { + case "q", "ctrl+c": + return m, tea.Quit + case "e": + m.showExited = !m.showExited + m.processOffset = 0 + return m, nil + case "alt+up": + m.processOffset = max(0, m.processOffset-1) + return m, nil + case "alt+down": + m.processOffset = min(max(0, len(m.visibleProcesses())-1), m.processOffset+1) + return m, nil + case "tab", "shift+tab": + delta := 1 + if key == "shift+tab" { + delta = -1 + } + m.active = (m.active + delta + len(m.tabs)) % len(m.tabs) + case "[", "]": + if len(m.options.Runs) < 2 { + return m, nil + } + delta := 1 + if key == "[" { + delta = -1 + } + m.saved[m.run] = runState{tabs: m.tabs, active: m.active} + m.run = (m.run + delta + len(m.options.Runs)) % len(m.options.Runs) + if state, ok := m.saved[m.run]; ok { + m.tabs, m.active = state.tabs, state.active + } else { + m.buildTabs() + } + m.processGeneration++ + m.generation++ + m.loading, m.processLoading = true, true + m.processes, m.processError, m.processOffset = read.ProcessSnapshot{}, nil, 0 + return m, tea.Batch(m.load(), m.loadProcesses()) + case "down", "j", "pgdown": + t.following = false + step := 1 + if key == "pgdown" { + step = max(1, m.height-8) + } + t.selected = max(0, min(m.rowCount()-1, t.selected+step)) + return m, nil + case "up", "k", "pgup": + if t.following { + t.selected = max(0, m.rowCount()-1) + t.following = false + } + step := 1 + if key == "pgup" { + step = max(1, m.height-8) + } + t.selected = max(0, t.selected-step) + return m, nil + case "home", "g": + t.following = false + t.selected = 0 + if t.file == "" { + t.browse = true + t.cursor = "" + t.history = nil + break + } + t.outputOffset = 0 + case "end", "G": + t.following = true + t.selected = max(0, m.rowCount()-1) + if t.file == "" { + t.browse = false + t.cursor = "" + t.history = nil + break + } + t.outputOffset = -1 + case "enter": + t.detail = !t.detail + return m, nil + case " ", "space": + t.paused = !t.paused + if t.paused { + return m, nil + } + case "n": + if t.file != "" { + if t.outputOffset < 0 { + t.outputOffset = min(-1, t.outputOffset+1) + } else { + t.outputOffset += (1 << 20) + } + t.selected = 0 + t.following = false + break + } + if t.page.NextCursor == "" { + return m, nil + } + t.history = append(t.history, t.cursor) + t.cursor = t.page.NextCursor + case "p": + if t.file != "" { + if t.outputOffset < 0 { + t.outputOffset-- + } else { + t.outputOffset = max(0, t.outputOffset-(1<<20)) + } + t.selected = 0 + t.following = false + break + } + if len(t.history) == 0 { + return m, nil + } + t.cursor = t.history[len(t.history)-1] + t.history = t.history[:len(t.history)-1] + case "r": + t.cursor = "" + t.history = nil + t.watermarks = nil + case "s": + if m.options.Sort == "seq" { + m.options.Sort = "timestamp" + } else { + m.options.Sort = "seq" + } + for i := range m.tabs { + m.tabs[i].cursor = "" + m.tabs[i].history = nil + } + default: + return m, nil + } + m.generation++ + m.loading = true + return m, m.load() +} + +func (m Model) Err() error { return m.err } + +func (m *Model) buildTabs() { + m.tabs = []tab{{name: "All events", following: !m.options.Replay}} + names := []string{} + for _, ref := range m.options.Refs { + if len(m.options.Runs) > 0 && ref.RunID != m.options.Runs[m.run].RunID { + continue + } + owner, result := ref.Runtime.Kinds.ResultOf(ref.Kind) + if result == "frames" { + continue + } + if owner == processcapture.Kind { + if !slices.Contains(names, owner) { + names = append(names, owner) + } + continue + } + m.tabs[0].refs = append(m.tabs[0].refs, ref) + if !slices.Contains(names, owner) { + names = append(names, owner) + } + } + m.tabs[0].refs = read.EventRefs(m.tabs[0].refs) + slices.Sort(names) + for _, name := range names { + t := tab{name: name, following: !m.options.Replay} + for _, ref := range m.options.Refs { + owner, _ := ref.Runtime.Kinds.ResultOf(ref.Kind) + if owner == name && (len(m.options.Runs) == 0 || ref.RunID == m.options.Runs[m.run].RunID) { + t.refs = append(t.refs, ref) + if plugin, ok := ref.Runtime.Kinds.Get(ref.Kind); ok && tracing.TopOf(plugin) != nil { + if t.top == nil { + t.top = &sampledTab{ref: ref} + } + t.top.refs = append(t.top.refs, ref) + } + } + } + m.tabs = append(m.tabs, t) + } + m.tabs = append(m.tabs, tab{name: "Stdout", file: "stdout", following: true, outputOffset: -1}, tab{name: "Stderr", file: "stderr", following: true, outputOffset: -1}) + m.active = 0 +} diff --git a/internal/traceui/model_test.go b/internal/traceui/model_test.go new file mode 100644 index 0000000..0c259b6 --- /dev/null +++ b/internal/traceui/model_test.go @@ -0,0 +1,79 @@ +package traceui + +import ( + tea "charm.land/bubbletea/v2" + "context" + "github.com/flanksource/telltale/pkg/traces" + "github.com/flanksource/telltale/pkg/traces/read" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/processcapture" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "testing" + "time" +) + +func TestTraceUI(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Trace UI") } + +var _ = Describe("record model", func() { + It("keeps protocol samples in their Top tabs and out of All events", func() { + kinds, err := traces.Kinds() + Expect(err).NotTo(HaveOccurred()) + runtime := &tracing.Runtime{Kinds: kinds} + var refs []read.Ref + for _, kind := range []string{"http", "sql", "jms", "http_top", "sql_top", "sql_top_active", "jms_top"} { + refs = append(refs, read.Ref{Runtime: runtime, TraceRef: tracing.TraceRef{Kind: kind, Stream: kind}}) + } + model := New(context.Background(), Options{Refs: refs}) + Expect(model.tabs[0].refs).To(ConsistOf(HaveField("Kind", "http"), HaveField("Kind", "sql"), HaveField("Kind", "jms"))) + var topKinds []string + for _, tab := range model.tabs { + if tab.top != nil { + topKinds = append(topKinds, tab.name) + } + } + Expect(topKinds).To(ConsistOf("http_top", "sql_top", "jms_top")) + }) + runtime := &tracing.Runtime{Kinds: tracing.NewKinds()} + It("keeps enabled empty kinds visible and uses an alternate screen", func() { + model := New(context.Background(), Options{Refs: []read.Ref{{Runtime: runtime, TraceRef: tracing.TraceRef{Kind: "sql", Stream: "s"}}, {Runtime: runtime, TraceRef: tracing.TraceRef{Kind: "http", Stream: "h"}}}}) + Expect(model.tabs).To(HaveLen(5)) + Expect(model.tabs[0].name).To(Equal("All events")) + Expect(model.tabs[0].refs).To(HaveLen(2)) + Expect(model.options.Sort).To(Equal("timestamp")) + Expect(model.View().AltScreen).To(BeTrue()) + }) + It("preserves selection and pause independently across tabs", func() { + model := New(context.Background(), Options{Refs: []read.Ref{{Runtime: runtime, TraceRef: tracing.TraceRef{Kind: "http", Stream: "h"}}, {Runtime: runtime, TraceRef: tracing.TraceRef{Kind: "sql", Stream: "s"}}}}) + model.tabs[0].selected = 3 + model.tabs[0].paused = true + updated, _ := model.Update(tea.KeyPressMsg{Code: tea.KeyTab}) + model = updated.(Model) + Expect(model.active).To(Equal(1)) + Expect(model.tabs[1].selected).To(BeZero()) + Expect(model.tabs[0].paused).To(BeTrue()) + }) + It("discards responses from a previously selected tab", func() { + model := New(context.Background(), Options{}) + updated, _ := model.Update(pageMsg{generation: -1, output: "stale"}) + Expect(updated.(Model).tabs[0].output).To(BeEmpty()) + }) + It("updates the process header while events are paused and reveals exited rows on demand", func() { + model := New(context.Background(), Options{}) + model.width, model.height = 160, 30 + model.tabs[0].paused = true + cpu, memory, rss := 125.0, 4.0, uint64(2048) + exited := time.Now() + snapshot := read.ProcessSnapshot{Armed: true, Rows: []read.ProcessRow{ + {ProcessRow: processcapture.ProcessRow{Key: "7@new", PID: 7, Name: "worker", CPUPercent: &cpu, MemoryPercent: &memory, RSSBytes: &rss}}, + {ProcessRow: processcapture.ProcessRow{Key: "7@old", PID: 7, Name: "old worker", ExitedAt: &exited}}, + }} + updated, _ := model.Update(processMsg{snapshot: snapshot}) + model = updated.(Model) + Expect(model.tabs[0].paused).To(BeTrue()) + Expect(model.visibleProcesses()).To(HaveLen(1)) + Expect(model.View().Content).To(ContainSubstring("125.0%")) + updated, _ = model.key("e") + Expect(updated.(Model).visibleProcesses()).To(HaveLen(2)) + }) +}) diff --git a/internal/traceui/processes.go b/internal/traceui/processes.go new file mode 100644 index 0000000..7e2eb94 --- /dev/null +++ b/internal/traceui/processes.go @@ -0,0 +1,81 @@ +package traceui + +import ( + "fmt" + "strings" + + tea "charm.land/bubbletea/v2" + "github.com/flanksource/clicky/api" + "github.com/flanksource/telltale/pkg/traces/read" +) + +type processMsg struct { + generation int + snapshot read.ProcessSnapshot + err error +} + +func (m Model) loadProcesses() tea.Cmd { + refs := []read.Ref{} + for _, ref := range m.options.Refs { + if len(m.options.Runs) == 0 || ref.RunID == m.options.Runs[m.run].RunID { + refs = append(refs, ref) + } + } + return func() tea.Msg { + snapshot, err := read.Processes(m.ctx, refs) + return processMsg{generation: m.processGeneration, snapshot: snapshot, err: err} + } +} + +func (m Model) visibleProcesses() []read.ProcessRow { + rows := []read.ProcessRow{} + for _, row := range m.processes.Rows { + if m.showExited || row.ExitedAt == nil { + rows = append(rows, row) + } + } + return rows +} + +func (m Model) processLines(width, height int) []string { + if m.processError != nil { + return []string{fmt.Sprintf("Process header: %v", m.processError)} + } + if !m.processes.Armed { + return nil + } + rows := m.visibleProcesses() + label := fmt.Sprintf("Processes · %d shown · e show exited", len(rows)) + if m.showExited { + label = fmt.Sprintf("Processes · %d shown · e hide exited (last samples)", len(rows)) + } + lines := []string{mutedStyle.Render(label)} + if len(rows) == 0 { + return append(lines, "No live processes · press e to show exited") + } + start := min(m.processOffset, len(rows)-1) + budget := max(1, height/3-1) + for _, row := range rows[start:min(len(rows), start+budget)] { + cpu, ram := "waiting", "waiting" + if row.CPUPercent != nil { + cpu = fmt.Sprintf("%s %.1f%%", processBar(*row.CPUPercent), *row.CPUPercent) + } + if row.RSSBytes != nil && row.MemoryPercent != nil { + ram = fmt.Sprintf("%s %s %.1f%%", processBar(*row.MemoryPercent), api.HumanizeBytes(int64(*row.RSSBytes)).String(), *row.MemoryPercent) + } + name := row.Name + status := "" + if row.ExitedAt != nil { + status = "exited" + } + lines = append(lines, fmt.Sprintf("%6d %-*.*s %-6s CPU %s RAM %s", row.PID, max(8, min(20, width/6)), max(8, min(20, width/6)), name, status, cpu, ram)) + } + return lines +} + +func processBar(percent float64) string { + const width = 6 + filled := max(0, min(width, int(percent/100*width+0.5))) + return "[" + titleStyle.Render(strings.Repeat("█", filled)) + mutedStyle.Render(strings.Repeat("░", width-filled)) + "]" +} diff --git a/internal/traceui/source.go b/internal/traceui/source.go new file mode 100644 index 0000000..8a8bc1b --- /dev/null +++ b/internal/traceui/source.go @@ -0,0 +1,157 @@ +package traceui + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "strings" + + tea "charm.land/bubbletea/v2" + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/pkg/traces/read" +) + +func (m Model) load() tea.Cmd { + m.tabs = slices.Clone(m.tabs) + m.options.Runs = slices.Clone(m.options.Runs) + t := m.tabs[m.active] + return func() tea.Msg { + if t.top != nil && !t.browse { + return m.loadTop(t) + } + result := pageMsg{generation: m.generation} + if t.file != "" { + result.output, result.err = m.readOutput(t.file) + return result + } + refs := slices.Clone(t.refs) + for _, ref := range refs { + _, ok := ref.Runtime.Kinds.Get(ref.Kind) + if !ok { + result.err = fmt.Errorf("unknown trace kind %s", ref.Kind) + return result + } + meta, err := ref.Runtime.Results.Backend.Meta(m.ctx, ref.Stream) + if err != nil { + result.err = err + return result + } + result.watermarks = append(result.watermarks, meta.HighSeq) + } + result.monitor = t.monitor + if !slices.Equal(t.watermarks, result.watermarks) || t.monitor == "" { + result.monitor = "" + for _, ref := range t.refs { + summary, err := ref.Runtime.Summarize(m.ctx, ref.TraceRef) + if err != nil { + result.err = err + return result + } + result.monitor += ref.Stream + " · " + summaryText(summary) + "\n" + } + } + if !m.options.Replay && !t.browse && t.cursor == "" { + for i := range refs { + refs[i].AfterSeq = max(refs[i].AfterSeq, result.watermarks[i]-100) + } + } + result.page, result.err = read.FeedPage(m.ctx, refs, read.FeedQuery{Sort: m.options.Sort, Query: read.Query{Cursor: t.cursor}}) + return result + } +} + +func (m Model) readOutput(channel string) (string, error) { + if len(m.options.Runs) == 0 { + return "Output unavailable: no command recording.", nil + } + run := m.options.Runs[m.run] + if run.Output == nil { + if channel == "stderr" { + return "Separate stderr unavailable in this recording.", nil + } + path := filepath.Join(run.Dir, telltale.OutputFile) + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + return "Output unavailable: process output was not recorded.", nil + } else if err != nil { + return "", err + } + return outputWindow(path, m.tabs[m.active].outputOffset) + } + name := run.Output.Stdout + if channel == "stderr" { + name = run.Output.Stderr + } + if name == "" || filepath.Base(name) != name { + return "", fmt.Errorf("invalid recorded %s path %q", channel, name) + } + return outputWindow(filepath.Join(run.Dir, name), m.tabs[m.active].outputOffset) +} + +func outputWindow(path string, offset int64) (_ string, err error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer func() { err = errors.Join(err, f.Close()) }() + info, err := f.Stat() + if err != nil { + return "", err + } + const limit = 1 << 20 + start := offset + if start < 0 { + start = max(int64(0), info.Size()+offset*limit) + } + start = min(start, max(int64(0), info.Size()-limit)) + if _, err = f.Seek(start, io.SeekStart); err != nil { + return "", err + } + data, err := io.ReadAll(io.LimitReader(f, limit)) + if err != nil { + return "", err + } + text := string(data) + if info.Size() > limit { + text = fmt.Sprintf("[bytes %d..%d of %d · Home/End start/tail · n/p blocks]\n", start, start+int64(len(data)), info.Size()) + text + } + return text, nil +} + +func summaryText(summary map[string]any) string { + keys := make([]string, 0, len(summary)) + for key := range summary { + keys = append(keys, key) + } + slices.Sort(keys) + parts := []string{} + for _, key := range keys { + switch value := summary[key].(type) { + case map[string]any: + parts = append(parts, key+" {"+summaryText(value)+"}") + case map[string]int: + nested := map[string]any{} + for k, v := range value { + nested[k] = v + } + parts = append(parts, key+" {"+summaryText(nested)+"}") + case map[string]int64: + nested := map[string]any{} + for k, v := range value { + nested[k] = v + } + parts = append(parts, key+" {"+summaryText(nested)+"}") + case []map[string]any: + if key != "requests" && key != "queries" { + for _, v := range value { + parts = append(parts, summaryText(v)) + } + } + case string, bool, int, int64, uint64, float64, float32: + parts = append(parts, fmt.Sprintf("%s=%v", key, value)) + } + } + return strings.Join(parts, " · ") +} diff --git a/internal/traceui/source_test.go b/internal/traceui/source_test.go new file mode 100644 index 0000000..0e7f850 --- /dev/null +++ b/internal/traceui/source_test.go @@ -0,0 +1,56 @@ +package traceui + +import ( + "context" + "os" + "path/filepath" + "strings" + + tea "charm.land/bubbletea/v2" + "github.com/charmbracelet/x/ansi" + "github.com/flanksource/telltale" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("trace terminal output", func() { + It("pages large saved output without retaining the whole file", func() { + path := filepath.Join(GinkgoT().TempDir(), "stdout.log") + Expect(os.WriteFile(path, []byte("first\n"+strings.Repeat("x", 2<<20)+"\nlast"), 0600)).To(Succeed()) + first, err := outputWindow(path, 0) + Expect(err).NotTo(HaveOccurred()) + Expect(first).To(ContainSubstring("first\n")) + Expect(len(first)).To(BeNumerically("<", (1<<20)+256)) + last, err := outputWindow(path, -1) + Expect(err).NotTo(HaveOccurred()) + Expect(last).To(HaveSuffix("\nlast")) + }) + It("keeps stdout and stderr separate including partial lines", func() { + dir := GinkgoT().TempDir() + Expect(os.WriteFile(filepath.Join(dir, telltale.StdoutFile), []byte("standard\npartial"), 0600)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(dir, telltale.StderrFile), []byte("diagnostic"), 0600)).To(Succeed()) + m := New(context.Background(), Options{Runs: []telltale.RunInfo{{Dir: dir, Output: &telltale.OutputRefs{Stdout: telltale.StdoutFile, Stderr: telltale.StderrFile}}}}) + out, err := m.readOutput("stdout") + Expect(err).NotTo(HaveOccurred()) + Expect(out).To(Equal("standard\npartial")) + errout, err := m.readOutput("stderr") + Expect(err).NotTo(HaveOccurred()) + Expect(errout).To(Equal("diagnostic")) + }) + It("rejects a missing declared output file", func() { + m := New(context.Background(), Options{Runs: []telltale.RunInfo{{Dir: GinkgoT().TempDir(), Output: &telltale.OutputRefs{Stdout: telltale.StdoutFile}}}}) + _, err := m.readOutput("stdout") + Expect(err).To(HaveOccurred()) + }) + It("bounds every rendered line and keeps the footer on small terminals", func() { + m := New(context.Background(), Options{}) + updated, _ := m.Update(tea.WindowSizeMsg{Width: 40, Height: 8}) + m = updated.(Model) + m.tabs[0].output = strings.Repeat("long output ", 100) + lines := strings.Split(m.View().Content, "\n") + Expect(lines).To(HaveLen(8)) + for _, line := range lines { + Expect(ansi.StringWidth(line)).To(BeNumerically("<=", 40)) + } + }) +}) diff --git a/internal/traceui/top.go b/internal/traceui/top.go new file mode 100644 index 0000000..d75e7e3 --- /dev/null +++ b/internal/traceui/top.go @@ -0,0 +1,281 @@ +package traceui + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + + tea "charm.land/bubbletea/v2" + "github.com/charmbracelet/x/ansi" + "github.com/flanksource/clicky/tui/table" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/flanksource/telltale/pkg/traces/read" + "github.com/flanksource/telltale/pkg/tracing" +) + +type sampledTab struct { + refs []read.Ref + source, metric string + ref read.Ref + frame string + frames []sampleframe.Frame + result *read.TopResult + table table.Model + search, filter, typing, sort string + desc, help bool + message string + linked bool +} + +func (m Model) loadTop(t tab) pageMsg { + result := pageMsg{generation: m.generation} + q := read.TopQuery{Frame: t.top.frame, Source: t.top.source, Metric: t.top.metric, Query: read.Query{All: true, Search: t.top.search, Sort: t.top.sort, Desc: t.top.desc}} + if t.top.filter != "" { + q.Filters = []string{t.top.filter} + } + top, err := read.Top(m.ctx, t.top.ref, q) + if err != nil { + result.topMessage = err.Error() + if !errors.Is(err, read.ErrNoCompleteSamples) { + result.topMessage = "Cannot read sample: " + err.Error() + } + return result + } + result.top = &top + result.frames, result.err = read.Frames(m.ctx, t.top.ref) + return result +} + +func (t *sampledTab) receive(result pageMsg) error { + t.message = result.topMessage + if result.top == nil { + return nil + } + plugin, _ := t.ref.Runtime.Kinds.Get(t.ref.Kind) + spec := tracing.TopOf(plugin) + ids := make([]string, len(result.top.Rows)) + for i, row := range result.top.Rows { + values := []any{result.top.Frame.Source} + for _, key := range spec.Identity { + values = append(values, row[key]) + } + if len(spec.Identity) == 0 { + values = append(values, row["seq"]) + } + data, err := json.Marshal(values) + if err != nil { + return err + } + ids[i] = string(data) + } + if err := t.table.Set(result.top.Display, ids); err != nil { + return err + } + t.result, t.frames = result.top, result.frames + if t.linked && result.top.Count == 0 { + t.message = "Referenced blocker was omitted from this completed sample." + } + t.linked = false + return nil +} + +func (m Model) sampleKey(key string) (tea.Model, tea.Cmd, bool) { + t := &m.tabs[m.active] + s := t.top + if s.typing != "" { + value := &s.search + if s.typing == "filter" { + value = &s.filter + } + switch key { + case "enter": + s.typing = "" + case "esc": + s.typing = "" + *value = "" + case "backspace": + runes := []rune(*value) + if len(runes) > 0 { + *value = string(runes[:len(runes)-1]) + } + return m, nil, true + default: + if key == "space" || key == " " { + *value += " " + } else if len([]rune(key)) == 1 { + *value += key + } + return m, nil, true + } + } else { + switch key { + case "b": + plugin, _ := s.ref.Runtime.Kinds.Get(s.ref.Kind) + relation := tracing.TopOf(plugin).Relation + index := s.table.Index() + if relation == nil || s.result == nil || index < 0 { + return m, nil, true + } + id := s.result.Rows[index][relation.Column] + if id == nil || fmt.Sprint(id) == "0" { + s.message = "Selected row has no blocker." + return m, nil, true + } + s.frame = s.result.Selection + s.search = "" + s.filter = relation.Target + "=" + fmt.Sprint(id) + s.table.Detail = false + t.paused = true + s.linked = true + case "c": + if len(s.refs) > 1 { + index := slices.IndexFunc(s.refs, func(ref read.Ref) bool { return ref.Stream == s.ref.Stream }) + s.ref = s.refs[(index+1)%len(s.refs)] + s.frame, s.source, s.metric = "", "", "" + s.table.Detail = false + } + case "o": + sources := []string{""} + for _, frame := range s.frames { + if !slices.Contains(sources, frame.Source) { + sources = append(sources, frame.Source) + } + } + s.source = sources[(slices.Index(sources, s.source)+1)%len(sources)] + s.frame = "" + s.table.Detail = false + case "m": + if s.result != nil && len(s.result.Metrics) > 0 { + metrics := s.result.Metrics + index := slices.IndexFunc(metrics, func(metric tracing.TopMetric) bool { return metric.Key == s.metric }) + s.metric = metrics[(index+1)%len(metrics)].Key + s.table.Detail = false + } + case "/": + s.typing = "search" + return m, nil, true + case "f": + s.typing = "filter" + return m, nil, true + case "?": + s.help = !s.help + return m, nil, true + case "h": + t.browse = true + case "s": + keys := []string{} + for _, column := range s.table.Table.Columns { + if column.SortKey != "" { + keys = append(keys, column.SortKey) + } + } + if len(keys) > 0 { + s.sort = keys[(slices.Index(keys, s.sort)+1)%len(keys)] + s.desc = true + } + case "S": + s.desc = !s.desc + case "n", "p": + frames := slices.DeleteFunc(slices.Clone(s.frames), func(frame sampleframe.Frame) bool { return s.source != "" && frame.Source != s.source }) + if len(frames) == 0 { + return m, nil, true + } + index := slices.IndexFunc(frames, func(f sampleframe.Frame) bool { return f.ID == s.frame }) + if index < 0 { + index = len(frames) - 1 + } + delta := 1 + if key == "p" { + delta = -1 + } + s.frame = frames[max(0, min(len(frames)-1, index+delta))].ID + s.table.Detail = false + t.paused = true + case "r": + s.frame, t.paused, s.table.Detail = "", false, false + case "space", " ": + if t.paused { + s.frame = "" + t.paused = false + s.table.Detail = false + } else { + t.paused = true + if s.result != nil { + s.frame = s.result.Selection + } + } + default: + if !s.table.Key(key) { + return m, nil, false + } + if s.table.Detail && s.result != nil { + t.paused = true + s.frame = s.result.Selection + m.generation++ + m.loading = false + } + return m, nil, true + } + } + m.generation++ + m.loading = true + return m, m.load(), true +} + +func (m Model) sampleView() tea.View { + t := m.tabs[m.active] + s := t.top + width, height := max(1, m.width), max(1, m.height) + mode := "live" + if m.options.Replay { + mode = "recorded" + } + if t.paused || s.frame != "" { + mode = "paused" + } + lines := []string{fmt.Sprintf("Trace · %s · Top · %s", t.name, mode)} + names := []string{} + for i, item := range m.tabs { + name := item.name + if i == m.active { + name = "[" + name + "]" + } + names = append(names, name) + } + lines = append(lines, strings.Join(names, " · ")) + if s.result != nil { + f := s.result.Frame + limit := "unlimited" + if f.Limit > 0 { + limit = fmt.Sprint(f.Limit) + } + if active, ok := f.Summary["activePlanLimit"]; ok { + limit = fmt.Sprintf("%v active + evictions", active) + } + lines = append(lines, fmt.Sprintf("%s · %s · %d/%d rows · limit %s", f.Source, f.CompletedAt.Local().Format("15:04:05"), f.Retained, f.Observed, limit)) + } + lines = append(lines, fmt.Sprintf("Search: %s · Filter: %s · Sort: %s · Metric: %s", s.search, s.filter, s.sort, s.metric)) + if s.typing != "" { + lines = append(lines, "Editing "+s.typing+" · enter apply · esc clear") + } + if s.message != "" { + lines = append(lines, s.message) + } + if s.help { + lines = append(lines, "↑↓ move / scroll detail · enter inspect (freezes sample) · b blocker", "/ search · f native filter: column=expression · s sort · S reverse", "m metric · o source · c capture · p/n previous/next sample", "space pause/resume · h history · r resume live · q quit · ? close help") + } else { + lines = append(lines, strings.Split(s.table.View(width, max(1, height-len(lines)-1)), "\n")...) + } + for len(lines) < height-1 { + lines = append(lines, "") + } + lines = append(lines, "tab kind · / search · f filter · enter detail · h history · ? help") + for i, line := range lines { + lines[i] = ansi.Truncate(line, width, "") + } + view := tea.NewView(strings.Join(lines[:min(len(lines), height)], "\n")) + view.AltScreen = true + return view +} diff --git a/internal/traceui/top_test.go b/internal/traceui/top_test.go new file mode 100644 index 0000000..b537075 --- /dev/null +++ b/internal/traceui/top_test.go @@ -0,0 +1,29 @@ +package traceui + +import ( + "context" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/flanksource/telltale/pkg/traces/read" + "github.com/flanksource/telltale/pkg/tracing" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("sample navigation", func() { + It("keeps historical navigation in the chosen source and freezes its sample", func() { + sampled := &sampledTab{source: "first", frame: "a2", frames: []sampleframe.Frame{{ID: "a1", Source: "first"}, {ID: "b1", Source: "second"}, {ID: "a2", Source: "first"}}} + model := Model{ctx: context.Background(), tabs: []tab{{top: sampled}}} + next, _, handled := model.sampleKey("p") + Expect(handled).To(BeTrue()) + Expect(next.(Model).tabs[0].top.frame).To(Equal("a1")) + Expect(next.(Model).tabs[0].paused).To(BeTrue()) + }) + It("cycles available metrics and keeps detail closed when changing the metric", func() { + sampled := &sampledTab{metric: "cpu", result: &read.TopResult{Metrics: []tracing.TopMetric{{Key: "cpu"}, {Key: "memory"}}}} + sampled.table.Detail = true + model := Model{ctx: context.Background(), tabs: []tab{{top: sampled}}} + next, _, _ := model.sampleKey("m") + Expect(next.(Model).tabs[0].top.metric).To(Equal("memory")) + Expect(next.(Model).tabs[0].top.table.Detail).To(BeFalse()) + }) +}) diff --git a/internal/traceui/view.go b/internal/traceui/view.go new file mode 100644 index 0000000..f31f556 --- /dev/null +++ b/internal/traceui/view.go @@ -0,0 +1,120 @@ +package traceui + +import ( + "fmt" + "strings" + "time" + + tea "charm.land/bubbletea/v2" + "github.com/charmbracelet/x/ansi" + "github.com/flanksource/clicky" + "github.com/flanksource/telltale/internal/terminalui" +) + +var titleStyle = terminalui.Title +var mutedStyle = terminalui.Muted +var selectedStyle = terminalui.Selected + +func (m Model) rowCount() int { + t := m.tabs[m.active] + if t.file != "" { + return len(strings.Split(t.output, "\n")) + } + return len(t.page.Rows) +} + +func (m Model) View() tea.View { + if m.tabs[m.active].top != nil && !m.tabs[m.active].browse { + return m.sampleView() + } + width, height := max(1, m.width), max(1, m.height) + t := m.tabs[m.active] + status := "Following" + if m.ended { + status = "Capture complete" + } + if t.paused { + status = "Paused" + } + title := "Trace · " + status + if len(m.options.Runs) > 0 { + r := m.options.Runs[m.run] + elapsed := time.Since(r.StartedAt) + if !r.EndedAt.IsZero() { + elapsed = r.EndedAt.Sub(r.StartedAt) + } + title = fmt.Sprintf("%s · %s · %s · %s", title, r.RunID, elapsed.Round(time.Second), strings.Join(r.Command, " ")) + } + names := []string{} + for i, tab := range m.tabs { + name := " " + tab.name + " " + if i == m.active { + name = selectedStyle.Render(name) + } + names = append(names, name) + } + tabLine := strings.Join(names, " ") + if ansi.StringWidth(tabLine) > width { + tabLine = "‹ " + strings.Join(names[m.active:], " ") + " ›" + } + lines := []string{titleStyle.Render(title), tabLine} + lines = append(lines, m.processLines(width, height)...) + monitor := strings.TrimSpace(t.monitor) + if monitor == "" { + monitor = "Waiting for " + t.name + "…" + } + if t.file != "" { + monitor = t.name + " · " + status + } + monitorLines := strings.Split(ansi.Wrap(monitor, width, ""), "\n") + lines = append(lines, monitorLines[:min(len(monitorLines), max(1, height/4))]...) + body := []string{} + if t.file != "" { + body = strings.Split(ansi.Strip(t.output), "\n") + } else { + for i, row := range t.page.Rows { + text, err := clicky.Format(row.Pretty(), clicky.FormatOptions{Format: "pretty"}) + if err != nil { + text = err.Error() + } + text = strings.ReplaceAll(text, "\n", " ") + if i == t.selected { + text = selectedStyle.Render(text) + } + body = append(body, text) + } + } + available := max(0, height-len(lines)-1) + detail := []string{} + if t.detail && t.file == "" && len(t.page.Rows) > 0 { + text, err := clicky.Format(t.page.Rows[t.selected].PrettyFull(), clicky.FormatOptions{Format: "pretty"}) + if err != nil { + text = err.Error() + } + detail = strings.Split(text, "\n") + detail = detail[:min(len(detail), available/2)] + available -= len(detail) + } + start := max(0, t.selected-available+1) + if t.file != "" && t.following { + start = max(0, len(body)-available) + } + for i := start; i < min(len(body), start+available); i++ { + lines = append(lines, body[i]) + } + for len(lines) < height-1-len(detail) { + lines = append(lines, "") + } + lines = append(lines, detail...) + footer := "q quit · tab kind · ↑↓ events · e exited · alt+↑↓ processes · space pause · n/p page · enter detail · s sort · [/] run" + lines = append(lines, mutedStyle.Render(footer)) + if len(lines) > height { + lines = append(lines[:height-1], mutedStyle.Render(footer)) + } + for i, line := range lines { + lines[i] = ansi.Truncate(line, width, "") + } + view := tea.NewView(strings.Join(lines, "\n")) + view.AltScreen = true + return view +} diff --git a/internal/traceui/view_test.go b/internal/traceui/view_test.go new file mode 100644 index 0000000..5ca4bc0 --- /dev/null +++ b/internal/traceui/view_test.go @@ -0,0 +1,57 @@ +package traceui + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + + "github.com/charmbracelet/x/ansi" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/pkg/traces/read" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("styled trace view", func() { + It("keeps HTTP pretty rows when the kind also has schema columns", func() { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + defer GinkgoRecover() + _, err := w.Write([]byte("recorded orders")) + Expect(err).NotTo(HaveOccurred()) + })) + DeferCleanup(server.Close) + dataDir := GinkgoT().TempDir() + run, err := telltale.Start(context.Background(), telltale.StartOptions{DataDir: dataDir, + Command: []string{"curl", "--noproxy", "", "-sS", server.URL + "/orders"}, + Traces: []telltale.TraceSpec{{Kind: "http", Params: json.RawMessage(`{"target":{"seam":"proxy"}}`)}}}) + Expect(err).NotTo(HaveOccurred()) + _, err = run.Wait() + Expect(err).NotTo(HaveOccurred()) + selected, err := read.Resolve("http", read.Scope{DataDir: dataDir, Runs: []string{run.ID()}}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(selected.Close) + model := New(context.Background(), Options{Refs: selected.Refs, Replay: true}) + model.width, model.height = 180, 20 + loaded := model.load()().(pageMsg) + Expect(loaded.err).NotTo(HaveOccurred()) + Expect(loaded.page.Rows).To(HaveLen(1)) + updated, _ := model.Update(loaded) + text := strings.Join(strings.Fields(ansi.Strip(updated.(Model).View().Content)), " ") + Expect(text).To(ContainSubstring("http #1 GET")) + Expect(text).To(ContainSubstring(server.URL + "/orders")) + }) + + It("keeps terminal event rows visible alongside snapshots", func() { + model := New(context.Background(), Options{Replay: true}) + model.width, model.height = 100, 20 + model.tabs = []tab{{name: "terminal", page: read.FeedResult{Rows: []read.Record{ + {Kind: "terminal", Seq: 1, Record: query.Row{"kind": "r", "width": 120, "height": 40}}, + {Kind: "terminal", Seq: 2, Record: query.Row{"kind": "snapshot", "data": "recorded terminal screen"}}, + }}}} + text := strings.Join(strings.Fields(ansi.Strip(model.View().Content)), " ") + Expect(text).To(And(ContainSubstring("terminal #1"), ContainSubstring("terminal #2"))) + }) +}) diff --git a/list.go b/list.go index 0491dae..3db4162 100644 --- a/list.go +++ b/list.go @@ -1,6 +1,7 @@ package telltale import ( + "cmp" "errors" "fmt" "os" @@ -11,6 +12,7 @@ import ( "time" "github.com/flanksource/telltale/pkg/traces" + "github.com/flanksource/telltale/pkg/tracing" ) // Status is where a run is. @@ -57,6 +59,7 @@ type RunInfo struct { // Process names the process capture's streams; nil for an unfinished run // and a host run. Process *traces.ProcessRefs `json:"process,omitempty"` + Output *OutputRefs `json:"output,omitempty"` // Live is set while the run is running, with what a client checkpoints. Live *LiveRun `json:"live,omitempty"` // Unavailable says why an unfinished run has nothing to read. @@ -85,11 +88,25 @@ func resultInfo(result *Result) RunInfo { if result.EndedAt.IsZero() { status, exitCode = StatusRunning, nil } - return RunInfo{ + info := RunInfo{ RunID: result.RunID, Dir: result.Dir, Status: status, Command: result.Command, PID: result.PID, StartedAt: result.StartedAt, - EndedAt: result.EndedAt, ExitCode: exitCode, Captures: result.Captures, Process: result.Process, + EndedAt: result.EndedAt, ExitCode: exitCode, Captures: result.Captures, Process: result.Process, Output: result.Output, CheckpointErrors: result.CheckpointErrors, Definition: result.Definition, Error: result.Error, } + refs := []tracing.TraceRef{} + for _, capture := range info.Captures { + refs = append(refs, capture.Ref) + } + if info.Process != nil { + refs = append(refs, info.Process.Tree, info.Process.Samples) + } + for _, ref := range refs { + if slices.Contains([]string{"process_tree", "process_samples", "process_groups"}, ref.Kind) { + info.Unavailable = "This recording uses the retired " + ref.Kind + " format. Open it with the build that recorded it; this build records the unified process format." + break + } + } + return info } // GetOptions name one run. @@ -193,9 +210,16 @@ func listRuns(options ListOptions) ([]RunInfo, error) { if !options.matches(info) { continue } - if runs = append(runs, info); options.Limit > 0 && len(runs) == options.Limit { - break + runs = append(runs, info) + } + slices.SortFunc(runs, func(a, b RunInfo) int { + if compared := b.StartedAt.Compare(a.StartedAt); compared != 0 { + return compared } + return cmp.Compare(b.RunID, a.RunID) + }) + if options.Limit > 0 && len(runs) > options.Limit { + runs = runs[:options.Limit] } return runs, nil } @@ -210,7 +234,7 @@ func (o ListOptions) matches(info RunInfo) bool { if len(o.Status) > 0 && !slices.Contains(o.Status, info.Status) { return false } - if o.Kind != "" && !slices.ContainsFunc(info.Captures, func(c CaptureResult) bool { return c.Kind == o.Kind }) { + if o.Kind != "" && (info.Process == nil || info.Process.Tree.Kind != o.Kind) && !slices.ContainsFunc(info.Captures, func(c CaptureResult) bool { return c.Kind == o.Kind }) { return false } return o.Since.IsZero() || info.Standing || (!info.StartedAt.IsZero() && !info.StartedAt.Before(o.Since)) diff --git a/list_test.go b/list_test.go index 712584a..053eb9d 100644 --- a/list_test.go +++ b/list_test.go @@ -2,6 +2,7 @@ package telltale_test import ( "context" + "encoding/json" "os" "path/filepath" "time" @@ -14,6 +15,20 @@ import ( ) var _ = Describe("listing runs", func() { + It("orders mixed hostnames and old IDs by start time before applying the limit", func() { + dir := GinkgoT().TempDir() + names := []string{"zworker20261105T000000Z", "run-2026-11-06T00-00-00Z", "aworker20261107T000000Z"} + for index, name := range names { + store := traces.StoreDir(dir, name) + Expect(os.MkdirAll(store, 0o755)).To(Succeed()) + manifest, err := json.Marshal(telltale.Result{RunID: name, Dir: store, StartedAt: time.Date(2026, 11, 5+index, 0, 0, 0, 0, time.UTC)}) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(store, telltale.ManifestFile), manifest, 0o600)).To(Succeed()) + } + runs, err := telltale.List(telltale.ListOptions{DataDir: dir, Limit: 2}) + Expect(err).NotTo(HaveOccurred()) + Expect(runs).To(HaveExactElements(HaveField("RunID", names[2]), HaveField("RunID", names[1]))) + }) var dataDir, succeeded, failed, unfinished string var started time.Time diff --git a/output.go b/output.go new file mode 100644 index 0000000..acf8ca9 --- /dev/null +++ b/output.go @@ -0,0 +1,8 @@ +package telltale + +func (r *Run) outputRefs() *OutputRefs { + if len(r.output) != 2 { + return nil + } + return &OutputRefs{Stdout: StdoutFile, Stderr: StderrFile} +} diff --git a/output_test.go b/output_test.go new file mode 100644 index 0000000..207c4f3 --- /dev/null +++ b/output_test.go @@ -0,0 +1,29 @@ +package telltale_test + +import ( + "context" + "github.com/flanksource/telltale" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "os" + "path/filepath" +) + +var _ = Describe("recorded process output", func() { + It("persists separate streams in running and completed metadata", func() { + run, err := telltale.Start(context.Background(), telltale.StartOptions{DataDir: GinkgoT().TempDir(), Command: []string{"sh", "-c", "printf standard\nprintf diagnostic >&2"}}) + Expect(err).NotTo(HaveOccurred()) + Expect(run.Info().Output).To(Equal(&telltale.OutputRefs{Stdout: telltale.StdoutFile, Stderr: telltale.StderrFile})) + result, err := run.Wait() + Expect(err).NotTo(HaveOccurred()) + Expect(result.ExitCode).To(BeZero()) + for name, want := range map[string]string{telltale.StdoutFile: "standard", telltale.StderrFile: "diagnostic"} { + data, err := os.ReadFile(filepath.Join(run.Dir(), name)) + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).To(Equal(want)) + } + info, err := telltale.Get(telltale.GetOptions{DataDir: filepath.Dir(filepath.Dir(run.Dir())), RunID: run.ID()}) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Output).To(Equal(result.Output)) + }) +}) diff --git a/packages/api/test/agent-http.spec.ts b/packages/api/test/agent-http.spec.ts new file mode 100644 index 0000000..f6622ee --- /dev/null +++ b/packages/api/test/agent-http.spec.ts @@ -0,0 +1,62 @@ +import { execFile, spawn } from "node:child_process"; +import { generateKeyPairSync, randomBytes, sign } from "node:crypto"; +import { once } from "node:events"; +import { copyFile, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { expect, it } from "vitest"; + +const run = promisify(execFile); + +it("keeps the JVM identity while upgrading the runtime behind its HTTP API", async () => { + const dir = await mkdtemp(join(process.cwd(), ".tmp", "agent-http-")); + const { privateKey, publicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }); + const issuer = "http-fixture"; + const audience = "http-fixture-jvm"; + const kid = "fixture-key"; + const encode = (value: unknown) => Buffer.from(JSON.stringify(value)).toString("base64url"); + const unsigned = `${encode({ alg: "RS256", typ: "JWT", kid })}.${encode({ iss: issuer, aud: audience, sub: "fixture", scope: "jvm:read", exp: Math.floor(Date.now() / 1000) + 60 })}`; + const token = `${unsigned}.${sign("RSA-SHA256", Buffer.from(unsigned), privateKey).toString("base64url")}`; + await run("javac", ["-d", dir, "pkg/agentinstall/testdata/Workload.java"]); + const config = join(dir, "agent.json"); + await writeFile(config, JSON.stringify({ + token: randomBytes(32).toString("hex"), service: "http-fixture", allowMutation: false, + resource: { "deployment.environment.name": "integration" }, + http: { bind: "127.0.0.1", port: 0, portFile: join(dir, "port"), issuer, audience, jwks: { keys: [{ ...publicKey.export({ format: "jwk" }), kid, alg: "RS256", use: "sig" }] } }, + })); + const child = spawn("java", [`-javaagent:${join(process.cwd(), "pkg/agentinstall/java/assets/telltale-agent.jar")}=${config}`, "-cp", dir, "Workload"]); + let output = ""; + child.stderr.on("data", (chunk: Buffer) => { output += chunk.toString(); }); + try { + await expect.poll(async () => (await readdir(dir)).includes("port"), { timeout: 10000, message: output }).toBe(true); + const base = `http://127.0.0.1:${await readFile(join(dir, "port"), "utf8")}`; + const headers = { "X-Telltale-Authorization": `Bearer ${token}` }; + const response = await fetch(`${base}/hello`, { headers }); + expect(response.status, await response.clone().text()).toBe(200); + const hello = await response.json() as { resource: Record }; + expect(hello.resource).toEqual({ + "service.name": "http-fixture", "process.pid": String(child.pid), + "deployment.environment.name": "integration", "telltale.source.instance": expect.any(String), + }); + expect(hello.resource["telltale.source.instance"]).toMatch(/^[0-9a-f-]{36}$/); + const again = await fetch(`${base}/hello`, { headers }); + expect(again.status).toBe(200); + expect(await again.json()).toMatchObject({ resource: hello.resource }); + const upgraded = join(dir, "upgraded.jar"); + const manifest = join(dir, "upgrade.mf"); + await copyFile("pkg/agentinstall/java/assets/telltale-agent.jar", upgraded); + await writeFile(manifest, "Implementation-Version: api-upgrade-fixture\n\n"); + await run("jar", ["--update", "--file", upgraded, "--manifest", manifest]); + await run("java", ["-cp", upgraded, "com.flanksource.telltale.Attach", String(child.pid), upgraded, config, "true"]); + const replacedBase = `http://127.0.0.1:${await readFile(join(dir, "port"), "utf8")}`; + const health = await fetch(`${replacedBase}/health`); + expect(health.status).toBe(200); + expect(await health.json()).toMatchObject({ status: "ok", runtimeVersion: "api-upgrade-fixture", upgradeApiVersion: 1 }); + const replaced = await fetch(`${replacedBase}/hello`, { headers }); + expect(replaced.status).toBe(200); + expect(await replaced.json()).toMatchObject({ sdkVersion: "api-upgrade-fixture", resource: hello.resource }); + } finally { + if (child.exitCode === null) { const stopped = once(child, "exit"); child.kill("SIGTERM"); await stopped; } + await rm(dir, { recursive: true, force: true }); + } +}, 30000); diff --git a/packages/api/test/agents.spec.ts b/packages/api/test/agents.spec.ts index 3740ea0..c92ceef 100644 --- a/packages/api/test/agents.spec.ts +++ b/packages/api/test/agents.spec.ts @@ -1,7 +1,7 @@ import { execFile, spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; import { randomBytes } from "node:crypto"; import { once } from "node:events"; -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { connect } from "node:net"; import { join } from "node:path"; import { promisify } from "node:util"; @@ -42,6 +42,105 @@ afterAll(async () => { }); describe("agents API", () => { + it("mirrors saved deployed inventory between the API and CLI without connecting targets", async () => { + const directory = join(testDir, "config", "telltale", "deployers"); + const file = join(directory, "fixture-orders.yaml"); + const target = "deployer/fixture-orders/fixture-ns/orders-0"; + const endpoint = "http://127.0.0.1:1"; + await mkdir(directory, { recursive: true }); + await writeFile(file, JSON.stringify({ + apiVersion: "telltale.flanksource.com/v1alpha1", kind: "Deployer", metadata: { name: "fixture-orders" }, + spec: { + selector: { kind: "Pod", namespace: "fixture-ns", name: "orders-0" }, + javaContainer: { kind: "attach", url: endpoint, directory: "/tmp/telltale", bind: "127.0.0.1", port: 7070, service: "orders" }, + auth: { issuer: "connection://example/issuer" }, + }, + status: { targets: [{ target, namespace: "fixture-ns", pod: "orders-0", container: "app", state: "deployed", endpoint }] }, + })); + try { + const response = await fetch(`${apiURL}/api/agents`); + expect(response.status).toBe(200); + const inventory = await response.json() as Array>; + const agent = inventory.find(entry => entry.agentId === target); + expect(agent).toMatchObject({ + agentId: target, connected: false, group: "namespace/fixture-ns", + resource: { "k8s.namespace.name": "fixture-ns", "k8s.pod.name": "orders-0", "k8s.container.name": "app" }, + known: { agentId: target, runtime: "java", state: "deployed", endpoint }, + }); + expect(agent).not.toHaveProperty("runtime"); + expect(agent).not.toHaveProperty("started"); + expect(agent).not.toHaveProperty("probes"); + const cli = await run(".bin/telltale", ["agents", "list", "--server", apiURL, "--format", "json"], { env: env() }); + const groups = JSON.parse(cli.stdout) as Record; + expect(groups["namespace/fixture-ns"]).toContainEqual(agent); + } finally { + await rm(file); + } + }); + + it("serves the live JVM catalog, preserves script integers and hot-loads an extension", async () => { + await run("javac", ["-d", testDir, "pkg/agentinstall/testdata/Workload.java"]); + const config = join(testDir, "java-agent.json"); + await writeFile(config, JSON.stringify({ url: apiURL, token, service: "api-jvm", allowMutation: true, extensionsDir: join(testDir, "extensions") })); + const child = spawn("java", [`-javaagent:${join(process.cwd(), "pkg/agentinstall/java/assets/telltale-agent.jar")}=${config}`, "-cp", testDir, "Workload"]); + let output = ""; + child.stderr.on("data", (chunk: Buffer) => { output += chunk.toString(); }); + try { + let id: string | undefined; + for (let attempt = 0; attempt < 100; attempt++) { + const agents = await (await fetch(`${apiURL}/api/agents`)).json() as Array<{ agentId: string; service: string }>; + id = agents.find(agent => agent.service === "api-jvm")?.agentId; + if (id || child.exitCode !== null) break; + await new Promise(resolve => setTimeout(resolve, 100)); + } + expect(id, output).toBeDefined(); + const snapshot = await fetch(`${apiURL}/api/v1/jvm/snapshot`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ agent: id, jfr: "1s", heapHistogram: true }) }); + expect(snapshot.status).toBe(200); + expect(snapshot.headers.get("content-type")).toBe("application/zip"); + expect(snapshot.headers.get("x-telltale-snapshot-status")).toBe("complete"); + const bundle = Buffer.from(await snapshot.arrayBuffer()); + expect(bundle.subarray(0, 2).toString()).toBe("PK"); + expect(bundle.includes(Buffer.from("manifest.json"))).toBe(true); + expect(bundle.includes(Buffer.from("recording.jfr"))).toBe(true); + const base = `${apiURL}/api/agents/${encodeURIComponent(id!)}`; + const catalog = await (await fetch(`${base}/catalog`)).json() as Array<{ name: string; paramsSchema: unknown }>; + expect(catalog).toContainEqual(expect.objectContaining({ name: "jvm.groovy.eval", paramsSchema: expect.objectContaining({ required: ["source"] }) })); + const execute = (body: string) => fetch(`${base}/call`, { method: "POST", headers: { "Content-Type": "application/json" }, body }); + const denied = await execute('{"operation":"jvm.groovy.eval","params":{"source":"return 7"}}'); + expect(denied.status).toBe(400); + const integer = await execute('{"operation":"jvm.groovy.eval","allowMutation":true,"params":{"source":"return args.id","args":{"id":9007199254740993}}}'); + expect(integer.status).toBe(200); + expect(await integer.text()).toContain('"value":9007199254740993'); + const invoked = await execute('{"operation":"jvm.jmx.invoke","allowMutation":true,"params":{"objectName":"java.lang:type=Memory","operation":"gc","args":[],"signature":[]}}'); + expect(invoked.status).toBe(200); + expect(await invoked.text()).toBe("null"); + const jar = (await readFile("sdk/java/build/libs/telltale-example-extension.jar")).toString("base64"); + expect((await execute(JSON.stringify({ operation: "jvm.extensions.install", allowMutation: true, params: { jar } }))).status).toBe(200); + expect(await (await fetch(`${base}/catalog`)).json()).toContainEqual(expect.objectContaining({ name: "extension.example.greet" })); + const greet = await execute(JSON.stringify({ operation: "extension.example.greet", params: { name: "operator" } })); + expect(await greet.json()).toEqual({ message: "Hello operator" }); + expect((await execute(JSON.stringify({ operation: "jvm.extensions.replace", allowMutation: true, params: { jar } }))).status).toBe(200); + expect((await execute('{"operation":"jvm.extensions.unload","allowMutation":true,"params":{"id":"example"}}')).status).toBe(200); + expect(await (await fetch(`${base}/catalog`)).json()).not.toContainEqual(expect.objectContaining({ name: "extension.example.greet" })); + } finally { + if (child.exitCode === null) { const stopped = once(child, "exit"); child.kill("SIGTERM"); await stopped; } + } + }, 30_000); + it("rejects malformed diagnostic calls before selecting a JVM", async () => { + for (const body of ['{}', 'null', '{"agent":"missing","output":"/some/path"}', '{"agent":"missing","jfr":"-1s"}']) { + const response = await fetch(`${apiURL}/api/v1/jvm/snapshot`, { method: "POST", headers: { "Content-Type": "application/json" }, body }); + expect(response.status).toBe(400); + expect(response.headers.get("content-type")).not.toBe("application/zip"); + } + for (const body of ['{}', 'null', '{"params":{}}', '{"operation":"jvm.catalog"} {}', '{"operation":"jvm.catalog","unexpected":true}']) { + const response = await fetch(`${apiURL}/api/agents/missing/call`, { + method: "POST", headers: { "Content-Type": "application/json" }, body, + }); + expect(response.status).toBe(400); + expect(await response.text()).not.toContain("is not connected"); + } + }); + it("requires API scope on non-loopback server routes", async () => { const apiToken = (await run(".bin/telltale", ["agents", "token", "create", "api-test", "--scope", "api"], { env: env() })).stdout.trim(); const publicServer = spawn(".bin/telltale", ["serve", "--host", "0.0.0.0", "--port", "0", "--data-dir", testDir], { env: env() }); @@ -61,6 +160,8 @@ describe("agents API", () => { expect(unauthenticated.status).toBe(401); expect(await unauthenticated.json()).toEqual({ error: "unauthorized" }); expect((await fetch(`${address}/api/traces`, { headers: { Authorization: `Bearer ${token}` } })).status).toBe(403); + expect((await fetch(`${address}/api/v1/jvm/snapshot`, { method: "POST", headers: { "Content-Type": "application/json" }, body: '{"agent":"missing"}' })).status).toBe(401); + expect((await fetch(`${address}/api/v1/jvm/snapshot`, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}` }, body: '{"agent":"missing"}' })).status).toBe(403); expect((await fetch(`${address}/api/traces`, { headers: { Authorization: `Bearer ${apiToken}` } })).status).toBe(200); } finally { if (publicServer.exitCode === null) { @@ -113,7 +214,7 @@ describe("agents API", () => { }); const result = await response.json() as unknown; expect(response.status, JSON.stringify(result)).toBe(201); - expect(result).toMatchObject({ captures: [expect.objectContaining({ kind: "http" })] }); + expect(result).toMatchObject({ captures: [expect.objectContaining({ kind: "http", target: { seam: "agent", selector: { "k8s.pod.name": "api-pod" } } })] }); } finally { child.stdin.end(); if (child.exitCode === null) await once(child, "exit"); diff --git a/packages/api/test/definitions.spec.ts b/packages/api/test/definitions.spec.ts index df2b9ee..52a8187 100644 --- a/packages/api/test/definitions.spec.ts +++ b/packages/api/test/definitions.spec.ts @@ -58,6 +58,58 @@ function put(resourceVersion: string, definition: object): Promise { } describe("trace definitions", () => { + it("validates mixed Prometheus endpoint schemes without connecting to Kubernetes", async () => { + const response = await fetch(`${apiURL}/api/traces/definitions/validate`, { + method: "POST", headers: json, + body: JSON.stringify({ definition: { ...probe, spec: { collect: { prometheus: { scrape: [{ endpoints: [ + "port-forward://metrics-exporter.monitoring:9100/metrics", + "proxy://metrics-exporter.monitoring:9100/metrics", + "https://metrics.example.com/metrics", + ] }] } } } } }), + }); + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ errors: [], entries: [expect.objectContaining({ kind: "prometheus" })] }); + }); + + it("publishes JVM protocol entries and validates agent SQL top without a database connection", async () => { + const protocols = { + http: [{ agent: { id: "java/checkout" }, features: ["http"] }], + httpTop: [{ agent: { id: "java/checkout" }, direction: "both" }], + sql: [{ agent: { id: "java/checkout" } }], + sqlTop: [{ agent: { id: "java/checkout" }, poll: "5s" }], + jms: [{ agent: { id: "java/checkout" }, direction: "incoming" }], + spans: [{ agent: { id: "java/checkout" } }], + }; + const response = await fetch(`${apiURL}/api/traces/definitions/validate`, { + method: "POST", headers: json, + body: JSON.stringify({ definition: { ...probe, spec: { instrument: { jvm: protocols } } } }), + }); + expect(response.status).toBe(200); + const validation = await response.json() as { errors: unknown[]; entries: Entry[] }; + expect(validation.errors).toEqual([]); + expect(validation.entries.map(entry => ({ path: entry.path, kind: entry.kind }))).toEqual([ + { path: "instrument.jvm.http", kind: "http" }, + { path: "instrument.jvm.httpTop", kind: "http_top" }, + { path: "instrument.jvm.sql", kind: "sql" }, + { path: "instrument.jvm.sqlTop", kind: "sql_top" }, + { path: "instrument.jvm.jms", kind: "jms" }, + { path: "instrument.jvm.spans", kind: "spans" }, + ]); + const form = await (await fetch(`${apiURL}/api/traces/start`)).json() as { schema: { properties: { traces: { properties: Record } } } }; + expect(form.schema.properties.traces.properties).toHaveProperty("http_top"); + expect(form.schema.properties.traces.properties).toHaveProperty("jms"); + }); + + it("rejects the Go-only SQL self selector at the JVM definition entry", async () => { + const response = await fetch(`${apiURL}/api/traces/definitions/validate`, { + method: "POST", headers: json, + body: JSON.stringify({ definition: { ...probe, spec: { instrument: { jvm: { sql: [{ agent: { id: "java/checkout" }, connections: ["self"] }] } } } } }), + }); + expect(response.status).toBe(200); + const validation = await response.json() as { errors: { instancePath: string }[] }; + expect(validation.errors).toEqual([expect.objectContaining({ instancePath: "/spec/instrument/jvm/sql/0" })]); + }); + it("creates, lists and starts a definition, refusing a stale write, and records it on the run", async () => { const created = await put("", probe); expect(created.status).toBe(200); diff --git a/packages/api/test/jvm-top.spec.ts b/packages/api/test/jvm-top.spec.ts new file mode 100644 index 0000000..39e142e --- /dev/null +++ b/packages/api/test/jvm-top.spec.ts @@ -0,0 +1,60 @@ +import { execFile, spawn, type ChildProcess } from "node:child_process"; +import { once } from "node:events"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { expect, it } from "vitest"; + +const run = promisify(execFile); + +it("reads live JVM Top through RPC and remote CLI and inspects a sampled thread", async () => { + const dir = await mkdtemp(join(process.cwd(), ".tmp", "jvm-top-")); + const env = { ...process.env, XDG_CONFIG_HOME: join(dir, "config") }; + const children: ChildProcess[] = []; + try { + const token = (await run(".bin/telltale", ["agents", "token", "create", "top-test"], { env })).stdout.trim(); + const apiToken = (await run(".bin/telltale", ["agents", "token", "create", "top-api-test", "--scope", "api"], { env })).stdout.trim(); + const headers = { Authorization: `Bearer ${apiToken}`, "Content-Type": "application/json" }; + const server = spawn(".bin/telltale", ["serve", "--host", "0.0.0.0", "--port", "0", "--data-dir", dir], { env }); + children.push(server); + const address = await new Promise((resolve, reject) => { + let output = ""; + server.stdout.on("data", (data: Buffer) => { output += data.toString(); const match = /Telltale API: (http:\/\/\S+)/.exec(output); if (match) resolve(match[1]!); }); + server.once("error", reject); + server.once("exit", code => reject(new Error(`Server exited ${code}: ${output}`))); + }); + await run("javac", ["-d", dir, "pkg/agentinstall/testdata/Workload.java"]); + const config = join(dir, "agent.json"); + await writeFile(config, JSON.stringify({ url: address, token, service: "jvm-top-test", allowMutation: false })); + const java = spawn("java", [`-javaagent:${join(process.cwd(), "pkg/agentinstall/java/assets/telltale-agent.jar")}=${config}`, "-cp", dir, "Workload"]); + children.push(java); + let id: string | undefined; + for (let attempt = 0; attempt < 100; attempt++) { + const agents = await (await fetch(`${address}/api/agents`, { headers })).json() as Array<{ agentId: string; service: string }>; + id = agents.find(agent => agent.service === "jvm-top-test")?.agentId; + if (id) break; + await new Promise(resolve => setTimeout(resolve, 100)); + } + expect(id).toBeDefined(); + expect((await fetch(`${address}/api/v1/jvm/top`, { method: "POST", body: "{}" })).status).toBe(401); + expect((await fetch(`${address}/api/v1/jvm/top`, { method: "POST", headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, body: "{}" })).status).toBe(403); + const response = await fetch(`${address}/api/v1/jvm/top`, { method: "POST", headers, body: JSON.stringify({ agent: id, format: "json" }) }); + expect(response.status).toBe(200); + const envelope = await response.json() as { success: boolean; output: string }; + expect(envelope.success).toBe(true); + const frame = JSON.parse(envelope.output) as { agentId: string; metrics: { memory: unknown }; threads: { threads: Array<{ id: number }> } }; + expect(frame.agentId).toBe(id); + expect(frame.metrics.memory).toBeDefined(); + expect(frame.threads.threads.length).toBeGreaterThan(0); + const cli = await run(".bin/telltale", ["jvm", "top", "--server", address, "--agent", id!, "--once", "--format", "json"], { env: { ...env, TELLTALE_API_TOKEN: apiToken } }); + expect(JSON.parse(cli.stdout)).toMatchObject({ agentId: id, threads: { cpuTimeEnabled: expect.any(Boolean) } }); + const stack = await fetch(`${address}/api/agents/${encodeURIComponent(id!)}/call`, { method: "POST", headers, body: JSON.stringify({ operation: "jvm.threads.stack", params: { id: frame.threads.threads[0]!.id } }) }); + expect(stack.status).toBe(200); + expect(await stack.json()).toMatchObject({ text: expect.any(String), frames: expect.any(Array) }); + const spec = await (await fetch(`${address}/api/openapi.json`, { headers })).json() as { components: { schemas: Record } }; + expect(spec.components.schemas.JVMTopFrame).toBeDefined(); + } finally { + for (const child of children.reverse()) { if (child.exitCode === null) { const stopped = once(child, "exit"); child.kill("SIGTERM"); await stopped; } } + await rm(dir, { recursive: true, force: true }); + } +}, 60000); diff --git a/packages/api/test/otel.spec.ts b/packages/api/test/otel.spec.ts index 9157ec6..f551cea 100644 --- a/packages/api/test/otel.spec.ts +++ b/packages/api/test/otel.spec.ts @@ -114,9 +114,12 @@ describe("OpenTelemetry search API", () => { expect(await (await fetch(`${apiURL}/api/otel/sources`)).json()).toEqual([ { id: "records", name: "Recorded traces", type: "records" }, ]); - expect( - await (await fetch(`${apiURL}/api/openapi.json`)).json(), - ).toHaveProperty("paths./api/otel"); + const spec = await (await fetch(`${apiURL}/api/openapi.json`)).json(); + expect(spec).toHaveProperty("paths./api/otel"); + expect(spec).toHaveProperty("paths./v1/profiles.post.parameters", [ + expect.objectContaining({ name: "OTLP-Profiles-Development-Version", in: "header", required: true }), + ]); + expect((await fetch(`${apiURL}/v1/profiles`, { method: "POST" })).status).toBe(404); }); it.each([ "limit=1001", diff --git a/packages/api/test/server.spec.ts b/packages/api/test/server.spec.ts index 8977001..fd4c4b4 100644 --- a/packages/api/test/server.spec.ts +++ b/packages/api/test/server.spec.ts @@ -41,11 +41,18 @@ describe("standalone API and embedded UI", () => { expect(await response.json()).toMatchObject({ info: { title: "Telltale" }, openapi: expect.stringMatching(/^3\./) }); }); + it("includes explain in the generated command catalog", async () => { + const response = await fetch(`${apiURL}/api/openapi.json`); + expect(response.status).toBe(200); + const spec = await response.json() as { paths: Record }; + expect(Object.keys(spec.paths).some((path) => path.startsWith('/api/v1/explain'))).toBe(true); + }); + it("derives run arguments and checkpoint choices from kind schemas", async () => { const response = await fetch(`${apiURL}/api/openapi.json`); expect(response.status).toBe(200); const spec = await response.json() as { paths: Record } }> } } }> }; - const http = spec.paths["/api/v1/run/{flags}/http"].post?.requestBody?.content["application/json"].schema.properties; + const http = spec.paths["/api/v1/run/http"].post?.requestBody?.content["application/json"].schema.properties; expect(http).toMatchObject({ mode: { type: "string", enum: ["connect", "mitm"], default: "connect" }, seam: { type: "string", enum: ["self", "agent", "proxy"], default: "self" }, @@ -53,7 +60,7 @@ describe("standalone API and embedded UI", () => { pid: { type: "integer" }, }); expect(http).not.toHaveProperty("format"); - const checkpoint = spec.paths["/api/v1/run/{flags}/goprof"].post?.requestBody?.content["application/json"].schema.properties; + const checkpoint = spec.paths["/api/v1/run/goprof"].post?.requestBody?.content["application/json"].schema.properties; expect(checkpoint).toMatchObject({ profile: { type: "string", enum: ["cpu", "heap", "allocs", "mutex", "block", "goroutine"], default: "heap" }, source: { type: "string", enum: ["pprof", "gops", "file"], default: "gops" }, diff --git a/packages/api/test/shutdown.spec.ts b/packages/api/test/shutdown.spec.ts new file mode 100644 index 0000000..0187ce7 --- /dev/null +++ b/packages/api/test/shutdown.spec.ts @@ -0,0 +1,214 @@ +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { connect } from "node:net"; +import { join, resolve } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +interface Exit { code: number | null; signal: NodeJS.Signals | null } +interface Command { + child: ChildProcessWithoutNullStreams; + exited: Promise; + output: () => string; +} +interface Run { runId: string; status: string } + +let directory: string; +let commands: Command[]; + +function command(executable: string, args: string[]): Command { + const child = spawn(executable, args, { + env: { ...process.env, XDG_CONFIG_HOME: join(directory, "config"), XDG_CACHE_HOME: join(directory, "cache"), PATH: `${join(directory, "bin")}:${process.env.PATH}` }, + }); + let output = ""; + child.stdout.on("data", (chunk: Buffer) => { output += chunk.toString(); }); + child.stderr.on("data", (chunk: Buffer) => { output += chunk.toString(); }); + const exited = new Promise((resolveExit, reject) => { + child.once("error", reject); + child.once("exit", (code, signal) => resolveExit({ code, signal })); + }); + const running = { child, exited, output: () => output }; + commands.push(running); + return running; +} + +async function ready(running: Command, timeout = 10000): Promise { + await expect.poll(() => { + if (running.child.exitCode !== null) throw new Error(running.output()); + return /http:\/\/(?:127\.0\.0\.1|localhost|\[::1\]):\d+/.exec(running.output())?.[0]; + }, { timeout }).toBeDefined(); + return /http:\/\/(?:127\.0\.0\.1|localhost|\[::1\]):\d+/.exec(running.output())![0]; +} + +async function runs(url: string): Promise { + const response = await fetch(`${url}/api/traces`, { signal: AbortSignal.timeout(5000) }); + expect(response.status).toBe(200); + return ((await response.json()) as { runs: Run[] }).runs; +} + +beforeEach(async () => { + await mkdir(".tmp", { recursive: true }); + directory = await mkdtemp(resolve(".tmp/shutdown-api-")); + commands = []; + await mkdir(join(directory, "bin")); + for (const opener of ["open", "xdg-open"]) { + await writeFile(join(directory, "bin", opener), "#!/bin/sh\nexit 0\n", { mode: 0o755 }); + } +}); + +afterEach(async () => { + for (const running of commands) { + if (running.child.exitCode === null && running.child.signalCode === null) running.child.kill("SIGKILL"); + await running.exited; + } + await rm(directory, { recursive: true, force: true }); +}); + +describe.sequential("command shutdown", () => { + it("exits on one Ctrl+C after repeated live viewer polling and leaves its attached target alive", async () => { + const target = command("sleep", ["60"]); + expect(target.child.pid).toBeDefined(); + const running = command(".bin/telltale", ["run", "process", "--pid", String(target.child.pid), "--ui", "--data-dir", directory]); + const url = await ready(running); + const run = (await runs(url))[0]!; + for (let batch = 0; batch < 10; batch++) { + await Promise.all(Array.from({ length: 4 }, async () => { + const response = await fetch(`${url}/api/traces/${run.runId}/processes`, { signal: AbortSignal.timeout(5000) }); + expect(response.status).toBe(200); + await response.arrayBuffer(); + })); + } + running.child.kill("SIGINT"); + expect(await running.exited).toEqual({ code: 0, signal: null }); + expect(target.child.exitCode).toBeNull(); + const manifest = JSON.parse(await readFile(join(directory, "traces", run.runId, "run.json"), "utf8")) as { + runId: string; endedAt: string; exitCode: number; error?: string; captures: { kind: string; ref: { toSeq: number } }[]; + }; + expect(manifest).toMatchObject({ runId: run.runId, exitCode: 0, endedAt: expect.any(String) }); + expect(Date.parse(manifest.endedAt)).toBeGreaterThan(0); + expect(manifest.error).toBeUndefined(); + expect(manifest.captures[0]?.kind).toBe("process"); + expect(manifest.captures[0]?.ref.toSeq).toBeGreaterThan(0); + }); + + it("keeps the viewer open after natural completion and exits on its next Ctrl+C", async () => { + const running = command(".bin/telltale", ["trace", "run", "--ui", "--data-dir", directory, "--", "sleep", "0.2"]); + const url = await ready(running); + await expect.poll(async () => (await runs(url))[0]?.status, { timeout: 10000 }).toBe("succeeded"); + expect(running.child.exitCode).toBeNull(); + running.child.kill("SIGINT"); + expect(await running.exited).toEqual({ code: 0, signal: null }); + }); + + it("closes a replay viewer on its first Ctrl+C", async () => { + const recorded = command(".bin/telltale", ["trace", "run", "--data-dir", directory, "--", "true"]); + expect(await recorded.exited).toEqual({ code: 0, signal: null }); + const ids = await readdir(join(directory, "traces")); + expect(ids).toHaveLength(1); + const running = command(".bin/telltale", ["trace", "replay", ids[0]!, "--ui", "--data-dir", directory]); + const url = await ready(running); + expect(await runs(url)).toHaveLength(1); + running.child.kill("SIGINT"); + expect(await running.exited).toEqual({ code: 0, signal: null }); + }); + + it("stops the HTTP server gracefully on SIGTERM", async () => { + const running = command(".bin/telltale", ["serve", "--port", "0", "--data-dir", directory]); + await ready(running); + running.child.kill("SIGTERM"); + expect(await running.exited).toEqual({ code: 0, signal: null }); + }); + + it("reports a VictoriaMetrics startup failure before announcing the API", async () => { + const state = join(directory, "services", "victoriametrics"); + await mkdir(state, { recursive: true }); + await writeFile(join(state, "state.json"), `{"port":0}`); + const serving = command(".bin/telltale", ["serve", "--prometheus", "--port", "0", "--data-dir", directory]); + expect(await serving.exited).toEqual({ code: 1, signal: null }); + expect(serving.output()).toContain("invalid VictoriaMetrics process identity"); + expect(serving.output()).not.toContain("Telltale API:"); + }); + + it.skipIf(!process.env.TELLTALE_PROMETHEUS_LIVE)("reuses the server's VictoriaMetrics across a separate run and a server restart", async () => { + const exporter = createServer((_req, res) => res.end("# TYPE telltale_api_gauge gauge\ntelltale_api_gauge 42\n")); + const servers: Command[] = []; + await new Promise((resolveListen, reject) => { + exporter.once("error", reject); + exporter.listen(0, "127.0.0.1", resolveListen); + }); + try { + const address = exporter.address(); + if (address === null || typeof address === "string") throw new Error("expected a TCP exporter address"); + const startServer = () => { + const server = command(".bin/telltale", ["serve", "--prometheus", "--port", "0", "--data-dir", directory]); + servers.push(server); + return server; + }; + const statePath = join(directory, "services", "victoriametrics", "state.json"); + const serving = startServer(); + const url = await ready(serving, 60000); + const state = await readFile(statePath, "utf8"); + const { port } = JSON.parse(state) as { port: number }; + const list = await fetch(`${url}/api/v1/connection?type=prometheus`); + expect(list.status).toBe(200); + const connections = await list.json() as { id: string; name: string; read_only: boolean }[]; + expect(connections).toMatchObject([{ name: "telltale-prometheus", read_only: true }]); + const id = connections[0]!.id; + const info = await fetch(`${url}/api/v1/connection/${id}/info`); + expect(await info.json()).toMatchObject({ + connection: { resolvedEndpoint: `http://127.0.0.1:${port}` }, + server: { product: "VictoriaMetrics", version: expect.stringMatching(/^v?1\.156\.0$/) }, + }); + + const start = new Date(Math.floor(Date.now() / 1000) * 1000).toISOString(); + const capture = command(".bin/telltale", ["run", "prometheus", "--data-dir", directory, "--endpoints", + `http://127.0.0.1:${address.port}/metrics`, "--interval", "1s", "--duration", "5s"]); + expect(await capture.exited, capture.output()).toEqual({ code: 0, signal: null }); + expect(await readFile(statePath, "utf8")).toBe(state); + expect(await readdir(join(directory, "victoriametrics", "targets"))).toEqual([]); + const end = new Date(Math.ceil(Date.now() / 1000) * 1000).toISOString(); + + async function checkSamples(base: string) { + await expect.poll(async () => { + const response = await fetch(`${base}/api/v1/connection/${id}/browser/query`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ query: "telltale_api_gauge", options: { range: { start, end, step: "1s" } } }), + }); + const body: unknown = await response.json(); + expect(response.status, JSON.stringify(body)).toBe(200); + return body; + }, { timeout: 15000 }).toMatchObject({ rows: expect.arrayContaining([expect.objectContaining({ value: 42 })]) }); + } + await checkSamples(url); + serving.child.kill("SIGTERM"); + expect(await serving.exited).toEqual({ code: 0, signal: null }); + await expect(fetch(`http://127.0.0.1:${port}/health`)).rejects.toThrow(); + + const restarted = startServer(); + await checkSamples(await ready(restarted, 60000)); + restarted.child.kill("SIGTERM"); + expect(await restarted.exited).toEqual({ code: 0, signal: null }); + } finally { + for (const server of servers) { + if (server.child.exitCode === null && server.child.signalCode === null) server.child.kill("SIGTERM"); + } + await Promise.all(servers.map(server => server.exited)); + await new Promise((resolveClose, reject) => exporter.close(err => err ? reject(err) : resolveClose())); + } + }, 150000); + + it("forces exit on a second Ctrl+C while HTTP shutdown is waiting for a connection", async () => { + const running = command(".bin/telltale", ["serve", "--port", "0", "--data-dir", directory]); + const url = new URL(await ready(running)); + const socket = connect(Number(url.port), url.hostname.replace(/^\[|\]$/g, "")); + await new Promise((resolveConnect, reject) => { socket.once("connect", resolveConnect); socket.once("error", reject); }); + socket.write("GET /health HTTP/1.1\r\nHost: localhost\r\n"); + running.child.kill("SIGINT"); + await expect.poll(running.output).toContain("Shutting down"); + const before = performance.now(); + running.child.kill("SIGINT"); + expect(await running.exited).toEqual({ code: 130, signal: null }); + expect(performance.now() - before).toBeLessThan(1000); + socket.destroy(); + }); +}); diff --git a/packages/api/test/top-feed.spec.ts b/packages/api/test/top-feed.spec.ts new file mode 100644 index 0000000..b5c8675 --- /dev/null +++ b/packages/api/test/top-feed.spec.ts @@ -0,0 +1,50 @@ +import { execFile, spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { once } from "node:events"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { promisify } from "node:util"; +import { afterAll, beforeAll, expect, it } from "vitest"; + +let directory: string; +let server: ChildProcessWithoutNullStreams; +let apiURL: string; + +beforeAll(async () => { + await mkdir(".tmp", { recursive: true }); + directory = await mkdtemp(resolve(".tmp/top-feed-")); + const fixture = join(directory, "handlers.pb"); + await promisify(execFile)("go", ["run", "./hack/jms-profile-fixture", fixture]); + server = spawn(".bin/telltale", ["serve", "--port", "0", "--data-dir", directory, "--otlp", "--otlp-grpc-listen", "localhost:0"], { env: { ...process.env, XDG_CONFIG_HOME: join(directory, "config") } }); + apiURL = await new Promise((resolveURL, reject) => { + let output = ""; + server.stdout.on("data", (chunk: Buffer) => { + output += chunk.toString(); + const match = /Telltale API: (http:\/\/\S+)/.exec(output); + if (match) resolveURL(match[1]!); + }); + server.stderr.on("data", (chunk: Buffer) => { output += chunk.toString(); }); + server.once("error", reject); + server.once("exit", code => reject(new Error(`Server exited (${code}): ${output}`))); + }); + const received = await fetch(`${apiURL}/v1/profiles`, { method: "POST", headers: { "Content-Type": "application/x-protobuf", "OTLP-Profiles-Development-Version": "1" }, body: new Uint8Array(await readFile(fixture)) }); + expect(received.status).toBe(200); +}, 60000); + +afterAll(async () => { + if (server?.exitCode === null) { + const stopped = once(server, "exit"); + server.kill("SIGTERM"); + await stopped; + } + if (directory) await rm(directory, { recursive: true, force: true }); +}); + +it("excludes Top snapshots from paged and live All events while retaining their Top view", async () => { + const mount = `${apiURL}/api/traces/receiver`; + expect(await (await fetch(`${mount}/records`)).json()).toEqual({ rows: [], nextCursor: "" }); + expect(await (await fetch(`${mount}/records?follow=1&format=ndjson`)).text()).toBe(""); + expect(await (await fetch(`${mount}/records?follow=1`)).text()).toBe("event: end\ndata: {}\n\n"); + const top = await fetch(`${mount}/traces/jms_top/records?view=top`); + expect(top.status).toBe(200); + expect(await top.json()).toMatchObject({ count: 2, rows: expect.arrayContaining([expect.objectContaining({ destination: "orders", state: "busy" }), expect.objectContaining({ destination: "replies", state: "idle" })]) }); +}); diff --git a/packages/api/test/top.spec.ts b/packages/api/test/top.spec.ts index ca1ede5..14b4b91 100644 --- a/packages/api/test/top.spec.ts +++ b/packages/api/test/top.spec.ts @@ -1,6 +1,6 @@ import { execFileSync, spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; import { once } from "node:events"; -import { mkdir, mkdtemp, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; @@ -26,6 +26,10 @@ function serve(): Promise { beforeAll(async () => { await mkdir(".tmp", { recursive: true }); testDir = await mkdtemp(resolve(".tmp", "telltale-top-")); + const connections = join(testDir, "config", "flanksource", "connections"); + await mkdir(connections, { recursive: true }); + await writeFile(join(connections, "warehouse.yaml"), "apiVersion: mission-control.flanksource.com/v1\nkind: Connection\nmetadata: {name: warehouse, namespace: tenant}\nspec: {postgres: {url: {value: 'postgres://localhost/example'}}}\n"); + await writeFile(join(connections, "metrics.yaml"), "apiVersion: mission-control.flanksource.com/v1\nkind: Connection\nmetadata: {name: metrics}\nspec: {type: prometheus, url: {value: 'http://localhost:9090'}}\n"); apiURL = await serve(); }); @@ -41,8 +45,48 @@ afterAll(async () => { interface Usage { processes: number; rssBytes: number; + swapBytes?: number; + compressedBytes?: number; + hogs?: Record; } +describe("completed sample records",()=>{ + it("publishes the same SQL connection chooser for CLI operation and web start forms",async()=>{ + const response=await fetch(`${apiURL}/api/openapi.json`); + expect(response.status).toBe(200); + const api=await response.json() as {paths:Record}}}}};get:{parameters:{name:string}[]}}>}; + expect(api.paths["/api/v1/run/sql_top"]?.post.requestBody.content["application/json"].schema.properties.connection).toMatchObject({title:"Connection","x-clicky-lookup":{url:"/api/v1/connection?types=postgres,sql_server",filter:"connection"}}); + expect(api.paths["/api/v1/connection"]?.get.parameters.map(parameter=>parameter.name)).toEqual(expect.arrayContaining(["types","__lookup","__lookup_filter","__lookup_q"])); + const form=await (await fetch(`${apiURL}/api/traces/start`)).json() as {schema:{properties:{traces:{properties:Record}}[]}>}}}}; + expect(form.schema.properties.traces.properties.sql_top?.allOf[0]?.then.properties.connection).toMatchObject({title:"Connection","x-clicky-lookup":{url:"/api/v1/connection?types=postgres,sql_server",filter:"connection"}}); + const choices=await fetch(`${apiURL}/api/v1/connection?types=postgres,sql_server&__lookup=filters`); + expect(choices.status).toBe(200); + expect(await choices.json()).toMatchObject({filters:{connection:{options:{"connection://tenant/warehouse":{plain:"warehouse (postgres)"}},total:1}}}); + }); + it("uses captured query columns and applies native filters and order before paging",async()=>{ + const saved=await fetch(`${apiURL}/api/traces/definitions/sample-top-test`,{method:"PUT",headers:{"Content-Type":"application/json"},body:JSON.stringify({layer:"global",resourceVersion:"",definition:{apiVersion:"telltale.flanksource.com/v1alpha1",kind:"TraceDefinition",metadata:{name:"sample-top-test"},spec:{collect:{query:{sql:[{type:"sqlite",url:":memory:",query:"WITH RECURSIVE valueset(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM valueset WHERE n<150) SELECT n FROM valueset"}]}}}}})}); + expect(saved.status).toBe(200); + const started=await fetch(`${apiURL}/api/traces`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({duration:"500ms",definition:"sample-top-test"})}); + expect(started.status).toBe(201); + const run=await started.json() as {mount:string;runId:string}; + await expect.poll(async()=>((await (await fetch(`${apiURL}/api/traces`)).json()) as {runs:{runId:string;status:string}[]}).runs.find(item=>item.runId===run.runId)?.status,{timeout:10000}).toBe("succeeded"); + const endpoint=`${apiURL}${run.mount}/traces/query/records`; + const frames=await (await fetch(`${endpoint}?view=frames`)).json() as {id:string;retained:number}[]; + expect(frames).toHaveLength(1);expect(frames[0]?.retained).toBe(150); + const params=new URLSearchParams({view:"top",filter:"n=>148",sort:"n",desc:"true",limit:"1"}); + const response=await fetch(`${endpoint}?${params}`);const payload=await response.text();expect(response.status,payload).toBe(200); + const page=JSON.parse(payload) as {selection:string;rows:{n:number}[];count:number;nextCursor:string}; + expect(page.count).toBe(2);expect(page.rows[0]?.n).toBe(150); + params.set("cursor",page.nextCursor);params.set("frame",page.selection); + const next=await (await fetch(`${endpoint}?${params}`)).json() as {rows:{n:number}[]}; + expect(next.rows.map(row=>row.n)).toEqual([149]); + const empty=await (await fetch(`${endpoint}?view=top&filter=n%3D%3C0`)).json() as {rows:unknown[];frame:{retained:number}}; + expect(empty.rows).toEqual([]);expect(empty.frame.retained).toBe(150); + const schema=await fetch(`${endpoint}?view=top`,{headers:{Accept:"application/schema+json"}}); + expect(schema.status).toBe(200);expect(await schema.json()).toMatchObject({columns:expect.arrayContaining([expect.objectContaining({name:"n",type:"number"})])}); + }); +}); + interface Frame { sampledAt: string; tick: number; @@ -50,6 +94,21 @@ interface Frame { } describe("live process groups", () => { + it("reports platform memory columns without substituting compressed memory for disk swap", async () => { + const response = await fetch(`${apiURL}/api/top/processes?pid=${server.pid}&all=true`); + expect(response.status).toBe(200); + const reading = await response.json() as { processes: Array }; + const own = reading.processes.find(process => process.pid === server.pid); + expect(own?.rssBytes).toBeGreaterThan(0); + if (process.platform === "darwin") { + expect(own?.compressedBytes).toBeGreaterThanOrEqual(0); + expect(own?.swapBytes).toBeUndefined(); + } else if (process.platform === "linux") { + expect(own?.swapBytes).toBeGreaterThanOrEqual(0); + expect(own?.compressedBytes).toBeUndefined(); + } + }); + it("carries Git identity in raw frames without adding presentation buckets", async () => { const response = await fetch(`${apiURL}/api/top`); expect(response.status).toBe(200); diff --git a/packages/api/test/traces.spec.ts b/packages/api/test/traces.spec.ts index 64c196c..6e7a11e 100644 --- a/packages/api/test/traces.spec.ts +++ b/packages/api/test/traces.spec.ts @@ -64,6 +64,7 @@ interface ListedRun { command: string[]; exitCode?: number; captures: Captured[]; + process?: { tree: { kind: string; stream: string }; samples: { kind: string; stream: string }; facets: { kind: string; stream: string } }; } interface Listed { @@ -71,6 +72,58 @@ interface Listed { } describe("recorded trace runs", () => { + it("serves pinned process measurements separately from the styled event stream", async () => { + const listed = await (await fetch(`${apiURL}/api/traces`)).json() as Listed; + const recorded = listed.runs[0]; + const snapshot = await fetch(`${apiURL}${recorded.mount}/processes`); + expect(snapshot.status).toBe(200); + expect(await snapshot.json()).toMatchObject({ armed: true, rows: expect.arrayContaining([expect.objectContaining({ runId: recorded.runId, key: expect.any(String), pid: expect.any(Number), rssBytes: expect.any(Number), memoryPercent: expect.any(Number) })]) }); + const events = await fetch(`${apiURL}${recorded.mount}/records?excludeKind=process&presentation=ansi&sort=timestamp&limit=1`); + expect(events.status).toBe(200); + const page = await events.json() as { rows: { kind: string; ansi: string }[] }; + expect(page.rows).toHaveLength(1); + expect(page.rows[0].kind).toBe("http"); + expect(page.rows[0].ansi).toContain("GET"); + expect(page.rows[0].ansi).toContain("\u001b["); + const raw = await (await fetch(`${apiURL}${recorded.mount}/records?kind=http&limit=1`)).json() as { rows: Record[] }; + expect(raw.rows[0]).not.toHaveProperty("ansi"); + }); + it("exposes one process kind with identity, numeric metrics, facet events and derived groups", async () => { + const listed = await (await fetch(`${apiURL}/api/traces`)).json() as Listed; + const recorded = listed.runs[0]; + expect(recorded.process).toMatchObject({ tree: { kind: "process" }, samples: { kind: "process_metrics" }, facets: { kind: "process_facets" } }); + const catalog = await (await fetch(`${apiURL}${recorded.mount}/api/v1/traces/kinds`)).json() as { name: string; results?: { name: string }[] }[]; + expect(catalog.filter(kind => kind.name.startsWith("process"))).toEqual([expect.objectContaining({ name: "process", results: [expect.objectContaining({ name: "facets" }), expect.objectContaining({ name: "metrics" }), expect.objectContaining({ name:"frames" })] })]); + const metrics = await fetch(`${apiURL}${recorded.mount}/traces/process/records?result=metrics&limit=1`); + expect(metrics.status).toBe(200); + expect(await metrics.json()).toMatchObject({ rows: [expect.objectContaining({ entity: expect.any(String), rssBytes: expect.any(Number) })] }); + const groups = await fetch(`${apiURL}${recorded.mount}/process/groups?stream=${recorded.process!.tree.stream}`); + expect(groups.status).toBe(200); + expect(await groups.json()).toMatchObject({ highSeq: expect.any(Number), frames: expect.arrayContaining([expect.objectContaining({ groups: expect.arrayContaining([expect.objectContaining({ total: expect.objectContaining({ processes: expect.any(Number) }) })]) })]) }); + }); + it("replays the API's compact run ID through the same stored capture feed", async () => { + const listed = await (await fetch(`${apiURL}/api/traces`)).json() as Listed; + const recorded = listed.runs[0]; + expect(recorded.runId).toMatch(/^[A-Za-z0-9]{1,9}\d{8}T\d{6}Z(?:_\d+)?$/); + const replayed = await run(".bin/telltale", ["trace", "replay", "--data-dir", testDir, "--format", "ndjson", recorded.runId]); + const rows = replayed.stdout.trim().split("\n").map(line => JSON.parse(line) as { runId: string; kind: string; record: { url: string } }); + expect(rows).toContainEqual(expect.objectContaining({ runId: recorded.runId, kind: "http", record: expect.objectContaining({ url: upstreamURL }) })); + expect(rows).toContainEqual(expect.objectContaining({ runId: recorded.runId, kind: "process", result: "metrics" })); + }); + it("pages the unified armed-capture feed and serves a completed NDJSON stream", async () => { + const listed = await (await fetch(`${apiURL}/api/traces`)).json() as Listed; + const recorded = listed.runs[0]; + expect(recorded).toBeDefined(); + const response = await fetch(`${apiURL}${recorded.mount}/records?kind=http&sort=seq&limit=1`); + expect(response.status).toBe(200); + const page = await response.json() as { rows: { runId: string; kind: string; stream: string; seq: number; record: Record }[] }; + expect(page.rows).toHaveLength(1); + expect(page.rows[0]).toMatchObject({ runId: recorded.runId, kind: "http", seq: 1, record: { request: expect.anything() } }); + const streamed = await fetch(`${apiURL}${recorded.mount}/records?kind=http&follow=1&format=ndjson`); + expect(streamed.status).toBe(200); + expect(JSON.parse((await streamed.text()).trim())).toMatchObject(page.rows[0]); + }); + it("serves typed result-profile columns and query rows for a recorded capture", async () => { const listed = await (await fetch(`${apiURL}/api/traces`)).json() as Listed; const recorded = listed.runs[0]; @@ -166,7 +219,7 @@ describe("runs started over the API", () => { const response = await fetch(`${apiURL}/api/traces/start`); const form = await response.json() as { schema: { properties: { traces: { properties: Record }; pid: { type: string }; duration: { type: string } } } }; expect(response.status).toBe(200); - expect(form.schema.properties.traces.properties).toMatchObject({ sql_xevent: { title: "SQL Server XEvents" }, sql_deadlock: { title: "SQL Server deadlocks" }, jvm_jfr: { title: "JVM JFR" } }); + expect(form.schema.properties.traces.properties).toMatchObject({ sql_xevent: { title: "SQL Server XEvents" }, sql_deadlock: { title: "SQL Server deadlocks" }, jvm_jfr: { title: "JVM JFR" }, jms_top: { title: "JMS top" } }); expect(form.schema.properties.pid.type).toBe("integer"); expect(form.schema.properties.duration.type).toBe("string"); }); @@ -190,7 +243,7 @@ describe("runs started over the API", () => { const response = await start({ command: `curl -sS ${upstreamURL}`, traces: { http: { enabled: true, target: { seam: "proxy" } } } }); expect(response.status).toBe(201); const started = await response.json() as ListedRun; - expect(started).toMatchObject({ status: "running", command: ["sh", "-c", `curl -sS ${upstreamURL}`] }); + expect(started).toMatchObject({ status: "running", command: ["sh", "-c", `curl -sS ${upstreamURL}`], output: { stdout: "stdout.log", stderr: "stderr.log" } }); expect(await settled(started.runId)).toMatchObject({ status: "succeeded", exitCode: 0, captures: [{ kind: "http", summary: { count: 1 } }] }); }); diff --git a/pkg/agentclient/client.go b/pkg/agentclient/client.go index ebe2ebe..baecad8 100644 --- a/pkg/agentclient/client.go +++ b/pkg/agentclient/client.go @@ -1,7 +1,6 @@ package agentclient import ( - "bytes" "context" "crypto/rand" "encoding/hex" @@ -17,19 +16,26 @@ import ( "github.com/flanksource/telltale/sdk/protocol" ) -// Agent is an authenticated connection to a local agent. +// Agent is an authenticated connection to an agent, over the frame protocol +// (a local socket or the WebSocket hub) or direct HTTP to a deployed agent. type Agent struct { - Hello protocol.Hello - localPID int - stream net.Conn - conn *protocol.Conn - mu sync.Mutex - sessions map[string]*Session - calls map[string]chan protocol.Result - callSeen map[string]bool - done chan struct{} - pong chan struct{} - failure error + Hello protocol.Hello + localPID int + transport transport + mu sync.Mutex + done chan struct{} + doneOnce sync.Once + failure error +} + +// transport carries one agent's captures and calls. Session state that every +// transport shares (its channels, its end) stays on Session. +type transport interface { + arm(ctx context.Context, s *Session, request ArmRequest) error + checkpoint(ctx context.Context, s *Session, params json.RawMessage) (CheckpointResult, error) + disarm(ctx context.Context, s *Session) error + call(ctx context.Context, request CallRequest) (json.RawMessage, error) + close() error } // ArmRequest selects a probe and its session settings. @@ -106,14 +112,7 @@ func Connect(ctx context.Context, stream net.Conn, token string) (*Agent, error) _ = stream.Close() return nil, fmt.Errorf("clear agent deadline: %w", err) } - return newAgent(&protocol.Client{Stream: stream, Conn: conn, Hello: hello}), nil -} - -func newAgent(client *protocol.Client) *Agent { - agent := &Agent{Hello: client.Hello, stream: client.Stream, conn: client.Conn, sessions: make(map[string]*Session), calls: make(map[string]chan protocol.Result), callSeen: make(map[string]bool), done: make(chan struct{}), pong: make(chan struct{}, 1)} - go agent.read() - go agent.heartbeat() - return agent + return newWireAgent(&protocol.Client{Stream: stream, Conn: conn, Hello: hello}), nil } // Dial connects to a local discovery endpoint and verifies its identity. @@ -130,42 +129,25 @@ func Dial(ctx context.Context, endpoint Endpoint) (*Agent, error) { _ = client.Close() return nil, fmt.Errorf("%w: agent identity mismatch", protocol.ErrStaleEndpoint) } - return newAgent(client), nil + return newWireAgent(client), nil } -func (a *Agent) heartbeat() { - ticker := time.NewTicker(10 * time.Second) - defer ticker.Stop() - for { - select { - case <-ticker.C: - if err := write(a.conn, "ping", protocol.Ping{}); err != nil { - _ = a.stream.Close() - return - } - timer := time.NewTimer(10 * time.Second) - select { - case <-a.pong: - timer.Stop() - case <-timer.C: - _ = a.stream.Close() - return - case <-a.done: - timer.Stop() - return - } - case <-a.done: - return - } - } +// fail ends the connection once with cause; every later caller sees cause. +func (a *Agent) fail(cause error) { + a.doneOnce.Do(func() { + a.mu.Lock() + a.failure = cause + a.mu.Unlock() + close(a.done) + }) } -func write(conn *protocol.Conn, kind string, body any) error { - data, err := json.Marshal(body) - if err != nil { - return err +func newSession(agent *Agent) (*Session, error) { + var id [16]byte + if _, err := rand.Read(id[:]); err != nil { + return nil, err } - return conn.Write(protocol.Envelope{Type: kind, Body: data}, nil) + return &Session{agent: agent, ID: hex.EncodeToString(id[:]), pending: make(map[string]*checkpointCall), seenCheckpoints: make(map[string]struct{}), Records: make(chan protocol.Records, 64), Blobs: make(chan Blob, 64), Reports: make(chan protocol.Report, 64), Warns: make(chan protocol.Warn, 64), Ended: make(chan protocol.Ended, 1), armed: make(chan protocol.Armed, 1), finished: make(chan struct{})}, nil } // Arm waits for the agent to accept a capture. @@ -176,42 +158,14 @@ func (a *Agent) Arm(ctx context.Context, request ArmRequest) (*Session, error) { if err := a.validateArm(request); err != nil { return nil, err } - var id [16]byte - if _, err := rand.Read(id[:]); err != nil { + session, err := newSession(a) + if err != nil { return nil, err } - s := &Session{agent: a, ID: hex.EncodeToString(id[:]), pending: make(map[string]*checkpointCall), seenCheckpoints: make(map[string]struct{}), Records: make(chan protocol.Records, 64), Blobs: make(chan Blob, 64), Reports: make(chan protocol.Report, 64), Warns: make(chan protocol.Warn, 64), Ended: make(chan protocol.Ended, 1), armed: make(chan protocol.Armed, 1), finished: make(chan struct{})} - a.mu.Lock() - a.sessions[s.ID] = s - a.mu.Unlock() - if err := write(a.conn, "arm", protocol.Arm{Session: s.ID, Probe: request.Probe, Params: request.Params, AllowMutation: request.AllowMutation, StopAt: request.StopAt, Limits: request.Limits}); err != nil { - a.mu.Lock() - delete(a.sessions, s.ID) - a.mu.Unlock() + if err := a.transport.arm(ctx, session, request); err != nil { return nil, err } - select { - case reply := <-s.armed: - if reply.Error != "" { - a.mu.Lock() - delete(a.sessions, s.ID) - a.mu.Unlock() - return nil, errors.New(reply.Error) - } - return s, nil - case <-ctx.Done(): - a.mu.Lock() - delete(a.sessions, s.ID) - a.mu.Unlock() - go func() { - if err := write(a.conn, protocol.TypeDisarm, protocol.Disarm{Session: s.ID}); err != nil { - _ = a.stream.Close() - } - }() - return nil, ctx.Err() - case <-a.done: - return nil, fmt.Errorf("arm: %w", a.connectionError()) - } + return session, nil } func (a *Agent) validateArm(request ArmRequest) error { @@ -229,40 +183,7 @@ func (a *Agent) validateArm(request ArmRequest) error { // Checkpoint requests an on-demand reading. func (s *Session) Checkpoint(ctx context.Context, params json.RawMessage) (CheckpointResult, error) { - var random [16]byte - if _, err := rand.Read(random[:]); err != nil { - return CheckpointResult{}, err - } - id := hex.EncodeToString(random[:]) - call := &checkpointCall{done: make(chan protocol.Checkpointed, 1), params: bytes.Clone(params)} - s.mu.Lock() - s.pending[id] = call - s.mu.Unlock() - defer func() { s.mu.Lock(); delete(s.pending, id); s.mu.Unlock() }() - if err := write(s.agent.conn, "checkpoint", protocol.Checkpoint{Session: s.ID, ID: id, Params: params}); err != nil { - return CheckpointResult{}, fmt.Errorf("checkpoint: %w", err) - } - select { - case done := <-call.done: - if done.Error != "" { - return call.result, fmt.Errorf("agent checkpoint: %s", done.Error) - } - return call.result, nil - case <-s.finished: - select { - case done := <-call.done: - if done.Error != "" { - return call.result, fmt.Errorf("agent checkpoint: %s", done.Error) - } - return call.result, nil - default: - } - return CheckpointResult{}, s.endError() - case <-s.agent.done: - return CheckpointResult{}, fmt.Errorf("checkpoint: %w", s.agent.connectionError()) - case <-ctx.Done(): - return CheckpointResult{}, ctx.Err() - } + return s.agent.transport.checkpoint(ctx, s, params) } // Disarm releases the capture. @@ -272,9 +193,11 @@ func (s *Session) Disarm(ctx context.Context) error { return s.releaseError() default: } - if err := write(s.agent.conn, "disarm", protocol.Disarm{Session: s.ID}); err != nil { - return fmt.Errorf("disarm: %w", err) - } + return s.agent.transport.disarm(ctx, s) +} + +// awaitRelease waits for a requested disarm to end the session. +func (s *Session) awaitRelease(ctx context.Context) error { select { case <-s.finished: return s.releaseError() @@ -301,30 +224,7 @@ func (a *Agent) Call(ctx context.Context, request CallRequest) (json.RawMessage, return nil, fmt.Errorf("operation %s: caller mutation disabled", request.Op) } } - var id [16]byte - if _, err := rand.Read(id[:]); err != nil { - return nil, err - } - callID := hex.EncodeToString(id[:]) - response := make(chan protocol.Result, 1) - a.mu.Lock() - a.calls[callID] = response - a.mu.Unlock() - defer func() { a.mu.Lock(); delete(a.calls, callID); a.mu.Unlock() }() - if err := write(a.conn, "call", protocol.Call{CallID: callID, Op: request.Op, Params: request.Params, AllowMutation: request.AllowMutation}); err != nil { - return nil, fmt.Errorf("agent call: %w", err) - } - select { - case result := <-response: - if result.Error != "" { - return nil, fmt.Errorf("agent call %s: %s", request.Op, result.Error) - } - return result.Value, nil - case <-a.done: - return nil, fmt.Errorf("call: %w", a.connectionError()) - case <-ctx.Done(): - return nil, ctx.Err() - } + return a.transport.call(ctx, request) } func (s *Session) endError() error { @@ -357,9 +257,7 @@ func (s *Session) finish(value protocol.Ended) { } // Close closes the connection and all its captures. -func (a *Agent) Close() error { return a.stream.Close() } - -func (a *Agent) read() { a.readLoop() } +func (a *Agent) Close() error { return a.transport.close() } func deliver[T any](channel chan<- T, value T) bool { select { diff --git a/pkg/agentclient/direct.go b/pkg/agentclient/direct.go new file mode 100644 index 0000000..7f62b60 --- /dev/null +++ b/pkg/agentclient/direct.go @@ -0,0 +1,322 @@ +package agentclient + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "maps" + "net/http" + "sync" + "time" + + "github.com/flanksource/telltale/sdk/protocol" +) + +// HTTPEndpoint is a deployed agent a deployer recorded: its target id (the +// JWT audience and the agent's identity), its execution URL and the +// attributes a capture selects it by. +type HTTPEndpoint struct { + ID string + URL string + Client *http.Client + Tokens TokenSource + Resource map[string]string + // Poll is how often a streaming capture is read; zero reads at the + // agent's own hub cadence, DefaultPoll. + Poll time.Duration +} + +// DefaultPoll matches the cadence the agent streams captures at over the hub. +const DefaultPoll = 250 * time.Millisecond + +const ( + renewEvery = 30 * time.Second + requestTimeout = 30 * time.Second + closeTimeout = 10 * time.Second +) + +// DialHTTP connects to a deployed agent over direct HTTP. It refuses an agent +// speaking another agent API rather than guessing at its routes. +func DialHTTP(ctx context.Context, endpoint HTTPEndpoint) (*Agent, error) { + client := HTTPClient{Target: endpoint.ID, URL: endpoint.URL, Client: endpoint.Client, Tokens: endpoint.Tokens} + hello, err := client.Hello(ctx) + if err != nil { + return nil, fmt.Errorf("dial agent %s: %w", endpoint.ID, err) + } + if hello.APIVersion != AgentAPIVersion { + return nil, fmt.Errorf("dial agent %s: it speaks agent API %d, this client speaks %d", endpoint.ID, hello.APIVersion, AgentAPIVersion) + } + if err := protocol.CheckVersion(hello.Protocol); err != nil { + return nil, fmt.Errorf("dial agent %s: %w", endpoint.ID, err) + } + poll := endpoint.Poll + if poll == 0 { + poll = DefaultPoll + } + resource := make(map[string]string, len(endpoint.Resource)+len(hello.Resource)) + maps.Copy(resource, endpoint.Resource) + maps.Copy(resource, hello.Resource) + agent := &Agent{done: make(chan struct{}), Hello: protocol.Hello{AgentID: endpoint.ID, Service: resource["service.name"], SDKVersion: hello.SDKVersion, Protocol: hello.Protocol, Started: hello.Started, Runtime: hello.Runtime, Resource: resource, Probes: hello.Probes, Ops: hello.Ops}} + agent.transport = &httpTransport{agent: agent, client: client, poll: poll, ops: hello.Ops, captures: make(map[*Session]*httpCapture)} + return agent, nil +} + +type httpTransport struct { + agent *Agent + client HTTPClient + poll time.Duration + mu sync.Mutex + ops []protocol.OpSpec + captures map[*Session]*httpCapture +} + +// httpCapture is one armed capture: the agent holds its state, the client +// reads, renews and finishes it. +type httpCapture struct { + Capture + scope string + stop chan struct{} + stopOnce sync.Once + seq uint64 + sent int64 + maxBytes int64 + report string +} + +func (t *httpTransport) probeScope(name string) (string, error) { + for _, probe := range t.agent.Hello.Probes { + if probe.Name == name && probe.Scope != "" { + return probe.Scope, nil + } + } + return "", fmt.Errorf("agent %s names no scope for probe %q", t.agent.Hello.AgentID, name) +} + +// opScope is the scope the agent's catalog names for op, re-reading the +// catalog once for an operation an extension added since the hello. +func (t *httpTransport) opScope(ctx context.Context, op string) (string, error) { + if scope := t.lookupOp(op); scope != "" { + return scope, nil + } + ops, err := t.client.Catalog(ctx) + if err != nil { + return "", fmt.Errorf("refresh catalog of %s: %w", t.agent.Hello.AgentID, err) + } + t.mu.Lock() + t.ops = ops + t.mu.Unlock() + if scope := t.lookupOp(op); scope != "" { + return scope, nil + } + return "", fmt.Errorf("agent %s does not advertise operation %q with a scope", t.agent.Hello.AgentID, op) +} + +func (t *httpTransport) lookupOp(op string) string { + t.mu.Lock() + defer t.mu.Unlock() + for _, spec := range t.ops { + if spec.Name == op { + return spec.Scope + } + } + return "" +} + +func (t *httpTransport) call(ctx context.Context, request CallRequest) (json.RawMessage, error) { + scope, err := t.opScope(ctx, request.Op) + if err != nil { + return nil, err + } + return t.client.Call(ctx, scope, request) +} + +func (t *httpTransport) arm(ctx context.Context, s *Session, request ArmRequest) error { + if !request.StopAt.IsZero() && !request.StopAt.After(time.Now()) { + return errors.New("capture deadline already passed") + } + if request.Limits.MaxBytes < 0 { + return errors.New("capture byte limit must not be negative") + } + scope, err := t.probeScope(request.Probe) + if err != nil { + return err + } + capture, err := t.client.Open(ctx, scope, request.Probe, request.Params, request.AllowMutation) + if err != nil { + return err + } + s.ID = capture.ID + c := &httpCapture{Capture: capture, scope: scope, stop: make(chan struct{}), maxBytes: request.Limits.MaxBytes} + t.mu.Lock() + t.captures[s] = c + t.mu.Unlock() + go t.pump(s, c, request.StopAt) + return nil +} + +func (t *httpTransport) lookup(s *Session) *httpCapture { + t.mu.Lock() + defer t.mu.Unlock() + return t.captures[s] +} + +// pump reads a streaming capture, renews its lease, and ends it on +// completion, deadline, disarm or close. +func (t *httpTransport) pump(s *Session, c *httpCapture, stopAt time.Time) { + var poll, deadline <-chan time.Time + if c.Streaming { + ticker := time.NewTicker(t.poll) + defer ticker.Stop() + poll = ticker.C + } + if !stopAt.IsZero() { + timer := time.NewTimer(time.Until(stopAt)) + defer timer.Stop() + deadline = timer.C + } + renew := time.NewTicker(renewEvery) + defer renew.Stop() + for { + select { + case <-poll: + complete, err := t.stream(s, c) + if err != nil || complete { + t.end(s, c, err, err == nil) + return + } + case <-renew.C: + if err := t.request(func(ctx context.Context) error { return t.client.Renew(ctx, c.scope, c.ID) }); err != nil { + t.end(s, c, err, false) + return + } + case <-deadline: + t.end(s, c, nil, true) + return + case <-c.stop: + t.end(s, c, nil, false) + return + } + } +} + +func (t *httpTransport) request(run func(context.Context) error) error { + ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) + defer cancel() + return run(ctx) +} + +func (t *httpTransport) stream(s *Session, c *httpCapture) (bool, error) { + var dump Dump + err := t.request(func(ctx context.Context) (err error) { dump, err = t.client.Dump(ctx, c.scope, c.ID); return err }) + if err != nil { + return false, err + } + blob, err := t.blob(s, c, "", dump.Data) + if err != nil { + return false, err + } + if !deliver(s.Blobs, blob) { + s.dropped.Add(1) + } + t.deliverReport(s, c, dump.Report) + return dump.Complete, nil +} + +// deliverReport hands the session a report that changed since the last one, +// after the blob it describes, as the agent sends report frames over the hub. +func (t *httpTransport) deliverReport(s *Session, c *httpCapture, report json.RawMessage) { + if report == nil { + return + } + t.mu.Lock() + changed := string(report) != c.report + c.report = string(report) + t.mu.Unlock() + if changed && !deliver(s.Reports, protocol.Report{Session: s.ID, Summary: report}) { + s.dropped.Add(1) + } +} + +func (t *httpTransport) blob(s *Session, c *httpCapture, checkpoint string, data []byte) (Blob, error) { + t.mu.Lock() + defer t.mu.Unlock() + if c.maxBytes > 0 && int64(len(data)) > c.maxBytes-c.sent { + return Blob{}, errors.New("capture byte limit exceeded") + } + c.sent += int64(len(data)) + c.seq++ + return Blob{Info: protocol.Blob{Session: s.ID, Checkpoint: checkpoint, Seq: c.seq, Name: c.FileName, ContentType: c.ContentType, Len: len(data)}, Data: data}, nil +} + +// end finishes the capture on the agent, which restores what it changed, +// delivers its last reading and ends the session. +func (t *httpTransport) end(s *Session, c *httpCapture, cause error, complete bool) { + var final Dump + err := t.request(func(ctx context.Context) (err error) { final, err = t.client.Finish(ctx, c.scope, c.ID); return err }) + if err == nil { + var blob Blob + if blob, err = t.blob(s, c, "", final.Data); err == nil { + if !deliver(s.Blobs, blob) { + s.dropped.Add(1) + } + t.deliverReport(s, c, final.Report) + } + } + t.mu.Lock() + delete(t.captures, s) + t.mu.Unlock() + ended := protocol.Ended{Session: s.ID, Complete: complete && err == nil} + if failure := errors.Join(cause, err); failure != nil { + ended.Error = failure.Error() + } + s.finish(ended) +} + +func (t *httpTransport) checkpoint(ctx context.Context, s *Session, params json.RawMessage) (CheckpointResult, error) { + c := t.lookup(s) + if c == nil { + return CheckpointResult{}, s.endError() + } + var id [16]byte + if _, err := rand.Read(id[:]); err != nil { + return CheckpointResult{}, err + } + dump, err := t.client.Dump(ctx, c.scope, c.ID) + if err != nil { + return CheckpointResult{}, fmt.Errorf("agent checkpoint: %w", err) + } + blob, err := t.blob(s, c, hex.EncodeToString(id[:]), dump.Data) + if err != nil { + return CheckpointResult{}, err + } + blob.CheckpointParams = params + t.deliverReport(s, c, dump.Report) + return CheckpointResult{Blobs: []Blob{blob}}, nil +} + +func (t *httpTransport) disarm(ctx context.Context, s *Session) error { + if c := t.lookup(s); c != nil { + c.stopOnce.Do(func() { close(c.stop) }) + } + return s.awaitRelease(ctx) +} + +// close finishes every live capture before the agent counts as gone, so no +// instrumentation outlives the client. +func (t *httpTransport) close() error { + t.mu.Lock() + live := maps.Clone(t.captures) + t.mu.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), closeTimeout) + defer cancel() + var errs []error + for session, c := range live { + c.stopOnce.Do(func() { close(c.stop) }) + errs = append(errs, session.awaitRelease(ctx)) + } + t.agent.fail(errors.New("agent connection closed")) + return errors.Join(errs...) +} diff --git a/pkg/agentclient/direct_client.go b/pkg/agentclient/direct_client.go new file mode 100644 index 0000000..3a51c87 --- /dev/null +++ b/pkg/agentclient/direct_client.go @@ -0,0 +1,287 @@ +package agentclient + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/flanksource/telltale/sdk/protocol" +) + +// AgentAPIVersion is the direct-HTTP agent API this client speaks: the routes +// and JWT scopes of telltale-diag.war and the attached agent's HTTP listener. +const AgentAPIVersion = 1 + +// ScopeRead is the scope of an agent's hello and catalog. +const ScopeRead = "jvm:read" + +// TokenHeader carries the execution token. The Kubernetes apiserver deletes +// Authorization from every request it proxies (pods/proxy, services/proxy), +// so the token never travels in Authorization, nor in a query parameter. +const TokenHeader = "X-Telltale-Authorization" + +// ReportHeader carries, beside a capture dump, the JSON object the capture +// reports about what it withheld (jvm_trace: filtered, omitted): over direct +// HTTP what a report frame carries over the hub. +const ReportHeader = "X-Telltale-Report" + +// maxArtifact bounds a capture dump read from an agent. +const maxArtifact = 64 << 20 + +// TokenSource mints the short-lived bearer token one request presents. +type TokenSource interface { + Token(ctx context.Context, audience, scope string) (string, error) +} + +// HTTPClient calls one deployed agent's routes: /health, /hello, /catalog, +// /call and /captures. The audience of every token is the target id. +type HTTPClient struct { + Target string + URL string + Client *http.Client + Tokens TokenSource +} + +// Health is an agent's unauthenticated readiness and version report. +type Health struct { + Status string `json:"status"` + APIVersion int `json:"apiVersion"` + Required int `json:"required,omitempty"` + ConfigDigest string `json:"configDigest,omitempty"` + UpgradeAPIVersion int `json:"upgradeApiVersion,omitempty"` + RuntimeVersion string `json:"runtimeVersion,omitempty"` + ArtifactSHA256 string `json:"artifactSHA256,omitempty"` + Error string `json:"error,omitempty"` +} + +// AgentHello is what an agent says about itself over HTTP. +type AgentHello struct { + SDKVersion string `json:"sdkVersion"` + APIVersion int `json:"apiVersion"` + Protocol int `json:"protocol"` + Started time.Time `json:"started"` + Runtime protocol.Runtime `json:"runtime"` + Resource map[string]string `json:"resource"` + Probes []protocol.ProbeSpec `json:"probes"` + Ops []protocol.OpSpec `json:"ops"` +} + +// Capture is an armed capture as the agent describes it. +type Capture struct { + ID string `json:"id"` + Streaming bool `json:"streaming"` + ContentType string `json:"contentType"` + FileName string `json:"fileName"` +} + +// Dump is one reading of a capture. +type Dump struct { + Data []byte + Complete bool + // Report is the capture's report object, nil when the agent sent none. + Report json.RawMessage +} + +func (c HTTPClient) validate() error { + switch { + case c.Target == "": + return errors.New("direct agent: target id is required") + case c.URL == "": + return fmt.Errorf("direct agent %s: endpoint URL is required", c.Target) + case c.Client == nil: + return fmt.Errorf("direct agent %s: HTTP client is required", c.Target) + case c.Tokens == nil: + return fmt.Errorf("direct agent %s: token source is required", c.Target) + } + return nil +} + +// Health reads /health. A 503 outdated agent is a Health, not an error. +func (c HTTPClient) Health(ctx context.Context) (Health, error) { + response, err := c.send(ctx, http.MethodGet, "/health", "", nil) + if err != nil { + return Health{}, err + } + var health Health + if err := decodeHealth(response, &health); err != nil { + return Health{}, fmt.Errorf("agent %s health: %w", c.Target, err) + } + return health, nil +} + +func decodeHealth(response *http.Response, health *Health) error { + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK && response.StatusCode != http.StatusServiceUnavailable { + return statusError(response) + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(health); err != nil { + return fmt.Errorf("decode HTTP %d: %w", response.StatusCode, err) + } + if health.Status == "" { + return fmt.Errorf("HTTP %d without a status", response.StatusCode) + } + return nil +} + +// Hello reads /hello. +func (c HTTPClient) Hello(ctx context.Context) (AgentHello, error) { + var hello AgentHello + return hello, c.json(ctx, http.MethodGet, "/hello", ScopeRead, nil, &hello) +} + +// Catalog reads /catalog. +func (c HTTPClient) Catalog(ctx context.Context) ([]protocol.OpSpec, error) { + var ops []protocol.OpSpec + return ops, c.json(ctx, http.MethodGet, "/catalog", ScopeRead, nil, &ops) +} + +// Call runs one operation under scope. +func (c HTTPClient) Call(ctx context.Context, scope string, request CallRequest) (json.RawMessage, error) { + params := request.Params + if params == nil { + params = json.RawMessage(`{}`) + } + body := map[string]any{"operation": request.Op, "params": params, "allowMutation": request.AllowMutation} + var value json.RawMessage + if err := c.json(ctx, http.MethodPost, "/call", scope, body, &value); err != nil { + return nil, fmt.Errorf("agent call %s: %w", request.Op, err) + } + return value, nil +} + +// Open arms a capture of kind under scope. +func (c HTTPClient) Open(ctx context.Context, scope, kind string, params json.RawMessage, allowMutation bool) (Capture, error) { + var capture Capture + body := map[string]any{"kind": kind, "params": params, "allowMutation": allowMutation} + if err := c.json(ctx, http.MethodPost, "/captures", scope, body, &capture); err != nil { + return Capture{}, fmt.Errorf("arm %s: %w", kind, err) + } + if capture.ID == "" { + return Capture{}, fmt.Errorf("arm %s: agent %s returned no capture id", kind, c.Target) + } + return capture, nil +} + +// Dump reads the capture so far. +func (c HTTPClient) Dump(ctx context.Context, scope, id string) (Dump, error) { + return c.artifact(ctx, http.MethodPost, "/captures/"+id, scope) +} + +// Renew extends the capture's JVM-owned lease. +func (c HTTPClient) Renew(ctx context.Context, scope, id string) error { + response, err := c.send(ctx, http.MethodPost, "/captures/"+id+"/renew", scope, nil) + if err != nil { + return err + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusNoContent { + return fmt.Errorf("renew capture %s: %w", id, statusError(response)) + } + return nil +} + +// Finish ends the capture and returns its last reading. +func (c HTTPClient) Finish(ctx context.Context, scope, id string) (Dump, error) { + return c.artifact(ctx, http.MethodDelete, "/captures/"+id, scope) +} + +func (c HTTPClient) artifact(ctx context.Context, method, path, scope string) (Dump, error) { + response, err := c.send(ctx, method, path, scope, nil) + if err != nil { + return Dump{}, err + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK { + return Dump{}, fmt.Errorf("%s %s: %w", method, path, statusError(response)) + } + data, err := io.ReadAll(io.LimitReader(response.Body, maxArtifact+1)) + if err != nil { + return Dump{}, fmt.Errorf("%s %s: %w", method, path, err) + } + if len(data) > maxArtifact { + return Dump{}, fmt.Errorf("%s %s: capture exceeds %d bytes", method, path, maxArtifact) + } + report, err := captureReport(response.Header) + if err != nil { + return Dump{}, fmt.Errorf("%s %s: %w", method, path, err) + } + return Dump{Data: data, Complete: response.Header.Get("X-Telltale-Complete") == "true", Report: report}, nil +} + +func captureReport(header http.Header) (json.RawMessage, error) { + raw := header.Get(ReportHeader) + if raw == "" { + return nil, nil + } + var values map[string]any + if err := json.Unmarshal([]byte(raw), &values); err != nil || values == nil { + return nil, fmt.Errorf("malformed %s header %q: want a JSON object", ReportHeader, raw) + } + return json.RawMessage(raw), nil +} + +func (c HTTPClient) json(ctx context.Context, method, path, scope string, body, into any) error { + response, err := c.send(ctx, method, path, scope, body) + if err != nil { + return err + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK { + return statusError(response) + } + if err := json.NewDecoder(io.LimitReader(response.Body, maxArtifact)).Decode(into); err != nil { + return fmt.Errorf("decode %s %s: %w", method, path, err) + } + return nil +} + +// send issues one request; an empty scope sends no token (only /health). +func (c HTTPClient) send(ctx context.Context, method, path, scope string, body any) (*http.Response, error) { + if err := c.validate(); err != nil { + return nil, err + } + var reader io.Reader + if body != nil { + data, err := json.Marshal(body) + if err != nil { + return nil, err + } + reader = bytes.NewReader(data) + } + request, err := http.NewRequestWithContext(ctx, method, strings.TrimRight(c.URL, "/")+path, reader) + if err != nil { + return nil, err + } + if body != nil { + request.Header.Set("Content-Type", "application/json") + } + if scope != "" { + token, err := c.Tokens.Token(ctx, c.Target, scope) + if err != nil { + return nil, fmt.Errorf("token for %s on %s: %w", scope, c.Target, err) + } + request.Header.Set(TokenHeader, "Bearer "+token) + } + response, err := c.Client.Do(request) + if err != nil { + return nil, fmt.Errorf("agent %s: %w", c.Target, err) + } + return response, nil +} + +func statusError(response *http.Response) error { + data, _ := io.ReadAll(io.LimitReader(response.Body, 64<<10)) + var body struct { + Error string `json:"error"` + } + if json.Unmarshal(data, &body) == nil && body.Error != "" { + return fmt.Errorf("HTTP %d: %s", response.StatusCode, body.Error) + } + return fmt.Errorf("HTTP %d: %s", response.StatusCode, strings.TrimSpace(string(data))) +} diff --git a/pkg/agentclient/direct_spec_test.go b/pkg/agentclient/direct_spec_test.go new file mode 100644 index 0000000..196220f --- /dev/null +++ b/pkg/agentclient/direct_spec_test.go @@ -0,0 +1,230 @@ +package agentclient_test + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/sdk/protocol" +) + +const directTarget = "deployer:example/orders/orders-0" + +// grant is one token the fake agent was asked for. +type grant struct{ audience, scope string } + +// recordingTokens mints "|" so the fake agent can check what +// each request was granted. +type recordingTokens struct { + mu sync.Mutex + grants []grant +} + +func (r *recordingTokens) Token(_ context.Context, audience, scope string) (string, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.grants = append(r.grants, grant{audience, scope}) + return audience + "|" + scope, nil +} + +// fakeAgent is the direct-HTTP agent contract: a hello naming each op's and +// probe's scope, a call, and a streaming capture that completes on dump +// completeAfter (never when zero). Every authenticated route checks the +// bearer carries its scope. +type fakeAgent struct { + apiVersion int + completeAfter int + resource map[string]string + // report, when set, rides every dump and the final reading as X-Telltale-Report. + report string + mu sync.Mutex + dumps int + renewed int + finished bool +} + +func (f *fakeAgent) handler() http.Handler { + mux := http.NewServeMux() + scoped := func(scope string, next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get(agentclient.TokenHeader) != "Bearer "+directTarget+"|"+scope || r.Header.Get("Authorization") != "" { + w.WriteHeader(http.StatusForbidden) + _, _ = io.WriteString(w, `{"error":"want `+scope+`"}`) + return + } + next(w, r) + } + } + mux.HandleFunc("GET /hello", scoped("jvm:read", func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{"apiVersion": f.apiVersion, "protocol": 1, "started": "2026-01-02T03:04:05Z", "runtime": map[string]string{"lang": "java"}, + "resource": f.resource, + "probes": []map[string]any{{"name": "jvm_trace", "checkpoints": true, "mutates": true, "reverts": true, "scope": "jvm:trace"}}, + "ops": []map[string]any{{"name": "jvm.groovy.eval", "mutates": true, "scope": "jvm:eval"}}}) + })) + mux.HandleFunc("GET /catalog", scoped("jvm:read", func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `[{"name":"jvm.groovy.eval","mutates":true,"scope":"jvm:eval"}]`) + })) + mux.HandleFunc("POST /call", scoped("jvm:eval", func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + Expect(json.NewDecoder(r.Body).Decode(&body)).To(Succeed()) + Expect(body).To(HaveKeyWithValue("operation", "jvm.groovy.eval")) + _, _ = io.WriteString(w, `{"value":7}`) + })) + mux.HandleFunc("POST /captures", scoped("jvm:trace", func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"id":"c1","streaming":true,"contentType":"application/json","fileName":"trace.json"}`) + })) + mux.HandleFunc("POST /captures/c1", scoped("jvm:trace", func(w http.ResponseWriter, _ *http.Request) { + f.mu.Lock() + f.dumps++ + complete := f.completeAfter > 0 && f.dumps >= f.completeAfter + f.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Telltale-Complete", map[bool]string{true: "true", false: "false"}[complete]) + f.writeReport(w) + _, _ = io.WriteString(w, `{"events":[]}`) + })) + mux.HandleFunc("POST /captures/c1/renew", scoped("jvm:trace", func(w http.ResponseWriter, _ *http.Request) { + f.mu.Lock() + f.renewed++ + f.mu.Unlock() + w.WriteHeader(http.StatusNoContent) + })) + mux.HandleFunc("DELETE /captures/c1", scoped("jvm:trace", func(w http.ResponseWriter, _ *http.Request) { + f.mu.Lock() + f.finished = true + f.mu.Unlock() + f.writeReport(w) + _, _ = io.WriteString(w, `{"events":["final"]}`) + })) + return mux +} + +func (f *fakeAgent) writeReport(w http.ResponseWriter) { + if f.report != "" { + w.Header().Set(agentclient.ReportHeader, f.report) + } +} + +func dialFake(fake *fakeAgent, tokens agentclient.TokenSource) (*agentclient.Agent, error) { + server := httptest.NewServer(fake.handler()) + DeferCleanup(server.Close) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return agentclient.DialHTTP(ctx, agentclient.HTTPEndpoint{ID: directTarget, URL: server.URL, Client: server.Client(), Tokens: tokens, Resource: map[string]string{"telltale.deployer": "orders"}, Poll: 10 * time.Millisecond}) +} + +var _ = Describe("Direct HTTP agents", func() { + DescribeTable("preserves the live JVM resource alongside discovery attributes", func(discovered map[string]string) { + live := map[string]string{"service.name": "orders-live", "process.pid": "712", "telltale.source.instance": "jvm-instance", "deployment.environment.name": "integration"} + server := httptest.NewServer((&fakeAgent{apiVersion: agentclient.AgentAPIVersion, resource: live}).handler()) + DeferCleanup(server.Close) + agent, err := agentclient.DialHTTP(context.Background(), agentclient.HTTPEndpoint{ID: directTarget, URL: server.URL, Client: server.Client(), Tokens: &recordingTokens{}, Resource: discovered}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(agent.Close) + expected := map[string]string{"service.name": "orders-live", "process.pid": "712", "telltale.source.instance": "jvm-instance", "deployment.environment.name": "integration"} + if discovered != nil { + expected["telltale.deployer"] = "orders" + Expect(discovered).To(Equal(map[string]string{"telltale.deployer": "orders", "service.name": "orders-discovered", "telltale.source.instance": "stale-instance"})) + } + Expect(agent.Hello.Resource).To(Equal(expected)) + Expect(agent.Hello.Service).To(Equal("orders-live")) + }, Entry("without discovery attributes", nil), Entry("with discovery attributes and stale runtime values", map[string]string{"telltale.deployer": "orders", "service.name": "orders-discovered", "telltale.source.instance": "stale-instance"})) + + It("identifies a deployed agent by its target and asks each call for the scope the agent names", func() { + tokens := &recordingTokens{} + agent, err := dialFake(&fakeAgent{apiVersion: agentclient.AgentAPIVersion}, tokens) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(agent.Close) + Expect(agent.Hello.AgentID).To(Equal(directTarget)) + Expect(agent.Hello.Resource).To(HaveKeyWithValue("telltale.deployer", "orders")) + Expect(agent.HasProbe("jvm_trace")).To(BeTrue()) + value, err := agent.Call(context.Background(), agentclient.CallRequest{Op: "jvm.groovy.eval", Params: json.RawMessage(`{"source":"3+4"}`), AllowMutation: true}) + Expect(err).ToNot(HaveOccurred()) + Expect(value).To(MatchJSON(`{"value":7}`)) + _, err = agent.Call(context.Background(), agentclient.CallRequest{Op: "jvm.unknown"}) + Expect(err).To(MatchError(ContainSubstring(`does not advertise operation "jvm.unknown"`))) + Expect(tokens.grants).To(ContainElements(grant{directTarget, "jvm:read"}, grant{directTarget, "jvm:eval"})) + }) + + It("refuses an agent speaking another agent API", func() { + _, err := dialFake(&fakeAgent{apiVersion: agentclient.AgentAPIVersion + 1}, &recordingTokens{}) + Expect(err).To(MatchError(ContainSubstring("agent API"))) + }) + + It("streams a capture's dumps as blobs and ends it complete when the agent says so", func() { + fake := &fakeAgent{apiVersion: agentclient.AgentAPIVersion, completeAfter: 3} + agent, err := dialFake(fake, &recordingTokens{}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(agent.Close) + session, err := agent.Arm(context.Background(), agentclient.ArmRequest{Probe: "jvm_trace", Params: json.RawMessage(`{}`), Mutates: true, AllowMutation: true}) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ID).To(Equal("c1"), "the session must use the authenticated capture identity returned by the agent") + var blob agentclient.Blob + Eventually(session.Blobs, 2*time.Second).Should(Receive(&blob)) + Expect(blob.Info).To(And(HaveField("Name", "trace.json"), HaveField("ContentType", "application/json"), HaveField("Session", session.ID))) + var ended protocol.Ended + Eventually(session.Ended, 2*time.Second).Should(Receive(&ended)) + Expect(ended).To(And(HaveField("Complete", true), HaveField("Error", ""))) + Expect(session.Disarm(context.Background())).To(Succeed()) + fake.mu.Lock() + defer fake.mu.Unlock() + Expect(fake.finished).To(BeTrue(), "a completed capture is finished on the agent") + }) + + It("delivers the capture's X-Telltale-Report as a session report, once per change", func() { + const report = `{"filtered":41,"omitted":3}` + fake := &fakeAgent{apiVersion: agentclient.AgentAPIVersion, completeAfter: 3, report: report} + agent, err := dialFake(fake, &recordingTokens{}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(agent.Close) + session, err := agent.Arm(context.Background(), agentclient.ArmRequest{Probe: "jvm_trace", Params: json.RawMessage(`{}`), Mutates: true, AllowMutation: true}) + Expect(err).ToNot(HaveOccurred()) + var ended protocol.Ended + Eventually(session.Ended, 2*time.Second).Should(Receive(&ended)) + Expect(ended).To(And(HaveField("Complete", true), HaveField("Error", ""))) + Expect(session.Reports).To(HaveLen(1), "three dumps and the final reading carried one unchanged report") + Expect(<-session.Reports).To(And(HaveField("Session", session.ID), HaveField("Summary", MatchJSON(report)))) + }) + + It("ends a capture with an error when its X-Telltale-Report is not a JSON object", func() { + fake := &fakeAgent{apiVersion: agentclient.AgentAPIVersion, report: `filtered=41`} + agent, err := dialFake(fake, &recordingTokens{}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(agent.Close) + session, err := agent.Arm(context.Background(), agentclient.ArmRequest{Probe: "jvm_trace", Params: json.RawMessage(`{}`), Mutates: true, AllowMutation: true}) + Expect(err).ToNot(HaveOccurred()) + var ended protocol.Ended + Eventually(session.Ended, 2*time.Second).Should(Receive(&ended)) + Expect(ended).To(And(HaveField("Complete", false), HaveField("Error", ContainSubstring("malformed X-Telltale-Report")))) + Expect(session.Reports).To(BeEmpty()) + }) + + It("answers a checkpoint with a dump and returns the final dump on disarm", func() { + fake := &fakeAgent{apiVersion: agentclient.AgentAPIVersion} + agent, err := dialFake(fake, &recordingTokens{}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(agent.Close) + session, err := agent.Arm(context.Background(), agentclient.ArmRequest{Probe: "jvm_trace", Params: json.RawMessage(`{}`), Mutates: true, AllowMutation: true}) + Expect(err).ToNot(HaveOccurred()) + checkpoint, err := session.Checkpoint(context.Background(), json.RawMessage(`{"reason":"step"}`)) + Expect(err).ToNot(HaveOccurred()) + Expect(checkpoint.Blobs).To(HaveLen(1)) + Expect(checkpoint.Blobs[0].CheckpointParams).To(MatchJSON(`{"reason":"step"}`)) + Expect(session.Disarm(context.Background())).To(Succeed()) + var final agentclient.Blob + Eventually(session.Blobs).Should(Receive(&final)) + for strings.Contains(string(final.Data), `"events":[]`) { + Eventually(session.Blobs).Should(Receive(&final)) + } + Expect(final.Data).To(MatchJSON(`{"events":["final"]}`)) + }) +}) diff --git a/pkg/agentclient/hub.go b/pkg/agentclient/hub.go index 6236e11..0331123 100644 --- a/pkg/agentclient/hub.go +++ b/pkg/agentclient/hub.go @@ -29,16 +29,13 @@ func (h *Hub) List(w http.ResponseWriter, r *http.Request) { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return } - if err := h.registry.Discover(r.Context(), ""); err != nil { + infos, err := h.registry.Inventory(r.Context()) + if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - hellos := make([]protocol.Hello, 0) - for _, agent := range h.registry.Match(nil) { - hellos = append(hellos, agent.Hello) - } w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(hellos) + _ = json.NewEncoder(w).Encode(infos) } func (h *Hub) ServeHTTP(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/agentclient/hub_inventory_test.go b/pkg/agentclient/hub_inventory_test.go new file mode 100644 index 0000000..1d9fc0d --- /dev/null +++ b/pkg/agentclient/hub_inventory_test.go @@ -0,0 +1,32 @@ +package agentclient_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + + "github.com/flanksource/telltale/pkg/agentclient" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("agent inventory HTTP API", func() { + It("serves saved targets with the same connection state and namespace group as the CLI", func() { + registry := agentclient.NewRegistry() + registry.Dir = filepath.Join(GinkgoT().TempDir(), "no-local-agents") + registry.SetLister("deployer/", func(context.Context) ([]agentclient.Known, error) { + return []agentclient.Known{{ID: "deployer/orders/shop/orders-0", Runtime: "java", Names: []string{"orders"}, State: "deployed", + Resource: map[string]string{"k8s.namespace.name": "shop"}, Endpoint: "http://orders-0:7070"}}, nil + }) + hub := agentclient.NewHub(registry, agentclient.NewTokenStore(nil)) + response := httptest.NewRecorder() + hub.List(response, httptest.NewRequest(http.MethodGet, "/api/agents", nil)) + Expect(response.Code).To(Equal(http.StatusOK)) + var infos []agentclient.Info + Expect(json.Unmarshal(response.Body.Bytes(), &infos)).To(Succeed()) + Expect(infos).To(ConsistOf(And(HaveField("AgentID", "deployer/orders/shop/orders-0"), + HaveField("Group", "namespace/shop"), HaveField("Connected", false), HaveField("Known.State", "deployed")))) + }) +}) diff --git a/pkg/agentclient/inventory.go b/pkg/agentclient/inventory.go new file mode 100644 index 0000000..36d5376 --- /dev/null +++ b/pkg/agentclient/inventory.go @@ -0,0 +1,72 @@ +package agentclient + +import ( + "context" + "fmt" + + "github.com/flanksource/telltale/sdk/protocol" +) + +// Info describes a connected agent or a known target without inventing a handshake. +type Info struct { + *protocol.Hello + AgentID string `json:"agentId"` + Resource map[string]string `json:"resource"` + Connected bool `json:"connected"` + Group string `json:"group"` + Known *Known `json:"known,omitempty"` +} + +func ConnectedInfo(hello protocol.Hello) Info { + return Info{Hello: &hello, AgentID: hello.AgentID, Resource: hello.Resource, Connected: true, Group: inventoryGroup(hello.Resource)} +} + +func (r *Registry) Inventory(ctx context.Context) ([]Info, error) { + if err := r.Discover(ctx, ""); err != nil { + return nil, err + } + candidates, err := r.candidates(ctx, Want{}) + if err != nil { + return nil, err + } + infos := make([]Info, 0, len(candidates)) + seen := map[string]bool{} + for _, candidate := range candidates { + if candidate.id == "" || seen[candidate.id] { + return nil, fmt.Errorf("agent inventory has an empty or duplicate id %q", candidate.id) + } + seen[candidate.id] = true + if candidate.agent != nil { + infos = append(infos, ConnectedInfo(candidate.agent.Hello)) + continue + } + if candidate.known == nil { + return nil, fmt.Errorf("agent %q has no connection or known metadata", candidate.id) + } + infos = append(infos, Info{AgentID: candidate.id, Resource: candidate.known.Resource, Known: candidate.known, Group: inventoryGroup(candidate.known.Resource)}) + } + return infos, nil +} + +func inventoryGroup(resource map[string]string) string { + if namespace := resource["k8s.namespace.name"]; namespace != "" { + return "namespace/" + namespace + } + if provider := resource["cloud.provider"]; provider != "" { + return "cloud/" + provider + "/" + resource["cloud.region"] + } + if host := resource["host.name"]; host != "" { + return "host/" + host + } + return "other" +} + +type Groups map[string][]Info + +func GroupInventory(infos []Info) Groups { + groups := Groups{} + for _, info := range infos { + groups[info.Group] = append(groups[info.Group], info) + } + return groups +} diff --git a/pkg/agentclient/inventory_pretty.go b/pkg/agentclient/inventory_pretty.go new file mode 100644 index 0000000..865995b --- /dev/null +++ b/pkg/agentclient/inventory_pretty.go @@ -0,0 +1,47 @@ +package agentclient + +import ( + "maps" + "slices" + "strings" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" +) + +func (groups Groups) Pretty() api.Text { + if len(groups) == 0 { + return clicky.Text("No agents found.", "muted") + } + text := api.Text{} + for _, group := range slices.Sorted(maps.Keys(groups)) { + text = text.NewLine().AddText(group, "font-bold").NewLine().Add(api.NewTableFrom(groups[group])) + } + return text +} + +func (i Info) Columns() []api.ColumnDef { + return []api.ColumnDef{ + clicky.Column("agentId").Label("Agent").MaxWidth(60).Build(), + clicky.Column("runtime").Label("Runtime").Build(), + clicky.Column("connection").Label("Connection").Build(), + clicky.Column("deployment").Label("Deployment").Build(), + clicky.Column("endpoint").Label("Endpoint").MaxWidth(60).Build(), + } +} + +func (i Info) Row() map[string]any { + row := map[string]any{"agentId": i.AgentID, "connection": "Not connected"} + if i.Connected { + row["connection"] = "Connected" + } + if i.Hello != nil { + row["runtime"] = strings.TrimSpace(i.Runtime.Lang + " " + i.Runtime.Version) + } + if i.Known != nil { + row["runtime"] = i.Known.Runtime + row["deployment"] = i.Known.State + row["endpoint"] = i.Known.Endpoint + } + return row +} diff --git a/pkg/agentclient/inventory_test.go b/pkg/agentclient/inventory_test.go new file mode 100644 index 0000000..1070559 --- /dev/null +++ b/pkg/agentclient/inventory_test.go @@ -0,0 +1,85 @@ +package agentclient + +import ( + "context" + "encoding/json" + "errors" + "path/filepath" + "time" + + "github.com/flanksource/telltale/sdk/protocol" + . "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +var _ = Describe("agent inventory", func() { + const target = "deployer/orders/shop/orders-0" + var registry *Registry + var resolved int + BeforeEach(func() { + registry = NewRegistry() + registry.Dir = filepath.Join(GinkgoT().TempDir(), "no-local-agents") + resolved = 0 + registry.SetResolver("deployer/", func(context.Context, *Registry, string) error { + resolved++ + return errors.New("inventory must not connect targets") + }) + registry.SetLister("deployer/", func(context.Context) ([]Known, error) { + return []Known{{ID: target, Runtime: "java", Names: []string{"orders"}, Resource: map[string]string{"k8s.namespace.name": "shop"}, + State: "deployed", Endpoint: "http://orders-0:7070"}}, nil + }) + }) + + It("lists a deployed target with namespace and deployment metadata without connecting it", func() { + infos, err := registry.Inventory(context.Background()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + data, err := json.Marshal(infos) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(data).To(gomega.MatchJSON(`[{"agentId":"deployer/orders/shop/orders-0","resource":{"k8s.namespace.name":"shop"},"connected":false,"group":"namespace/shop","known":{"agentId":"deployer/orders/shop/orders-0","runtime":"java","names":["orders"],"resource":{"k8s.namespace.name":"shop"},"state":"deployed","endpoint":"http://orders-0:7070"}}]`)) + gomega.Expect(resolved).To(gomega.BeZero()) + }) + + It("prefers a live handshake and lists a connected deployed agent once", func() { + hello := protocol.Hello{AgentID: target, Service: "orders-live", Runtime: protocol.Runtime{Lang: "java", Version: "21"}, + Started: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), Resource: map[string]string{"k8s.namespace.name": "shop"}} + agent := &Agent{Hello: hello, done: make(chan struct{})} + gomega.Expect(registry.Add(agent)).To(gomega.Succeed()) + DeferCleanup(func() { close(agent.done) }) + infos, err := registry.Inventory(context.Background()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(infos).To(gomega.Equal([]Info{{Hello: &hello, AgentID: target, Resource: map[string]string{"k8s.namespace.name": "shop"}, Connected: true, Group: "namespace/shop"}})) + gomega.Expect(resolved).To(gomega.BeZero()) + }) + + It("reports lister failures instead of returning an empty list", func() { + registry.SetLister("deployer/", func(context.Context) ([]Known, error) { return nil, errors.New("deployer store is unreadable") }) + _, err := registry.Inventory(context.Background()) + gomega.Expect(err).To(gomega.MatchError(gomega.ContainSubstring("deployer store is unreadable"))) + }) + + DescribeTable("rejects invalid known identities", + func(known []Known) { + registry.SetLister("deployer/", func(context.Context) ([]Known, error) { return known, nil }) + _, err := registry.Inventory(context.Background()) + gomega.Expect(err).To(gomega.MatchError(gomega.ContainSubstring("empty or duplicate id"))) + }, + Entry("empty id", []Known{{Runtime: "java"}}), + Entry("duplicate id", []Known{{ID: target}, {ID: target}}), + ) + + DescribeTable("assigns the same group to connected and saved agents", + func(resource map[string]string, group string) { + gomega.Expect(ConnectedInfo(protocol.Hello{Resource: resource}).Group).To(gomega.Equal(group)) + registry.SetLister("deployer/", func(context.Context) ([]Known, error) { + return []Known{{ID: target, Resource: resource}}, nil + }) + infos, err := registry.Inventory(context.Background()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(GroupInventory(infos)).To(gomega.HaveKeyWithValue(group, infos)) + }, + Entry("namespace takes priority", map[string]string{"k8s.namespace.name": "shop", "cloud.provider": "aws", "host.name": "worker"}, "namespace/shop"), + Entry("cloud", map[string]string{"cloud.provider": "aws", "cloud.region": "eu-west-1", "host.name": "worker"}, "cloud/aws/eu-west-1"), + Entry("host", map[string]string{"host.name": "worker"}, "host/worker"), + Entry("other", map[string]string{}, "other"), + ) +}) diff --git a/pkg/agentclient/lookup.go b/pkg/agentclient/lookup.go new file mode 100644 index 0000000..8848607 --- /dev/null +++ b/pkg/agentclient/lookup.go @@ -0,0 +1,178 @@ +package agentclient + +import ( + "context" + "fmt" + "slices" + "strings" +) + +// Known is an agent a resolver can connect but that is not connected yet, +// such as a deployed target, with the short names it answers to. +type Known struct { + ID string `json:"agentId"` + // Runtime is the agent's language, when known before it connects. + Runtime string `json:"runtime,omitempty"` + Names []string `json:"names,omitempty"` + Resource map[string]string `json:"resource,omitempty"` + State string `json:"state,omitempty"` + Endpoint string `json:"endpoint,omitempty"` +} + +// Lister lists the agents its resolver can connect. +type Lister func(ctx context.Context) ([]Known, error) + +// SetLister makes lister name the agents the resolver of prefix connects, so +// a lookup can find them by a short name before they are connected. +func (r *Registry) SetLister(prefix string, lister Lister) { + if prefix == "" || lister == nil { + panic("agentclient: SetLister needs a prefix and a lister") + } + r.listMu.Lock() + defer r.listMu.Unlock() + if r.listers == nil { + r.listers = map[string]Lister{} + } + r.listers[prefix] = lister +} + +// Want narrows a lookup to the agents that can serve it. +type Want struct { + // Probe is a probe the agent must offer. + Probe string + // Runtime is the agent's language, such as java. + Runtime string +} + +func (w Want) String() string { + switch { + case w.Probe != "": + return "agent offering " + w.Probe + case w.Runtime != "": + return w.Runtime + " agent" + } + return "agent" +} + +// candidate is a connected or known agent and the names a query matches. +type candidate struct { + id, label string + names []string + agent *Agent + known *Known +} + +// Lookup is the one agent query names: its id, or a short name (its service, +// pod or deployer, the last part of its id, its pid, or a unique prefix of +// one), among the connected agents and those a resolver can connect. An empty +// query names the only agent there is. A known agent is connected first. +func (r *Registry) Lookup(ctx context.Context, query string, want Want) (*Agent, error) { + if query != "" { + if agent := r.ByID(query); agent != nil { + return agent, nil + } + if claimed, err := r.Resolve(ctx, query); claimed { + if err != nil { + return nil, err + } + return r.ByID(query), nil + } + } + if err := r.Discover(ctx, ""); err != nil { + return nil, err + } + candidates, err := r.candidates(ctx, want) + if err != nil { + return nil, err + } + matches := matching(candidates, query) + switch { + case len(matches) == 1 && matches[0].agent != nil: + return matches[0].agent, nil + case len(matches) == 1: + if _, err := r.Resolve(ctx, matches[0].id); err != nil { + return nil, err + } + if agent := r.ByID(matches[0].id); agent != nil { + return agent, nil + } + return nil, fmt.Errorf("agent %q: no resolver connects it", matches[0].id) + case len(candidates) == 0 && query == "": + return nil, fmt.Errorf("no %s is connected or deployed: attach one with telltale agents install java, or deploy one with telltale jvm deploy", want) + case len(matches) == 0: + return nil, fmt.Errorf("no %s matches %q; agents: %s", want, query, labels(candidates)) + case query == "": + return nil, fmt.Errorf("%d %ss are connected: %s; name one with --agent", len(matches), want, labels(matches)) + } + return nil, fmt.Errorf("agent %q matches %d agents: %s; name one by its id", query, len(matches), labels(matches)) +} + +func (r *Registry) candidates(ctx context.Context, want Want) ([]candidate, error) { + var result []candidate + connected := map[string]bool{} + for _, agent := range r.Match(nil) { + connected[agent.Hello.AgentID] = true + if (want.Probe != "" && !agent.HasProbe(want.Probe)) || (want.Runtime != "" && agent.Hello.Runtime.Lang != want.Runtime) { + continue + } + hello := agent.Hello + names := []string{hello.Service, lastPart(hello.AgentID), hello.Resource["k8s.pod.name"], hello.Resource["telltale.deployer"], hello.Resource["process.pid"]} + result = append(result, candidate{id: hello.AgentID, label: hello.Service, names: names, agent: agent}) + } + r.listMu.Lock() + listers := make([]Lister, 0, len(r.listers)) + for _, lister := range r.listers { + listers = append(listers, lister) + } + r.listMu.Unlock() + for _, lister := range listers { + known, err := lister(ctx) + if err != nil { + return nil, err + } + for _, agent := range known { + if connected[agent.ID] || (want.Runtime != "" && agent.Runtime != "" && agent.Runtime != want.Runtime) { + continue + } + label := "not connected" + if len(agent.Names) > 0 { + label = agent.Names[0] + } + result = append(result, candidate{id: agent.ID, label: label, names: append([]string{lastPart(agent.ID)}, agent.Names...), known: &agent}) + } + } + slices.SortFunc(result, func(a, b candidate) int { return strings.Compare(a.id, b.id) }) + return result, nil +} + +// matching are the candidates query names exactly, else those it prefixes a +// name of; every candidate for an empty query. +func matching(candidates []candidate, query string) []candidate { + if query == "" { + return candidates + } + for _, exact := range []bool{true, false} { + var found []candidate + for _, c := range candidates { + if slices.ContainsFunc(append([]string{c.id}, c.names...), func(name string) bool { + return name != "" && (name == query || (!exact && strings.HasPrefix(name, query))) + }) { + found = append(found, c) + } + } + if len(found) > 0 { + return found + } + } + return nil +} + +func lastPart(id string) string { return id[strings.LastIndex(id, "/")+1:] } + +func labels(candidates []candidate) string { + parts := make([]string, len(candidates)) + for i, c := range candidates { + parts[i] = c.id + " (" + c.label + ")" + } + return strings.Join(parts, ", ") +} diff --git a/pkg/agentclient/lookup_spec_test.go b/pkg/agentclient/lookup_spec_test.go new file mode 100644 index 0000000..7101f5d --- /dev/null +++ b/pkg/agentclient/lookup_spec_test.go @@ -0,0 +1,97 @@ +package agentclient_test + +import ( + "context" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/sdk/protocol" +) + +var _ = Describe("agent lookup", func() { + const deployed = "deployer/checkout/shop/checkout-7d9f-x2kq" + + java := func(id, service string, resource map[string]string) *agentclient.Agent { + return &agentclient.Agent{Hello: protocol.Hello{AgentID: id, Service: service, Runtime: protocol.Runtime{Lang: "java"}, Resource: resource, + Probes: []protocol.ProbeSpec{{Name: "jvm_trace"}, {Name: "jvm_jfr"}}}} + } + registryWith := func(agents ...*agentclient.Agent) *agentclient.Registry { + registry := agentclient.NewRegistry() + registry.Dir = filepath.Join(GinkgoT().TempDir(), "no-local-agents") + for _, agent := range agents { + Expect(registry.Add(agent)).To(Succeed()) + } + return registry + } + // withDeployed lets the registry list and connect one deployed target. + withDeployed := func(registry *agentclient.Registry) *agentclient.Registry { + registry.SetResolver("deployer/", func(_ context.Context, registry *agentclient.Registry, id string) error { + return registry.Add(java(id, "checkout", map[string]string{"k8s.pod.name": "checkout-7d9f-x2kq", "telltale.deployer": "checkout"})) + }) + registry.SetLister("deployer/", func(context.Context) ([]agentclient.Known, error) { + return []agentclient.Known{{ID: deployed, Runtime: "java", Names: []string{"checkout", "checkout-7d9f-x2kq", "shop/checkout-7d9f-x2kq"}}}, nil + }) + return registry + } + lookup := func(registry *agentclient.Registry, query string, want agentclient.Want) (string, error) { + agent, err := registry.Lookup(context.Background(), query, want) + if err != nil { + return "", err + } + return agent.Hello.AgentID, nil + } + + It("picks the only agent there is when none is named", func() { + Expect(lookup(registryWith(java("52234", "idle-smoke", nil)), "", agentclient.Want{})).To(Equal("52234")) + }) + + It("picks the only agent that offers the probe when none is named", func() { + goAgent := &agentclient.Agent{Hello: protocol.Hello{AgentID: "a-7f3c2e", Service: "gavel", Runtime: protocol.Runtime{Lang: "go"}, Probes: []protocol.ProbeSpec{{Name: "http"}}}} + Expect(lookup(registryWith(goAgent, java("52234", "idle-smoke", nil)), "", agentclient.Want{Probe: "jvm_trace"})).To(Equal("52234")) + }) + + It("connects the only deployed target when none is named and nothing is connected", func() { + Expect(lookup(withDeployed(registryWith()), "", agentclient.Want{Runtime: "java"})).To(Equal(deployed)) + }) + + DescribeTable("finds one agent by a short name", + func(query, want string) { + registry := withDeployed(registryWith(java("52234", "idle-smoke", map[string]string{"process.pid": "52234"}), java("java/orders-1", "orders", nil))) + Expect(lookup(registry, query, agentclient.Want{Runtime: "java"})).To(Equal(want)) + }, + Entry("its exact id", "java/orders-1", "java/orders-1"), + Entry("its service", "idle-smoke", "52234"), + Entry("the last part of its id", "orders-1", "java/orders-1"), + Entry("a deployed target's deployer, connecting it", "checkout", deployed), + Entry("a deployed target's pod", "checkout-7d9f-x2kq", deployed), + Entry("a unique prefix of a name", "idle", "52234"), + Entry("a unique prefix of a pod name", "checkout-7d", deployed), + ) + + It("prefers an exact name over agents it is only a prefix of", func() { + registry := registryWith(java("java/orders", "orders", nil), java("java/orders-batch", "orders-batch", nil)) + Expect(lookup(registry, "orders", agentclient.Want{})).To(Equal("java/orders")) + }) + + It("names every candidate when a short name matches several", func() { + registry := registryWith(java("java/orders-1", "orders", nil), java("java/orders-2", "orders", nil)) + _, err := lookup(registry, "orders", agentclient.Want{}) + Expect(err).To(MatchError(`agent "orders" matches 2 agents: java/orders-1 (orders), java/orders-2 (orders); name one by its id`)) + }) + + It("asks for an agent when several are connected and none is named", func() { + registry := registryWith(java("java/orders-1", "orders", nil), java("java/billing-1", "billing", nil)) + _, err := lookup(registry, "", agentclient.Want{Runtime: "java"}) + Expect(err).To(MatchError(`2 java agents are connected: java/billing-1 (billing), java/orders-1 (orders); name one with --agent`)) + }) + + It("says what it looked for when nothing matches", func() { + _, err := lookup(registryWith(java("52234", "idle-smoke", nil)), "billing", agentclient.Want{Probe: "jvm_trace"}) + Expect(err).To(MatchError(`no agent offering jvm_trace matches "billing"; agents: 52234 (idle-smoke)`)) + _, err = lookup(registryWith(), "", agentclient.Want{Runtime: "java"}) + Expect(err).To(MatchError("no java agent is connected or deployed: attach one with telltale agents install java, or deploy one with telltale jvm deploy")) + }) +}) diff --git a/pkg/agentclient/read_loop.go b/pkg/agentclient/read_loop.go index 2211798..359f2b9 100644 --- a/pkg/agentclient/read_loop.go +++ b/pkg/agentclient/read_loop.go @@ -10,42 +10,41 @@ import ( var ErrProtocol = errors.New("agent protocol error") -func (a *Agent) readLoop() { +func (w *wireTransport) readLoop() { var cause error defer func() { - _ = a.stream.Close() - a.mu.Lock() - a.failure = cause - sessions := make([]*Session, 0, len(a.sessions)) - for _, session := range a.sessions { + _ = w.stream.Close() + w.mu.Lock() + sessions := make([]*Session, 0, len(w.sessions)) + for _, session := range w.sessions { sessions = append(sessions, session) } - a.mu.Unlock() + w.mu.Unlock() for _, session := range sessions { session.finish(protocol.Ended{Session: session.ID, Error: cause.Error()}) } - close(a.done) + w.agent.fail(cause) }() for { - frame, data, err := a.conn.Read() + frame, data, err := w.conn.Read() if err != nil { cause = fmt.Errorf("agent connection closed: %w", err) return } - if err := a.dispatchFrame(frame, data); err != nil { + if err := w.dispatchFrame(frame, data); err != nil { cause = err return } } } -func (a *Agent) dispatchFrame(frame protocol.Envelope, data []byte) error { +func (w *wireTransport) dispatchFrame(frame protocol.Envelope, data []byte) error { if frame.Type == protocol.TypeResult { - return a.dispatchResult(frame.Body) + return w.dispatchResult(frame.Body) } if frame.Type == protocol.TypePong { select { - case a.pong <- struct{}{}: + case w.pong <- struct{}{}: default: } return nil @@ -59,35 +58,35 @@ func (a *Agent) dispatchFrame(frame protocol.Envelope, data []byte) error { if frame.Type == protocol.TypeArmed && event.Session == "" { return fmt.Errorf("%w: armed frame has no session", ErrProtocol) } - a.mu.Lock() - session := a.sessions[event.Session] - a.mu.Unlock() + w.mu.Lock() + session := w.sessions[event.Session] + w.mu.Unlock() if session == nil { return nil } - return a.dispatchSession(session, frame, data) + return w.dispatchSession(session, frame, data) } -func (a *Agent) dispatchResult(raw json.RawMessage) error { +func (w *wireTransport) dispatchResult(raw json.RawMessage) error { var result protocol.Result if err := json.Unmarshal(raw, &result); err != nil { return fmt.Errorf("%w: decode result: %w", ErrProtocol, err) } - a.mu.Lock() - call := a.calls[result.CallID] - if a.callSeen[result.CallID] { - a.mu.Unlock() + w.mu.Lock() + call := w.calls[result.CallID] + if w.callSeen[result.CallID] { + w.mu.Unlock() return fmt.Errorf("%w: duplicate result %q", ErrProtocol, result.CallID) } - a.callSeen[result.CallID] = true - a.mu.Unlock() + w.callSeen[result.CallID] = true + w.mu.Unlock() if call != nil && !deliver(call, result) { return fmt.Errorf("%w: duplicate result %q", ErrProtocol, result.CallID) } return nil } -func (a *Agent) dispatchSession(s *Session, frame protocol.Envelope, data []byte) error { +func (w *wireTransport) dispatchSession(s *Session, frame protocol.Envelope, data []byte) error { switch frame.Type { case protocol.TypeArmed: return s.receiveArmed(frame.Body) @@ -119,9 +118,7 @@ func (a *Agent) dispatchSession(s *Session, frame protocol.Envelope, data []byte return fmt.Errorf("%w: decode ended: %w", ErrProtocol, err) } s.finish(value) - a.mu.Lock() - delete(a.sessions, s.ID) - a.mu.Unlock() + w.forget(s) default: return fmt.Errorf("%w: unexpected frame %q", ErrProtocol, frame.Type) } diff --git a/pkg/agentclient/registry.go b/pkg/agentclient/registry.go index ffd0b27..1dbef68 100644 --- a/pkg/agentclient/registry.go +++ b/pkg/agentclient/registry.go @@ -19,6 +19,52 @@ type Registry struct { discoveryMu sync.Mutex agents map[string]*Agent subscribers map[chan *Agent]struct{} + resolveMu sync.Mutex + resolvers map[string]Resolver + listMu sync.Mutex + listers map[string]Lister +} + +// Resolver connects, and adds to registry, the agent id names when local +// discovery cannot find it, such as a deployed target reached over HTTP. +type Resolver func(ctx context.Context, registry *Registry, id string) error + +// SetResolver makes resolver answer every id starting with prefix, +// replacing the one set for prefix before. +func (r *Registry) SetResolver(prefix string, resolver Resolver) { + if prefix == "" || resolver == nil { + panic("agentclient: SetResolver needs a prefix and a resolver") + } + r.resolveMu.Lock() + defer r.resolveMu.Unlock() + if r.resolvers == nil { + r.resolvers = map[string]Resolver{} + } + r.resolvers[prefix] = resolver +} + +// Resolve connects the agent id names through the resolver of its prefix, +// unless it is already connected; claimed is false for an id no resolver +// answers, which only discovery can find. +func (r *Registry) Resolve(ctx context.Context, id string) (claimed bool, err error) { + r.resolveMu.Lock() + defer r.resolveMu.Unlock() + for prefix, resolver := range r.resolvers { + if !strings.HasPrefix(id, prefix) { + continue + } + if r.ByID(id) != nil { + return true, nil + } + if err := resolver(ctx, r, id); err != nil { + return true, fmt.Errorf("connect agent %q: %w", id, err) + } + if r.ByID(id) == nil { + return true, fmt.Errorf("connect agent %q: its resolver added no agent of that id", id) + } + return true, nil + } + return false, nil } // NewRegistry creates an empty registry. diff --git a/pkg/agentclient/wire.go b/pkg/agentclient/wire.go new file mode 100644 index 0000000..1e8606c --- /dev/null +++ b/pkg/agentclient/wire.go @@ -0,0 +1,181 @@ +package agentclient + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net" + "sync" + "time" + + "github.com/flanksource/telltale/sdk/protocol" +) + +// wireTransport is the frame protocol over a local socket or the hub's +// WebSocket: one stream, multiplexed by session and call id. +type wireTransport struct { + agent *Agent + stream net.Conn + conn *protocol.Conn + mu sync.Mutex + sessions map[string]*Session + calls map[string]chan protocol.Result + callSeen map[string]bool + pong chan struct{} +} + +func newWireAgent(client *protocol.Client) *Agent { + agent := &Agent{Hello: client.Hello, done: make(chan struct{})} + wire := &wireTransport{agent: agent, stream: client.Stream, conn: client.Conn, sessions: make(map[string]*Session), calls: make(map[string]chan protocol.Result), callSeen: make(map[string]bool), pong: make(chan struct{}, 1)} + agent.transport = wire + go wire.readLoop() + go wire.heartbeat() + return agent +} + +func (w *wireTransport) heartbeat() { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + if err := write(w.conn, "ping", protocol.Ping{}); err != nil { + _ = w.stream.Close() + return + } + timer := time.NewTimer(10 * time.Second) + select { + case <-w.pong: + timer.Stop() + case <-timer.C: + _ = w.stream.Close() + return + case <-w.agent.done: + timer.Stop() + return + } + case <-w.agent.done: + return + } + } +} + +func write(conn *protocol.Conn, kind string, body any) error { + data, err := json.Marshal(body) + if err != nil { + return err + } + return conn.Write(protocol.Envelope{Type: kind, Body: data}, nil) +} + +func (w *wireTransport) forget(s *Session) { + w.mu.Lock() + delete(w.sessions, s.ID) + w.mu.Unlock() +} + +func (w *wireTransport) arm(ctx context.Context, s *Session, request ArmRequest) error { + w.mu.Lock() + w.sessions[s.ID] = s + w.mu.Unlock() + if err := write(w.conn, "arm", protocol.Arm{Session: s.ID, Probe: request.Probe, Params: request.Params, AllowMutation: request.AllowMutation, StopAt: request.StopAt, Limits: request.Limits}); err != nil { + w.forget(s) + return err + } + select { + case reply := <-s.armed: + if reply.Error != "" { + w.forget(s) + return errors.New(reply.Error) + } + return nil + case <-ctx.Done(): + w.forget(s) + go func() { + if err := write(w.conn, protocol.TypeDisarm, protocol.Disarm{Session: s.ID}); err != nil { + _ = w.stream.Close() + } + }() + return ctx.Err() + case <-w.agent.done: + return fmt.Errorf("arm: %w", w.agent.connectionError()) + } +} + +func (w *wireTransport) checkpoint(ctx context.Context, s *Session, params json.RawMessage) (CheckpointResult, error) { + var random [16]byte + if _, err := rand.Read(random[:]); err != nil { + return CheckpointResult{}, err + } + id := hex.EncodeToString(random[:]) + call := &checkpointCall{done: make(chan protocol.Checkpointed, 1), params: bytes.Clone(params)} + s.mu.Lock() + s.pending[id] = call + s.mu.Unlock() + defer func() { s.mu.Lock(); delete(s.pending, id); s.mu.Unlock() }() + if err := write(w.conn, "checkpoint", protocol.Checkpoint{Session: s.ID, ID: id, Params: params}); err != nil { + return CheckpointResult{}, fmt.Errorf("checkpoint: %w", err) + } + select { + case done := <-call.done: + return checkpointed(call, done) + case <-s.finished: + select { + case done := <-call.done: + return checkpointed(call, done) + default: + } + return CheckpointResult{}, s.endError() + case <-w.agent.done: + return CheckpointResult{}, fmt.Errorf("checkpoint: %w", w.agent.connectionError()) + case <-ctx.Done(): + return CheckpointResult{}, ctx.Err() + } +} + +func checkpointed(call *checkpointCall, done protocol.Checkpointed) (CheckpointResult, error) { + if done.Error != "" { + return call.result, fmt.Errorf("agent checkpoint: %s", done.Error) + } + return call.result, nil +} + +func (w *wireTransport) disarm(ctx context.Context, s *Session) error { + if err := write(w.conn, "disarm", protocol.Disarm{Session: s.ID}); err != nil { + return fmt.Errorf("disarm: %w", err) + } + return s.awaitRelease(ctx) +} + +func (w *wireTransport) call(ctx context.Context, request CallRequest) (json.RawMessage, error) { + var id [16]byte + if _, err := rand.Read(id[:]); err != nil { + return nil, err + } + callID := hex.EncodeToString(id[:]) + response := make(chan protocol.Result, 1) + w.mu.Lock() + w.calls[callID] = response + w.mu.Unlock() + defer func() { w.mu.Lock(); delete(w.calls, callID); w.mu.Unlock() }() + if err := write(w.conn, "call", protocol.Call{CallID: callID, Op: request.Op, Params: request.Params, AllowMutation: request.AllowMutation}); err != nil { + return nil, fmt.Errorf("agent call: %w", err) + } + select { + case result := <-response: + if result.Error != "" { + return nil, fmt.Errorf("agent call %s: %s", request.Op, result.Error) + } + return result.Value, nil + case <-w.agent.done: + return nil, fmt.Errorf("call: %w", w.agent.connectionError()) + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (w *wireTransport) close() error { return w.stream.Close() } diff --git a/pkg/agentinstall/filters_spec_test.go b/pkg/agentinstall/filters_spec_test.go new file mode 100644 index 0000000..612559e --- /dev/null +++ b/pkg/agentinstall/filters_spec_test.go @@ -0,0 +1,104 @@ +package agentinstall_test + +import ( + "context" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/agentinstall" + "github.com/flanksource/telltale/pkg/tracing/jvm" + "github.com/flanksource/telltale/pkg/tracing/jvmtrace" + "github.com/flanksource/telltale/sdk/protocol" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Java diagnostics on a real JVM", func() { + var agent *agentclient.Agent + var ctx context.Context + + BeforeEach(func() { + directory := GinkgoT().TempDir() + output, err := exec.Command("javac", "-d", directory, "testdata/Workload.java").CombinedOutput() + Expect(err).NotTo(HaveOccurred(), string(output)) + child := exec.Command("java", "-cp", directory, "Workload") + Expect(child.Start()).To(Succeed()) + DeferCleanup(func() { _ = child.Process.Kill(); _ = child.Wait() }) + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) + DeferCleanup(cancel) + options := agentinstall.Options{Destination: agentinstall.Destination{Kind: "local"}, PID: child.Process.Pid, Directory: filepath.Join(directory, "agents"), Service: "fixture-jvm", AllowMutation: true} + _, err = (agentinstall.Installer{Runtime: agentinstall.Java{}}).Install(ctx, options) + Expect(err).NotTo(HaveOccurred()) + root, err := os.UserConfigDir() + Expect(err).NotTo(HaveOccurred()) + registry := agentclient.NewRegistry() + Expect(registry.Discover(ctx, filepath.Join(root, "telltale", "agents"))).To(Succeed()) + agent = registry.ByPID(child.Process.Pid) + Expect(agent).NotTo(BeNil()) + DeferCleanup(agent.Close) + }) + + It("publishes only the root calls whose argument matches and reports how many it filtered", func() { + Expect(agent.Hello.Probes).To(ContainElement(And( + HaveField("Name", jvmtrace.Kind), + HaveField("Version", BeNumerically(">=", jvmtrace.FilterProbeAPI))))) + // Workload's argument passes through a double, so past 2^53 it is only ever + // even: a trailing 0 matches at every magnitude and still filters most calls. + probe, err := agent.Arm(ctx, agentclient.ArmRequest{Probe: jvmtrace.Kind, AllowMutation: true, Params: json.RawMessage( + `{"targets":[{"class":"Workload","method":"compute"}],"args":true,"events":10,"match":[{"on":"arg","index":0,"op":"regex","value":"0$"}]}`)}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = probe.Disarm(context.Background()) }) + + var ended protocol.Ended + Eventually(probe.Ended, 10*time.Second).Should(Receive(&ended)) + Expect(ended.Error).To(BeEmpty()) + Expect(ended.Complete).To(BeTrue(), "the event budget completes the capture") + var events []jvmtrace.Event + for len(probe.Blobs) > 0 { + decoded, err := jvmtrace.Decode(<-probe.Blobs, nil) + Expect(err).NotTo(HaveOccurred()) + events = append(events, decoded...) + } + var enters int + for _, event := range events { + if event.Kind != "enter" { + continue + } + enters++ + var args []struct{ Value string } + Expect(json.Unmarshal(event.Args, &args)).To(Succeed()) + Expect(strings.HasSuffix(args[0].Value, "0")).To(BeTrue(), "an unmatched call was published: %s", event.Args) + } + Expect(enters).To(BeNumerically(">", 0)) + + Expect(probe.Reports).NotTo(BeEmpty(), "the agent reports what the filter withheld") + var report protocol.Report + for len(probe.Reports) > 0 { + report = <-probe.Reports + } + var summary struct{ Filtered, Omitted int64 } + Expect(json.Unmarshal(report.Summary, &summary)).To(Succeed()) + Expect(summary.Filtered).To(BeNumerically(">", 0)) + }) + + It("reads a typed heap histogram through the DiagnosticCommand MBean", func() { + raw, err := agent.Call(ctx, agentclient.CallRequest{Op: jvm.HeapHistogramOp, Params: json.RawMessage(`{"top":5,"classPattern":"java.lang.*"}`)}) + Expect(err).NotTo(HaveOccurred()) + histogram, err := jvm.DecodeHeapHistogram(raw) + Expect(err).NotTo(HaveOccurred()) + Expect(histogram.Rows).To(HaveLen(5)) + for i, row := range histogram.Rows { + Expect(row.ClassName).To(HavePrefix("java.lang.")) + if i > 0 { + Expect(row.Bytes).To(BeNumerically("<=", histogram.Rows[i-1].Bytes)) + } + } + Expect(histogram.Matched).To(BeNumerically(">=", 5)) + }) +}) diff --git a/pkg/agentinstall/install.go b/pkg/agentinstall/install.go index 4be4cb8..77c28d6 100644 --- a/pkg/agentinstall/install.go +++ b/pkg/agentinstall/install.go @@ -1,6 +1,7 @@ package agentinstall import ( + "archive/tar" "bytes" "context" "crypto/rand" @@ -12,6 +13,7 @@ import ( "net/http" "net/url" "os" + posixpath "path" "path/filepath" "strconv" "strings" @@ -21,14 +23,47 @@ import ( ) type Options struct { - Destination Destination `json:"destination"` - PID int `json:"pid"` - Directory string `json:"directory"` - JavaHome string `json:"javaHome,omitempty"` - HubURL string `json:"hubUrl,omitempty"` - TokenFile string `json:"tokenFile,omitempty"` - Service string `json:"service"` - DryRun bool `json:"dryRun,omitempty"` + Destination Destination `json:"destination"` + PID int `json:"pid"` + Directory string `json:"directory"` + JavaHome string `json:"javaHome,omitempty"` + HubURL string `json:"hubUrl,omitempty"` + TokenFile string `json:"tokenFile,omitempty"` + Service string `json:"service"` + DryRun bool `json:"dryRun,omitempty"` + Upgrade bool `json:"upgrade,omitempty"` + AllowMutation bool `json:"allowMutation,omitempty"` + ExtensionsDir string `json:"extensionsDir,omitempty"` + // HTTP starts the agent's HTTP listener, which executions reach directly + // with a token the listener verifies; it needs no hub. + HTTP *HTTPListener `json:"http,omitempty"` +} + +// HTTPListener is the attached agent's direct-HTTP endpoint and the key its +// executions are verified with. It carries no shared secret. +type HTTPListener struct { + Bind string `json:"bind"` + Port int `json:"port"` + PortFile string `json:"portFile,omitempty"` + Issuer string `json:"issuer"` + Audience string `json:"audience"` + JWKS json.RawMessage `json:"jwks,omitempty"` + JWKSURL string `json:"jwksUrl,omitempty"` + ConfigDigest string `json:"configDigest,omitempty"` +} + +func (h HTTPListener) validate() error { + switch { + case h.Bind == "": + return fmt.Errorf("agent HTTP listener bind address is required") + case h.Port < 0 || h.Port > 65535: + return fmt.Errorf("agent HTTP listener port %d is out of range", h.Port) + case h.Issuer == "" || h.Audience == "": + return fmt.Errorf("agent HTTP listener requires an issuer and an audience") + case len(h.JWKS) == 0 && h.JWKSURL == "": + return fmt.Errorf("agent HTTP listener requires jwks or jwksUrl") + } + return nil } type Result struct { @@ -44,14 +79,19 @@ func (o Options) Validate() error { if err := o.Destination.Validate(); err != nil { return err } - if o.PID <= 0 { - return fmt.Errorf("a positive target JVM pid is required") + if o.PID < 0 || (o.PID == 0 && o.Destination.Kind != "kubernetes") { + return fmt.Errorf("a positive target JVM pid is required (Kubernetes targets can discover it from /proc)") } if !strings.HasPrefix(o.Directory, "/") || strings.ContainsAny(o.Directory, "\x00\n\r") { return fmt.Errorf("an absolute installation directory is required") } - if o.Destination.Kind != "local" && (o.HubURL == "" || o.TokenFile == "") { - return fmt.Errorf("remote installation requires hub URL and agent token file") + if o.HTTP != nil { + if err := o.HTTP.validate(); err != nil { + return err + } + } + if o.Destination.Kind != "local" && o.HTTP == nil && (o.HubURL == "" || o.TokenFile == "") { + return fmt.Errorf("remote installation requires an HTTP listener, or a hub URL and agent token file") } if o.HubURL != "" { u, err := url.Parse(o.HubURL) @@ -69,14 +109,17 @@ type Artifact struct { type Attachment struct { PID int Artifact, Config string + JavaHome string + Upgrade bool } type Runtime interface { Artifact() (Artifact, error) Attach(Attachment) []string } type Installer struct { - Runtime Runtime - Transport Transport + Runtime Runtime + Transport Transport + DownloadJDK func(context.Context, JDKRequest) (JDK, error) } func (i Installer) Install(ctx context.Context, o Options) (Result, error) { @@ -114,11 +157,13 @@ func (i Installer) Install(ctx context.Context, o Options) (Result, error) { return result, dialErr } defer func() { _ = client.Close() }() - if client.Hello.Resource["telltale.artifact.sha256"] != result.SHA256 || client.Hello.Service != o.Service || client.Hello.Resource["telltale.hub.url"] != o.HubURL { - return result, fmt.Errorf("an incompatible agent is already installed; restart the JVM before changing its agent") + if !o.Upgrade { + if client.Hello.Resource["telltale.artifact.sha256"] != result.SHA256 || client.Hello.Service != o.Service || client.Hello.Resource["telltale.hub.url"] != o.HubURL || client.Hello.Resource["telltale.allowMutation"] != strconv.FormatBool(o.AllowMutation) || client.Hello.Resource["telltale.extensionsDir"] != o.ExtensionsDir { + return result, fmt.Errorf("an incompatible agent is already installed; use --upgrade to replace its runtime") + } + result.Agent = client.Hello.AgentID + return result, nil } - result.Agent = client.Hello.AgentID - return result, nil } else if !os.IsNotExist(statErr) { return result, statErr } @@ -127,21 +172,35 @@ func (i Installer) Install(ctx context.Context, o Options) (Result, error) { if transport == nil { transport = CommandTransport{Destination: o.Destination} } - identity, err := transport.Run(ctx, []string{"ps", "-o", "lstart=", "-p", strconv.Itoa(o.PID)}, nil) + platform, err := transport.Run(ctx, []string{"uname", "-s"}, nil) if err != nil { - return result, fmt.Errorf("inspect target JVM pid %d: %w", o.PID, err) + return result, err } - if strings.TrimSpace(string(identity)) == "" { - return result, fmt.Errorf("target JVM pid %d does not exist", o.PID) + linux := strings.TrimSpace(string(platform)) == "Linux" + if o.PID == 0 { + if !linux { + return result, fmt.Errorf("automatic Java PID discovery requires Linux /proc") + } + o.PID, err = javaPID(ctx, transport) + if err != nil { + return result, err + } + result.PID = o.PID } - if _, err = transport.Run(ctx, []string{"install", "-d", "-m", "700", dir}, nil); err != nil { + identity, err := processIdentity(ctx, transport, linux, o.PID) + if err != nil { return result, err } - if err = write(ctx, transport, result.Jar, jar); err != nil { + javaHome := o.JavaHome + if linux { + if javaHome, err = i.javaHome(ctx, transport, o); err != nil { + return result, err + } + } + if _, err = transport.Run(ctx, []string{"install", "-d", "-m", "700", dir}, nil); err != nil { return result, err } - platform, err := transport.Run(ctx, []string{"uname", "-s"}, nil) - if err != nil { + if err = write(ctx, transport, result.Jar, jar); err != nil { return result, err } checksumArgs := []string{"sha256sum", result.Jar} @@ -170,13 +229,19 @@ func (i Installer) Install(ctx context.Context, o Options) (Result, error) { return result, fmt.Errorf("agent token is invalid") } } - config := map[string]any{"token": secret, "service": o.Service} + config := map[string]any{"token": secret, "service": o.Service, "allowMutation": o.AllowMutation} + if o.ExtensionsDir != "" { + config["extensionsDir"] = o.ExtensionsDir + } installIdentity := sha256.Sum256(append([]byte(fmt.Sprintf("%+v/%d/", o.Destination, o.PID)), identity...)) installationID := hex.EncodeToString(installIdentity[:]) - config["resource"] = map[string]string{"telltale.install.id": installationID, "telltale.artifact.sha256": result.SHA256, "telltale.hub.url": o.HubURL} + config["resource"] = map[string]string{"telltale.install.id": installationID, "telltale.artifact.sha256": result.SHA256, "telltale.hub.url": o.HubURL, "telltale.allowMutation": strconv.FormatBool(o.AllowMutation), "telltale.extensionsDir": o.ExtensionsDir} if o.HubURL != "" { config["url"] = o.HubURL } + if o.HTTP != nil { + config["http"] = o.HTTP + } if o.Destination.Kind == "local" { root, rootErr := os.UserConfigDir() if rootErr != nil { @@ -192,27 +257,78 @@ func (i Installer) Install(ctx context.Context, o Options) (Result, error) { if err = write(ctx, transport, configPath, body); err != nil { return result, err } - current, err := transport.Run(ctx, []string{"ps", "-o", "lstart=", "-p", strconv.Itoa(o.PID)}, nil) + current, err := processIdentity(ctx, transport, linux, o.PID) if err != nil { return result, err } if !bytes.Equal(identity, current) { return result, fmt.Errorf("target process identity changed before attachment") } - _, err = transport.Run(ctx, i.Runtime.Attach(Attachment{PID: o.PID, Artifact: result.Jar, Config: configPath}), nil) + _, err = transport.Run(ctx, i.Runtime.Attach(Attachment{PID: o.PID, Artifact: result.Jar, Config: configPath, JavaHome: javaHome, Upgrade: o.Upgrade}), nil) if err != nil { return result, fmt.Errorf("attach Java agent (requires compatible JDK Attach API and permission to attach): %w", err) } + if o.Destination.Kind != "local" && o.HubURL == "" { + // A hub-less agent is ready when its listener answers, which only the + // caller can reach: the deployer reads its /health. + return result, nil + } return await(ctx, o, result, installationID) } +// processIdentity tells pid's process from a later one reusing the pid: its +// start time. Linux reads it from /proc//stat (field 22, counted after +// the parenthesised command, which may hold spaces), since slim JVM images +// ship no ps; elsewhere ps reports it. +func processIdentity(ctx context.Context, transport Transport, linux bool, pid int) ([]byte, error) { + args := []string{"ps", "-o", "lstart=", "-p", strconv.Itoa(pid)} + if linux { + args = []string{"cat", "/proc/" + strconv.Itoa(pid) + "/stat"} + } + out, err := transport.Run(ctx, args, nil) + if err != nil { + return nil, fmt.Errorf("inspect target JVM pid %d: %w", pid, err) + } + if !linux { + if len(bytes.TrimSpace(out)) == 0 { + return nil, fmt.Errorf("target JVM pid %d does not exist", pid) + } + return out, nil + } + end := bytes.LastIndexByte(out, ')') + // After the command: state is field 3, so start time (22) is the 20th. + if fields := strings.Fields(string(out[end+1:])); end >= 0 && len(fields) >= 20 { + return []byte(fields[19]), nil + } + return nil, fmt.Errorf("target JVM pid %d: unreadable /proc stat %q", pid, strings.TrimSpace(string(out))) +} + +// targetJavaHome is the JDK a Linux JVM runs from, read from its executable, +// so the attach helper is that JDK's own java (and, for Java 8, finds its +// tools.jar). A Java 8 JDK's jre/bin/java belongs to the JDK above jre. +func targetJavaHome(ctx context.Context, transport Transport, pid int) (string, error) { + out, err := transport.Run(ctx, []string{"readlink", "/proc/" + strconv.Itoa(pid) + "/exe"}, nil) + if err != nil { + return "", fmt.Errorf("find the JDK of JVM pid %d (or pass --java-home): %w", pid, err) + } + exe := strings.TrimSpace(string(out)) + home, ok := strings.CutSuffix(exe, "/bin/java") + if !ok || !strings.HasPrefix(home, "/") { + return "", fmt.Errorf("JVM pid %d runs %q, not a JDK's bin/java: pass --java-home", pid, exe) + } + return strings.TrimSuffix(home, "/jre"), nil +} + func write(ctx context.Context, t Transport, path string, data []byte) error { var nonce [12]byte if _, err := rand.Read(nonce[:]); err != nil { return err } - staged := path + ".stage-" + hex.EncodeToString(nonce[:]) - if _, err := t.Run(ctx, []string{"install", "-m", "600", "/dev/null", staged}, nil); err != nil { + dir, name := posixpath.Split(path) + stagedName := name + ".stage-" + hex.EncodeToString(nonce[:]) + staged := dir + stagedName + archive, err := tarFile(stagedName, data) + if err != nil { return err } defer func() { @@ -220,13 +336,31 @@ func write(ctx context.Context, t Transport, path string, data []byte) error { defer cancel() _, _ = t.Run(cleanup, []string{"rm", "-f", staged}, nil) }() - if _, err := t.Run(ctx, []string{"dd", "of=" + staged}, bytes.NewReader(data)); err != nil { + // A tar stream, as kubectl cp sends one: in a pod, a bare `dd of=` can read + // end of input before the exec attaches its stdin, staging an empty file. + if _, err := t.Run(ctx, []string{"tar", "-x", "-m", "--no-same-owner", "-f", "-", "-C", dir}, bytes.NewReader(archive)); err != nil { return err } - _, err := t.Run(ctx, []string{"mv", "-f", staged, path}, nil) + _, err = t.Run(ctx, []string{"mv", "-f", staged, path}, nil) return err } +// tarFile is an archive of one private file. +func tarFile(name string, data []byte) ([]byte, error) { + var archive bytes.Buffer + writer := tar.NewWriter(&archive) + if err := writer.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(data)), ModTime: time.Now()}); err != nil { + return nil, err + } + if _, err := writer.Write(data); err != nil { + return nil, err + } + if err := writer.Close(); err != nil { + return nil, err + } + return archive.Bytes(), nil +} + func await(ctx context.Context, o Options, result Result, installationID string) (Result, error) { ctx, cancel := context.WithTimeout(ctx, 15*time.Second) defer cancel() diff --git a/pkg/agentinstall/install_spec_test.go b/pkg/agentinstall/install_spec_test.go index 1f1a91e..e7ca8f9 100644 --- a/pkg/agentinstall/install_spec_test.go +++ b/pkg/agentinstall/install_spec_test.go @@ -10,10 +10,15 @@ import ( "testing" "time" + dbcontext "github.com/flanksource/commons-db/context" "github.com/flanksource/telltale/pkg/agentclient" "github.com/flanksource/telltale/pkg/agentinstall" + "github.com/flanksource/telltale/pkg/tracing" "github.com/flanksource/telltale/pkg/tracing/jfr" "github.com/flanksource/telltale/pkg/tracing/jfr/decode" + "github.com/flanksource/telltale/pkg/tracing/jvmtrace" + "github.com/flanksource/telltale/pkg/tracing/tracingtest" + "github.com/flanksource/telltale/sdk/protocol" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -30,7 +35,7 @@ var _ = Describe("Java installation", func() { DeferCleanup(func() { _ = child.Process.Kill(); _ = child.Wait() }) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - options := agentinstall.Options{Destination: agentinstall.Destination{Kind: "local"}, PID: child.Process.Pid, Directory: filepath.Join(directory, "agents"), Service: "fixture-jvm"} + options := agentinstall.Options{Destination: agentinstall.Destination{Kind: "local"}, PID: child.Process.Pid, Directory: filepath.Join(directory, "agents"), Service: "fixture-jvm", AllowMutation: true} installer := agentinstall.Installer{Runtime: agentinstall.Java{}} result, err := installer.Install(ctx, options) Expect(err).NotTo(HaveOccurred()) @@ -44,7 +49,60 @@ var _ = Describe("Java installation", func() { Expect(registry.Discover(ctx, filepath.Join(root, "telltale", "agents"))).To(Succeed()) agent := registry.ByPID(child.Process.Pid) Expect(agent).NotTo(BeNil()) - DeferCleanup(agent.Close) + DeferCleanup(func() error { + if agent == nil { + return nil + } + return agent.Close() + }) + jmx, err := agent.Call(ctx, agentclient.CallRequest{Op: "jvm.jmx.search", Params: json.RawMessage(`{"pattern":"java.lang:type=Runtime"}`)}) + Expect(err).NotTo(HaveOccurred()) + Expect(jmx).To(MatchJSON(`["java.lang:type=Runtime"]`)) + script, err := agent.Call(ctx, agentclient.CallRequest{Op: "jvm.groovy.eval", Params: json.RawMessage(`{"source":"return 3 + 4"}`), AllowMutation: true}) + Expect(err).NotTo(HaveOccurred()) + var evaluated struct { + Value int `json:"value"` + } + Expect(json.Unmarshal(script, &evaluated)).To(Succeed()) + Expect(evaluated.Value).To(Equal(7)) + probe, err := agent.Arm(ctx, agentclient.ArmRequest{Probe: "jvm_trace", AllowMutation: true, Params: json.RawMessage(`{"targets":[{"class":"Workload","method":"compute"}],"args":true,"return":true,"events":10}`)}) + Expect(err).NotTo(HaveOccurred()) + var methodEvents agentclient.Blob + Eventually(probe.Blobs, 5*time.Second).Should(Receive(&methodEvents)) + Expect(string(methodEvents.Data)).To(ContainSubstring(`"method":"compute"`)) + var ended protocol.Ended + Eventually(probe.Ended, 5*time.Second).Should(Receive(&ended)) + Expect(ended.Error).To(BeEmpty()) + Expect(ended.Complete).To(BeTrue()) + Expect(probe.Disarm(ctx)).To(Succeed()) + patches, err := agent.Call(ctx, agentclient.CallRequest{Op: "jvm.classes.patches", Params: json.RawMessage(`{}`)}) + Expect(err).NotTo(HaveOccurred()) + Expect(patches).To(MatchJSON(`[]`)) + deadline, err := agent.Arm(ctx, agentclient.ArmRequest{Probe: "jvm_trace", AllowMutation: true, StopAt: time.Now().Add(300 * time.Millisecond), Params: json.RawMessage(`{"targets":[{"class":"Workload","method":"compute"}],"events":1000000}`)}) + Expect(err).NotTo(HaveOccurred()) + Eventually(deadline.Ended, 5*time.Second).Should(Receive(&ended)) + Expect(ended.Error).To(BeEmpty()) + Expect(ended.Complete).To(BeTrue()) + redefined, err := agent.Arm(ctx, agentclient.ArmRequest{Probe: "jvm_redefine", AllowMutation: true, Params: json.RawMessage(`{"class":"Workload","method":"compute","source":"public static double compute(long value) { return 7.0; }"}`)}) + Expect(err).NotTo(HaveOccurred()) + changed, err := agent.Call(ctx, agentclient.CallRequest{Op: "jvm.ognl.eval", AllowMutation: true, Params: json.RawMessage(`{"expression":"@Workload@compute(1L)"}`)}) + Expect(err).NotTo(HaveOccurred()) + Expect(changed).To(MatchJSON(`{"value":7.0}`)) + Expect(redefined.Disarm(ctx)).To(Succeed()) + restored, err := agent.Call(ctx, agentclient.CallRequest{Op: "jvm.ognl.eval", AllowMutation: true, Params: json.RawMessage(`{"expression":"@Workload@compute(1L)"}`)}) + Expect(err).NotTo(HaveOccurred()) + Expect(restored).To(MatchJSON(`{"value":1.4142135623730951}`)) + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{jvmtrace.RedefineKind: jvmtrace.NewRedefineKind(registry)}) + nativeParams, err := json.Marshal(map[string]any{"target": map[string]any{"seam": "agent", "agent": result.Agent, "allowMutation": true}, "class": "Workload", "method": "compute", "source": "public static double compute(long value) { return 7.0; }"}) + Expect(err).NotTo(HaveOccurred()) + managed, err := env.Runtime.Start(dbcontext.New().Wrap(ctx), tracing.StartRequest{Kind: jvmtrace.RedefineKind, Params: nativeParams}) + Expect(err).NotTo(HaveOccurred()) + tracingtest.Running(managed.Session()) + Expect(env.Runtime.Checkpoint(tracing.WithCheckpointParams(ctx, json.RawMessage(`{"reason":"stop"}`)), managed)).To(Succeed()) + Expect(env.Runtime.Checkpoint(tracing.WithCheckpointParams(ctx, json.RawMessage(`{"unexpected":true}`)), managed)).To(MatchError(tracing.ErrCheckpointParams)) + _, err = managed.Stop(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(env.Rows(tracingtest.Ended(managed.Session()))).To(ContainElement(HaveKeyWithValue("restored", true))) session, err := agent.Arm(ctx, agentclient.ArmRequest{Probe: jfr.KindName, Params: json.RawMessage(`{"engine":"jfr","settings":"profile"}`)}) Expect(err).NotTo(HaveOccurred()) DeferCleanup(func() { @@ -59,5 +117,19 @@ var _ = Describe("Java installation", func() { profile, err := decode.Decode(checkpoint.Blobs[0].Data, decode.DecodeOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(profile.Samples).NotTo(BeEmpty()) + Expect(session.Disarm(ctx)).To(Succeed()) + _, err = agent.Arm(ctx, agentclient.ArmRequest{Probe: "jvm_redefine", AllowMutation: true, Params: json.RawMessage(`{"class":"Workload","method":"compute","source":"public static double compute(long value) { return 9.0; }"}`)}) + Expect(err).NotTo(HaveOccurred()) + Expect(agent.Close()).To(Succeed()) + agent = nil + reconnected := agentclient.NewRegistry() + Expect(reconnected.Discover(ctx, filepath.Join(root, "telltale", "agents"))).To(Succeed()) + againAgent := reconnected.ByPID(child.Process.Pid) + Expect(againAgent).NotTo(BeNil()) + DeferCleanup(againAgent.Close) + Eventually(func() (string, error) { + value, err := againAgent.Call(ctx, agentclient.CallRequest{Op: "jvm.classes.patches", Params: json.RawMessage(`{}`)}) + return string(value), err + }, 5*time.Second).Should(MatchJSON(`[]`)) }) }) diff --git a/pkg/agentinstall/java.go b/pkg/agentinstall/java.go index 7824870..cb78c12 100644 --- a/pkg/agentinstall/java.go +++ b/pkg/agentinstall/java.go @@ -1,22 +1,29 @@ package agentinstall import ( - javaagent "github.com/flanksource/telltale/pkg/agentinstall/java" "strconv" + + javaagent "github.com/flanksource/telltale/pkg/agentinstall/java" ) -type Java struct{ Home string } +type Java struct{} func (Java) Artifact() (Artifact, error) { data, err := javaagent.Jar() return Artifact{Name: "telltale-agent.jar", Data: data}, err } -func (j Java) Attach(a Attachment) []string { + +// Attach is the helper JVM that attaches the agent to a.PID. It runs without +// the JVM option variables: a container's JAVA_TOOL_OPTIONS loads its +// javaagents (a jmx-exporter, an OpenTelemetry agent) into every JVM, and one +// that binds a port the target already holds kills the helper. +func (Java) Attach(a Attachment) []string { launcher := "java" classpath := a.Artifact - if j.Home != "" { - launcher = j.Home + "/bin/java" - classpath += ":" + j.Home + "/lib/tools.jar" + if a.JavaHome != "" { + launcher = a.JavaHome + "/bin/java" + classpath += ":" + a.JavaHome + "/lib/tools.jar" } - return []string{launcher, "-cp", classpath, "com.flanksource.telltale.Attach", strconv.Itoa(a.PID), a.Artifact, a.Config} + return []string{"env", "-u", "JAVA_TOOL_OPTIONS", "-u", "_JAVA_OPTIONS", "-u", "JDK_JAVA_OPTIONS", + launcher, "-cp", classpath, "com.flanksource.telltale.Attach", strconv.Itoa(a.PID), a.Artifact, a.Config, strconv.FormatBool(a.Upgrade)} } diff --git a/pkg/agentinstall/java/assets.go b/pkg/agentinstall/java/assets.go index 1ee482a..4dcd35c 100644 --- a/pkg/agentinstall/java/assets.go +++ b/pkg/agentinstall/java/assets.go @@ -6,3 +6,6 @@ import "embed" var assets embed.FS func Jar() ([]byte, error) { return assets.ReadFile("assets/telltale-agent.jar") } + +// War is telltale-diag.war, which the tomcat deployer hot-deploys. +func War() ([]byte, error) { return assets.ReadFile("assets/telltale-diag.war") } diff --git a/pkg/agentinstall/java_home_spec_test.go b/pkg/agentinstall/java_home_spec_test.go new file mode 100644 index 0000000..7f7061e --- /dev/null +++ b/pkg/agentinstall/java_home_spec_test.go @@ -0,0 +1,121 @@ +package agentinstall_test + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/flanksource/telltale/pkg/agentinstall" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var errAttachRecorded = errors.New("fixture attach recorded") + +// linuxStat is a JVM's /proc//stat: its command holds a space and a +// parenthesis, and its start time (field 22) is 2318867. +const linuxStat = "321 (java (main)) S 0 321 321 0 -1 4194560 950 0 0 0 1 2 0 0 20 0 41 0 2318867 5000000 3000 18446744073709551615\n" + +type attachHomeRuntime struct{ attachment agentinstall.Attachment } + +func (*attachHomeRuntime) Artifact() (agentinstall.Artifact, error) { + return agentinstall.Artifact{Name: "fixture.jar", Data: []byte("fixture-agent")}, nil +} + +func (r *attachHomeRuntime) Attach(attachment agentinstall.Attachment) []string { + r.attachment = attachment + return []string{"fixture-attach"} +} + +// attachHomeTransport is a Linux target whose JVM runs exe. +type attachHomeTransport struct{ exe, processes string } + +func (t attachHomeTransport) Run(_ context.Context, args []string, _ io.Reader) ([]byte, error) { + switch args[0] { + case "sh": + if args[3] == "telltale-java-runtime" { + return []byte("__TELLTALE_PRESENT__=true\nJAVA_VERSION=\"1.8.0_512\"\n__TELLTALE_TOOLS__=true\n"), nil + } + return []byte(t.processes), nil + case "cat": + if args[1] != "/proc/321/stat" { + return nil, fmt.Errorf("cat of %q, not the target JVM's stat", args[1]) + } + return []byte(linuxStat), nil + case "uname": + return []byte("Linux"), nil + case "readlink": + if args[1] != "/proc/321/exe" { + return nil, fmt.Errorf("readlink of %q, not the target JVM's executable", args[1]) + } + return []byte(t.exe + "\n"), nil + case "sha256sum": + return []byte(fmt.Sprintf("%x %s", sha256.Sum256([]byte("fixture-agent")), args[1])), nil + case "fixture-attach": + return nil, errAttachRecorded + case "install", "tar", "mv", "rm": + return nil, nil + default: + return nil, fmt.Errorf("unexpected installer command %q", args[0]) + } +} + +var _ = Describe("target-compatible Java attachment", func() { + install := func(transport attachHomeTransport, javaHome string, pid ...int) (agentinstall.Attachment, error) { + runtime := &attachHomeRuntime{} + token := filepath.Join(GinkgoT().TempDir(), "agent-token") + Expect(os.WriteFile(token, []byte("fixture-token-for-attach-home"), 0o600)).To(Succeed()) + options := agentinstall.Options{ + Destination: agentinstall.Destination{Kind: "kubernetes", Namespace: "shop", Pod: "orders", Container: "app"}, PID: 321, + Directory: GinkgoT().TempDir(), JavaHome: javaHome, HubURL: "http://localhost:8080", TokenFile: token, + } + if len(pid) > 0 { + options.PID = pid[0] + } + _, err := (agentinstall.Installer{Runtime: runtime, Transport: transport}).Install(context.Background(), options) + return runtime.attachment, err + } + + It("passes the configured target JDK through the installer runtime boundary", func() { + attachment, err := install(attachHomeTransport{exe: "/usr/lib/jvm/other/bin/java"}, "/opt/target-jdk8") + Expect(errors.Is(err, errAttachRecorded)).To(BeTrue()) + Expect(attachment.JavaHome).To(Equal("/opt/target-jdk8")) + }) + + DescribeTable("defaults the target JDK to the one the JVM runs, so a Java 8 helper finds tools.jar", + func(exe, home string) { + attachment, err := install(attachHomeTransport{exe: exe}, "") + Expect(errors.Is(err, errAttachRecorded)).To(BeTrue(), fmt.Sprint(err)) + Expect(attachment.JavaHome).To(Equal(home)) + }, + Entry("a JDK's own java", "/usr/lib/jvm/temurin-8-jdk-amd64/bin/java", "/usr/lib/jvm/temurin-8-jdk-amd64"), + Entry("a Java 8 JDK's jre/bin/java", "/usr/lib/jvm/temurin-8-jdk-amd64/jre/bin/java", "/usr/lib/jvm/temurin-8-jdk-amd64"), + Entry("a JDK path with spaces", "/opt/Java Home/bin/java", "/opt/Java Home"), + ) + + It("discovers the container's JVM behind PID 1 and uses its Java home", func() { + attachment, err := install(attachHomeTransport{exe: "/opt/java/bin/java", processes: "321\n"}, "", 0) + Expect(errors.Is(err, errAttachRecorded)).To(BeTrue(), fmt.Sprint(err)) + Expect(attachment).To(And(HaveField("PID", 321), HaveField("JavaHome", "/opt/java"))) + }) + + It("refuses several container JVMs instead of picking one", func() { + _, err := install(attachHomeTransport{processes: "321\n456\n"}, "", 0) + Expect(err).To(MatchError(ContainSubstring("--pid"))) + }) + + It("identifies a Linux JVM by its /proc start time, so an image without ps attaches", func() { + attachment, err := install(attachHomeTransport{exe: "/usr/local/openjdk-8/bin/java"}, "") + Expect(errors.Is(err, errAttachRecorded)).To(BeTrue(), fmt.Sprint(err)) + Expect(attachment.JavaHome).To(Equal("/usr/local/openjdk-8")) + }) + + It("refuses a JVM executable it cannot place in a JDK, naming --java-home", func() { + _, err := install(attachHomeTransport{exe: "/opt/app/launcher"}, "") + Expect(err).To(MatchError(ContainSubstring("--java-home"))) + }) +}) diff --git a/pkg/agentinstall/java_runtime.sh b/pkg/agentinstall/java_runtime.sh new file mode 100644 index 0000000..5994c69 --- /dev/null +++ b/pkg/agentinstall/java_runtime.sh @@ -0,0 +1,21 @@ +set -eu +home=$1 +if [ ! -e "$home" ]; then + printf '__TELLTALE_PRESENT__=false\n' + exit 0 +fi +printf '__TELLTALE_PRESENT__=true\n' +[ -x "$home/bin/java" ] || { echo "Java home $home has no executable bin/java" >&2; exit 1; } +cat "$home/release" +printf '\n__TELLTALE_TOOLS__=' +if [ -r "$home/lib/tools.jar" ]; then + printf 'true\n' +else + printf 'false\n' +fi +printf '__TELLTALE_ATTACH_MODULE__=' +if [ -r "$home/jmods/jdk.attach.jmod" ]; then + printf 'true\n' +else + printf 'false\n' +fi diff --git a/pkg/agentinstall/jdk.go b/pkg/agentinstall/jdk.go new file mode 100644 index 0000000..b134a3d --- /dev/null +++ b/pkg/agentinstall/jdk.go @@ -0,0 +1,133 @@ +package agentinstall + +import ( + "context" + _ "embed" + "fmt" + "path" + "regexp" + "strconv" + "strings" +) + +type JDKRequest struct { + Major int + OS, Arch string +} + +type JDK struct { + Home, Version string +} + +//go:embed java_runtime.sh +var javaRuntimeScript string + +var javaMajorPattern = regexp.MustCompile(`^(?:1\.)?([0-9]+)(?:[._+\-]|$)`) + +type javaRuntime struct { + Present, Attach bool + Major int +} + +func (i Installer) javaHome(ctx context.Context, transport Transport, options Options) (string, error) { + home := options.JavaHome + if home == "" { + var err error + home, err = targetJavaHome(ctx, transport, options.PID) + if err != nil { + return "", err + } + } + info, err := inspectJavaHome(ctx, transport, home) + if err != nil { + return "", err + } + if !info.Present { + return "", fmt.Errorf("Java home %q does not exist", home) + } + if info.Attach { + return home, nil + } + if options.JavaHome != "" { + return "", fmt.Errorf("--java-home %q lacks the Java %d Attach API; choose a full compatible JDK", home, info.Major) + } + arch, err := transport.Run(ctx, []string{"uname", "-m"}, nil) + if err != nil { + return "", fmt.Errorf("inspect target JDK architecture: %w", err) + } + request := JDKRequest{Major: info.Major, OS: "linux"} + switch strings.TrimSpace(string(arch)) { + case "x86_64", "amd64": + request.Arch = "amd64" + case "aarch64", "arm64": + request.Arch = "arm64" + default: + return "", fmt.Errorf("unsupported target JDK architecture %q", strings.TrimSpace(string(arch))) + } + return i.provisionJDK(ctx, transport, options.Directory, request) +} + +func inspectJavaHome(ctx context.Context, transport Transport, home string) (javaRuntime, error) { + output, err := transport.Run(ctx, []string{"sh", "-c", javaRuntimeScript, "telltale-java-runtime", home}, nil) + if err != nil { + return javaRuntime{}, fmt.Errorf("inspect Java home %q: %w", home, err) + } + values := map[string]string{} + for _, line := range strings.Split(string(output), "\n") { + if key, value, ok := strings.Cut(line, "="); ok { + values[key] = strings.Trim(value, `"`) + } + } + if values["__TELLTALE_PRESENT__"] == "false" { + return javaRuntime{}, nil + } + match := javaMajorPattern.FindStringSubmatch(values["JAVA_VERSION"]) + if values["__TELLTALE_PRESENT__"] != "true" || len(match) != 2 { + return javaRuntime{}, fmt.Errorf("Java home %q has missing or invalid JAVA_VERSION metadata", home) + } + major, err := strconv.Atoi(match[1]) + if err != nil || major < 8 { + return javaRuntime{}, fmt.Errorf("Java home %q has unsupported JAVA_VERSION %q", home, values["JAVA_VERSION"]) + } + attach := values["__TELLTALE_TOOLS__"] == "true" + if major >= 9 { + attach = values["__TELLTALE_ATTACH_MODULE__"] == "true" || strings.Contains(" "+values["MODULES"]+" ", " jdk.attach ") + } + return javaRuntime{Present: true, Attach: attach, Major: major}, nil +} + +func (i Installer) provisionJDK(ctx context.Context, transport Transport, directory string, request JDKRequest) (string, error) { + home := path.Join(directory, fmt.Sprintf("jdk-%d-%s", request.Major, request.Arch)) + info, err := inspectJavaHome(ctx, transport, home) + if err != nil { + return "", err + } + if info.Present { + if info.Major != request.Major || !info.Attach { + return "", fmt.Errorf("staged JDK %q lacks the Java %d Attach API", home, request.Major) + } + return home, nil + } + download := i.DownloadJDK + if download == nil { + download = downloadJDK + } + jdk, err := download(ctx, request) + if err != nil { + return "", fmt.Errorf("download Java %d JDK with deps: %w", request.Major, err) + } + if jdk.Home == "" || jdk.Version == "" { + return "", fmt.Errorf("deps returned an incomplete JDK installation") + } + info, err = inspectJavaHome(ctx, CommandTransport{Destination: Destination{Kind: "local"}}, jdk.Home) + if err != nil { + return "", err + } + if !info.Present || info.Major != request.Major || !info.Attach { + return "", fmt.Errorf("downloaded JDK %q lacks the Java %d Attach API", jdk.Home, request.Major) + } + if err := copyJDK(ctx, transport, jdkCopy{Source: jdk.Home, Target: home, Major: request.Major}); err != nil { + return "", err + } + return home, nil +} diff --git a/pkg/agentinstall/jdk_copy.go b/pkg/agentinstall/jdk_copy.go new file mode 100644 index 0000000..c44d71d --- /dev/null +++ b/pkg/agentinstall/jdk_copy.go @@ -0,0 +1,122 @@ +package agentinstall + +import ( + "archive/tar" + "compress/gzip" + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "time" +) + +type jdkCopy struct { + Source, Target string + Major int +} + +func copyJDK(ctx context.Context, transport Transport, options jdkCopy) error { + archive, err := jdkArchive(options.Source) + if err != nil { + return err + } + defer func() { + _ = archive.Close() + _ = os.Remove(archive.Name()) + }() + var nonce [12]byte + if _, err := rand.Read(nonce[:]); err != nil { + return err + } + staged := options.Target + ".stage-" + hex.EncodeToString(nonce[:]) + defer func() { + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + _, _ = transport.Run(cleanup, []string{"rm", "-rf", "--", staged}, nil) + }() + if _, err := transport.Run(ctx, []string{"install", "-d", "-m", "700", staged}, nil); err != nil { + return err + } + if _, err := transport.Run(ctx, []string{"tar", "-x", "-z", "-m", "--no-same-owner", "-f", "-", "-C", staged}, archive); err != nil { + return fmt.Errorf("copy JDK into target: %w", err) + } + info, err := inspectJavaHome(ctx, transport, staged) + if err != nil { + return err + } + if !info.Present || info.Major != options.Major || !info.Attach { + return fmt.Errorf("copied JDK lacks the Java %d Attach API", options.Major) + } + if _, err := transport.Run(ctx, []string{"mv", "-T", staged, options.Target}, nil); err != nil { + return fmt.Errorf("publish copied JDK: %w", err) + } + return nil +} + +func jdkArchive(root string) (*os.File, error) { + if err := os.MkdirAll(".tmp/telltale-jdk", 0o700); err != nil { + return nil, err + } + file, err := os.CreateTemp(".tmp/telltale-jdk", "transfer-*.tar.gz") + if err != nil { + return nil, err + } + compressed := gzip.NewWriter(file) + archive := tar.NewWriter(compressed) + err = filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil || path == root { + return err + } + return archiveJDKFile(archive, root, path, entry) + }) + err = errors.Join(err, archive.Close(), compressed.Close()) + if err == nil { + _, err = file.Seek(0, io.SeekStart) + } + if err != nil { + _ = file.Close() + _ = os.Remove(file.Name()) + return nil, err + } + return file, nil +} + +func archiveJDKFile(archive *tar.Writer, root, path string, entry fs.DirEntry) error { + info, err := entry.Info() + if err != nil { + return err + } + name, err := filepath.Rel(root, path) + if err != nil { + return err + } + var link string + if info.Mode()&os.ModeSymlink != 0 { + link, err = os.Readlink(path) + if err != nil { + return err + } + if filepath.IsAbs(link) || !filepath.IsLocal(filepath.Join(filepath.Dir(name), link)) { + return fmt.Errorf("JDK symlink %q escapes its directory", name) + } + } + header, err := tar.FileInfoHeader(info, link) + if err != nil { + return err + } + header.Name = filepath.ToSlash(name) + if err := archive.WriteHeader(header); err != nil || !info.Mode().IsRegular() { + return err + } + file, err := os.Open(path) + if err != nil { + return err + } + _, err = io.Copy(archive, file) + return errors.Join(err, file.Close()) +} diff --git a/pkg/agentinstall/jdk_deps.go b/pkg/agentinstall/jdk_deps.go new file mode 100644 index 0000000..a711ede --- /dev/null +++ b/pkg/agentinstall/jdk_deps.go @@ -0,0 +1,34 @@ +package agentinstall + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/flanksource/deps" +) + +func downloadJDK(ctx context.Context, request JDKRequest) (JDK, error) { + cache, err := os.UserCacheDir() + if err != nil { + return JDK{}, err + } + root := filepath.Join(cache, "telltale", "jdks", request.OS+"-"+request.Arch, fmt.Sprint(request.Major)) + if err := os.MkdirAll(".tmp/telltale-jdk", 0o700); err != nil { + return JDK{}, err + } + result, err := deps.InstallWithContext(ctx, fmt.Sprintf("openjdk%d", request.Major), "latest", + deps.WithOS(request.OS, request.Arch), deps.WithStrictChecksum(true), + deps.WithAppDir(filepath.Join(root, "apps")), deps.WithBinDir(filepath.Join(root, "bin")), + deps.WithTmpDir(".tmp/telltale-jdk"), deps.WithCacheDir(filepath.Join(root, "downloads")), + deps.WithTimeout(30*time.Minute)) + if err != nil { + return JDK{}, err + } + if result == nil || result.Package.Name == "" || result.Version.Version == "" { + return JDK{}, fmt.Errorf("deps returned no installed JDK package or version") + } + return JDK{Home: filepath.Join(root, "apps", result.Package.FolderName(result.Version.Version)), Version: result.Version.Version}, nil +} diff --git a/pkg/agentinstall/jdk_spec_test.go b/pkg/agentinstall/jdk_spec_test.go new file mode 100644 index 0000000..45caf05 --- /dev/null +++ b/pkg/agentinstall/jdk_spec_test.go @@ -0,0 +1,146 @@ +package agentinstall + +import ( + "context" + "errors" + "io" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type jdkTransport struct { + home string + tars int + tarError error +} + +func (t *jdkTransport) Run(ctx context.Context, args []string, input io.Reader) ([]byte, error) { + if args[0] == "readlink" { + return []byte(t.home + "/bin/java\n"), nil + } + if args[0] == "uname" { + return []byte("x86_64\n"), nil + } + if args[0] == "tar" { + t.tars++ + if t.tarError != nil { + return nil, t.tarError + } + } + if args[0] == "mv" { + return nil, os.Rename(args[len(args)-2], args[len(args)-1]) + } + return (CommandTransport{Destination: Destination{Kind: "local"}}).Run(ctx, args, input) +} + +func javaFixture(root, version string, tools bool) string { + Expect(os.MkdirAll(filepath.Join(root, "bin"), 0o700)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(root, "release"), []byte("JAVA_VERSION=\""+version+"\"\n"), 0o600)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(root, "bin", "java"), []byte("fixture executable"), 0o755)).To(Succeed()) + if tools { + Expect(os.Mkdir(filepath.Join(root, "lib"), 0o700)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(root, "lib", "tools.jar"), []byte("fixture attach API"), 0o600)).To(Succeed()) + } + return root +} + +var _ = Describe("automatic attach JDK provisioning", func() { + var transport *jdkTransport + var installer Installer + var options Options + var downloads int + BeforeEach(func() { + root := GinkgoT().TempDir() + transport = &jdkTransport{home: javaFixture(filepath.Join(root, "runtime"), "1.8.0_212", false)} + options = Options{PID: 321, Directory: filepath.Join(root, "target")} + downloads = 0 + installer = Installer{DownloadJDK: func(_ context.Context, request JDKRequest) (JDK, error) { + downloads++ + Expect(request).To(Equal(JDKRequest{Major: 8, OS: "linux", Arch: "amd64"})) + home := javaFixture(filepath.Join(root, "download"), "1.8.0_512", true) + Expect(os.Symlink("java", filepath.Join(home, "bin", "java_g"))).To(Succeed()) + return JDK{Home: home, Version: "8.0.512+9"}, nil + }} + }) + + It("copies a compatible JDK with symlinks and executable modes and reuses it without downloading", func() { + home, err := installer.javaHome(context.Background(), transport, options) + Expect(err).NotTo(HaveOccurred()) + Expect(home).To(Equal(filepath.Join(options.Directory, "jdk-8-amd64"))) + data, err := os.ReadFile(filepath.Join(home, "lib", "tools.jar")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).To(Equal("fixture attach API")) + link, err := os.Readlink(filepath.Join(home, "bin", "java_g")) + Expect(err).NotTo(HaveOccurred()) + Expect(link).To(Equal("java")) + info, err := os.Stat(filepath.Join(home, "bin", "java")) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o755))) + again, err := installer.javaHome(context.Background(), transport, options) + Expect(err).NotTo(HaveOccurred()) + Expect(again).To(Equal(home)) + Expect([]int{downloads, transport.tars}).To(Equal([]int{1, 1})) + }) + + It("keeps a full target JDK without downloading", func() { + transport.home = javaFixture(filepath.Join(GinkgoT().TempDir(), "full"), "1.8.0_212", true) + home, err := installer.javaHome(context.Background(), transport, options) + Expect(err).NotTo(HaveOccurred()) + Expect(home).To(Equal(transport.home)) + Expect(downloads).To(BeZero()) + }) + + It("honors an explicit JDK and refuses an explicit JRE", func() { + options.JavaHome = javaFixture(filepath.Join(GinkgoT().TempDir(), "explicit"), "1.8.0_512", true) + home, err := installer.javaHome(context.Background(), transport, options) + Expect(err).NotTo(HaveOccurred()) + Expect(home).To(Equal(options.JavaHome)) + options.JavaHome = transport.home + _, err = installer.javaHome(context.Background(), transport, options) + Expect(err).To(MatchError(ContainSubstring("--java-home"))) + Expect(downloads).To(BeZero()) + }) + + It("reports download failures without staging an incomplete JDK", func() { + installer.DownloadJDK = func(context.Context, JDKRequest) (JDK, error) { return JDK{}, errors.New("download denied") } + _, err := installer.javaHome(context.Background(), transport, options) + Expect(err).To(MatchError(ContainSubstring("download denied"))) + Expect(transport.tars).To(BeZero()) + }) + + It("refuses a downloaded JRE without an Attach API", func() { + installer.DownloadJDK = func(context.Context, JDKRequest) (JDK, error) { return JDK{Home: transport.home, Version: "8"}, nil } + _, err := installer.javaHome(context.Background(), transport, options) + Expect(err).To(MatchError(ContainSubstring("Attach API"))) + Expect(transport.tars).To(BeZero()) + }) + + It("cleans a failed copy without publishing an incomplete JDK", func() { + transport.tarError = errors.New("incomplete tar stream") + _, err := installer.javaHome(context.Background(), transport, options) + Expect(err).To(MatchError(ContainSubstring("incomplete tar stream"))) + Expect(filepath.Glob(filepath.Join(options.Directory, "jdk-*"))).To(BeEmpty()) + }) + + It("refuses a staged JDK from another Java major without downloading over it", func() { + staged := javaFixture(filepath.Join(options.Directory, "jdk-8-amd64"), "17.0.17", false) + Expect(os.WriteFile(filepath.Join(staged, "release"), []byte("JAVA_VERSION=\"17.0.17\"\nMODULES=\"java.base jdk.attach\"\n"), 0o600)).To(Succeed()) + _, err := installer.javaHome(context.Background(), transport, options) + Expect(err).To(MatchError(ContainSubstring("Java 8 Attach API"))) + Expect(downloads).To(BeZero()) + }) + + It("recognizes a modular Attach API and rejects missing version metadata", func() { + options.JavaHome = javaFixture(filepath.Join(GinkgoT().TempDir(), "modular"), "17.0.17", false) + Expect(os.WriteFile(filepath.Join(options.JavaHome, "release"), []byte("JAVA_VERSION=\"17.0.17\"\nMODULES=\"java.base jdk.attach\"\n"), 0o600)).To(Succeed()) + home, err := installer.javaHome(context.Background(), transport, options) + Expect(err).NotTo(HaveOccurred()) + Expect(home).To(Equal(options.JavaHome)) + Expect(os.WriteFile(filepath.Join(options.JavaHome, "release"), nil, 0o600)).To(Succeed()) + _, err = installer.javaHome(context.Background(), transport, options) + Expect(err).To(MatchError(ContainSubstring("JAVA_VERSION"))) + }) +}) diff --git a/pkg/agentinstall/proc.go b/pkg/agentinstall/proc.go new file mode 100644 index 0000000..0830f20 --- /dev/null +++ b/pkg/agentinstall/proc.go @@ -0,0 +1,34 @@ +package agentinstall + +import ( + "context" + _ "embed" + "fmt" + "strconv" + "strings" +) + +//go:embed proc.sh +var javaProcessesScript string + +func javaPID(ctx context.Context, transport Transport) (int, error) { + output, err := transport.Run(ctx, []string{"sh", "-c", javaProcessesScript, "telltale-java-processes", "/proc"}, nil) + if err != nil { + return 0, fmt.Errorf("discover Java processes in /proc: %w", err) + } + processes := strings.Fields(string(output)) + if len(processes) == 0 { + return 0, fmt.Errorf("no Java process found in /proc; check process permissions or specify --pid") + } + for _, process := range processes { + pid, err := strconv.Atoi(process) + if err != nil || pid <= 0 { + return 0, fmt.Errorf("invalid Java process PID %q in /proc discovery", process) + } + } + if len(processes) != 1 { + return 0, fmt.Errorf("multiple Java processes found in /proc (%s); specify --pid", strings.Join(processes, ", ")) + } + pid, err := strconv.Atoi(processes[0]) + return pid, err +} diff --git a/pkg/agentinstall/proc.sh b/pkg/agentinstall/proc.sh new file mode 100644 index 0000000..dc17bcc --- /dev/null +++ b/pkg/agentinstall/proc.sh @@ -0,0 +1,18 @@ +set -eu +command -v readlink >/dev/null || { echo 'Java discovery requires readlink' >&2; exit 1; } +root=$1 +[ -d "$root/self" ] || { echo 'Java discovery requires a mounted /proc' >&2; exit 1; } +for entry in "$root"/[0-9]*/exe; do + [ -L "$entry" ] || continue + if executable=$(readlink "$entry"); then + case "$executable" in + */bin/java) + process=${entry%/exe} + printf '%s\n' "${process##*/}" + ;; + esac + elif [ -d "${entry%/exe}" ]; then + echo "Cannot inspect $entry; check process permissions or specify --pid" >&2 + exit 1 + fi +done diff --git a/pkg/agentinstall/proc_linux_spec_test.go b/pkg/agentinstall/proc_linux_spec_test.go new file mode 100644 index 0000000..bbfa2d6 --- /dev/null +++ b/pkg/agentinstall/proc_linux_spec_test.go @@ -0,0 +1,47 @@ +//go:build linux + +package agentinstall + +import ( + "context" + "os" + "os/exec" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("live Linux Java discovery", func() { + It("finds the JVM behind init and its home with only proc inspection tools on PATH", func() { + java, err := exec.LookPath("java") + Expect(err).NotTo(HaveOccurred()) + java, err = filepath.EvalSymlinks(java) + Expect(err).NotTo(HaveOccurred()) + directory := GinkgoT().TempDir() + output, err := exec.Command("javac", "-d", directory, "testdata/Workload.java").CombinedOutput() + Expect(err).NotTo(HaveOccurred(), string(output)) + child := exec.Command(java, "-cp", directory, "Workload") + Expect(child.Start()).To(Succeed()) + DeferCleanup(func() { _ = child.Process.Kill(); _ = child.Wait() }) + tools := filepath.Join(directory, "tools") + Expect(os.Mkdir(tools, 0o700)).To(Succeed()) + for _, name := range []string{"sh", "readlink", "cat"} { + path, err := exec.LookPath(name) + Expect(err).NotTo(HaveOccurred()) + Expect(os.Symlink(path, filepath.Join(tools, name))).To(Succeed()) + } + GinkgoT().Setenv("PATH", tools) + GinkgoT().Setenv("JAVA_HOME", "/not-the-running-jvm") + transport := CommandTransport{Destination: Destination{Kind: "local"}} + pid, err := javaPID(context.Background(), transport) + Expect(err).NotTo(HaveOccurred()) + Expect(pid).To(Equal(child.Process.Pid)) + home, err := targetJavaHome(context.Background(), transport, pid) + Expect(err).NotTo(HaveOccurred()) + Expect(home).To(Equal(filepath.Dir(filepath.Dir(java)))) + identity, err := processIdentity(context.Background(), transport, true, pid) + Expect(err).NotTo(HaveOccurred()) + Expect(identity).NotTo(BeEmpty()) + }) +}) diff --git a/pkg/agentinstall/proc_spec_test.go b/pkg/agentinstall/proc_spec_test.go new file mode 100644 index 0000000..a1c3300 --- /dev/null +++ b/pkg/agentinstall/proc_spec_test.go @@ -0,0 +1,66 @@ +package agentinstall + +import ( + "context" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type processTransport struct { + output string + err error +} + +func (t processTransport) Run(_ context.Context, args []string, _ io.Reader) ([]byte, error) { + Expect(args).To(Equal([]string{"sh", "-c", javaProcessesScript, "telltale-java-processes", "/proc"})) + return []byte(t.output), t.err +} + +var _ = Describe("container Java process discovery", func() { + DescribeTable("requires exactly one readable JVM", + func(output string, want int, message string) { + pid, err := javaPID(context.Background(), processTransport{output: output}) + if message != "" { + Expect(err).To(MatchError(ContainSubstring(message))) + } else { + Expect(err).NotTo(HaveOccurred()) + Expect(pid).To(Equal(want)) + } + }, + Entry("a JVM behind an init process", "321\n", 321, ""), + Entry("no JVM", "", 0, "no Java process"), + Entry("several JVMs", "321\n456\n", 0, "--pid"), + Entry("invalid PID", "invalid\n", 0, "invalid"), + Entry("zero PID", "0\n", 0, "invalid"), + ) + + It("reports a failed proc inspection", func() { + _, err := javaPID(context.Background(), processTransport{err: errors.New("permission denied")}) + Expect(err).To(MatchError(ContainSubstring("permission denied"))) + }) + + It("scans proc executables with no ps, jps, java or PATH Java home", func() { + root := GinkgoT().TempDir() + tools := filepath.Join(root, "tools") + Expect(os.Mkdir(tools, 0o700)).To(Succeed()) + readlink, err := exec.LookPath("readlink") + Expect(err).NotTo(HaveOccurred()) + Expect(os.Symlink(readlink, filepath.Join(tools, "readlink"))).To(Succeed()) + Expect(os.Mkdir(filepath.Join(root, "self"), 0o700)).To(Succeed()) + for pid, executable := range map[string]string{"1": "/sbin/init", "321": "/opt/Java Home/jre/bin/java", "456": "/usr/bin/sh"} { + Expect(os.Mkdir(filepath.Join(root, pid), 0o700)).To(Succeed()) + Expect(os.Symlink(executable, filepath.Join(root, pid, "exe"))).To(Succeed()) + } + command := exec.Command("/bin/sh", "-c", javaProcessesScript, "telltale-java-processes", root) + command.Env = []string{"PATH=" + tools} + output, err := command.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), string(output)) + Expect(string(output)).To(Equal("321\n")) + }) +}) diff --git a/pkg/agentinstall/profiles_http_spec_test.go b/pkg/agentinstall/profiles_http_spec_test.go new file mode 100644 index 0000000..0342818 --- /dev/null +++ b/pkg/agentinstall/profiles_http_spec_test.go @@ -0,0 +1,75 @@ +package agentinstall_test + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "encoding/json" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "time" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/agentinstall" + "github.com/flanksource/telltale/pkg/deploy/auth" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type profilesHTTPAgent struct { + registry *agentclient.Registry + agent *agentclient.Agent + attachment agentinstall.Attachment + client agentclient.HTTPClient + portFile string +} + +func attachProfilesHTTP(workload profilesWorkload) *agentclient.Registry { + return connectProfilesHTTP(workload).registry +} + +func connectProfilesHTTP(workload profilesWorkload) profilesHTTPAgent { + GinkgoHelper() + directory := GinkgoT().TempDir() + key, err := rsa.GenerateKey(rand.Reader, 2048) + Expect(err).NotTo(HaveOccurred()) + issuer, err := auth.NewIssuer("profiles-integration", key) + Expect(err).NotTo(HaveOccurred()) + const target = "profiles-http-fixture" + portFile := filepath.Join(directory, "port") + config, err := json.Marshal(map[string]any{"token": "profiles-local-fixture-token", "service": "profile-fixture", "allowMutation": true, + "resource": map[string]string{"deployment.environment.name": "profiles-integration"}, + "http": map[string]any{"bind": "127.0.0.1", "port": 0, "portFile": portFile, "issuer": issuer.Name(), "audience": target, "jwks": issuer.JWKS()}}) + Expect(err).NotTo(HaveOccurred()) + configPath := filepath.Join(directory, "agent.json") + Expect(os.WriteFile(configPath, config, 0o600)).To(Succeed()) + artifact, err := (agentinstall.Java{}).Artifact() + Expect(err).NotTo(HaveOccurred()) + jar := filepath.Join(directory, artifact.Name) + Expect(os.WriteFile(jar, artifact.Data, 0o600)).To(Succeed()) + attachment := agentinstall.Attachment{PID: workload.pid, Artifact: jar, Config: configPath, JavaHome: workload.javaHome} + args := (agentinstall.Java{}).Attach(attachment) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + attached := exec.CommandContext(ctx, args[0], args[1:]...) + attached.Stderr = GinkgoWriter + Expect(attached.Run()).To(Succeed()) + port, err := os.ReadFile(portFile) + Expect(err).NotTo(HaveOccurred()) + agent, err := agentclient.DialHTTP(ctx, agentclient.HTTPEndpoint{ID: target, URL: "http://127.0.0.1:" + string(port), Client: http.DefaultClient, + Tokens: auth.Tokens{Issuer: issuer, Subject: "profiles-fixture", TTL: time.Minute}, Resource: map[string]string{"telltale.deployer": "profiles-fixture"}, Poll: 100 * time.Millisecond}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(agent.Close) + Expect(agent.Hello.Resource).To(SatisfyAll( + HaveKeyWithValue("telltale.deployer", "profiles-fixture"), HaveKeyWithValue("service.name", "profile-fixture"), + HaveKeyWithValue("process.pid", strconv.Itoa(workload.pid)), HaveKeyWithValue("deployment.environment.name", "profiles-integration"), + )) + Expect(agent.Hello.Resource["telltale.source.instance"]).NotTo(BeEmpty()) + registry := agentclient.NewRegistry() + Expect(registry.Add(agent)).To(Succeed()) + return profilesHTTPAgent{registry: registry, agent: agent, attachment: attachment, portFile: portFile, + client: agentclient.HTTPClient{Target: target, URL: "http://127.0.0.1:" + string(port), Client: http.DefaultClient, Tokens: auth.Tokens{Issuer: issuer, Subject: "profiles-fixture", TTL: time.Minute}}} +} diff --git a/pkg/agentinstall/profiles_spec_test.go b/pkg/agentinstall/profiles_spec_test.go new file mode 100644 index 0000000..9ef5c80 --- /dev/null +++ b/pkg/agentinstall/profiles_spec_test.go @@ -0,0 +1,211 @@ +package agentinstall_test + +import ( + "bufio" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/agentinstall" + "github.com/flanksource/telltale/pkg/credentials/apitoken" + "github.com/flanksource/telltale/pkg/datadir" + "github.com/flanksource/telltale/pkg/otlp/profiles" + "github.com/flanksource/telltale/pkg/traces/read" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/httptop" + "github.com/flanksource/telltale/pkg/tracing/sqltop" + "github.com/flanksource/telltale/pkg/tracing/tracingtest" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type profilesWorkload struct { + input io.WriteCloser + lines chan string + pid int + javaHome string +} + +func startProfilesWorkload() profilesWorkload { + GinkgoHelper() + directory := GinkgoT().TempDir() + javaHome := os.Getenv("TELLTALE_JVM_INTEGRATION_JAVA_HOME") + output, err := exec.Command(profilesExecutable(javaHome, "javac"), "-d", directory, "testdata/ProfilesWorkload.java").CombinedOutput() + Expect(err).NotTo(HaveOccurred(), string(output)) + dependency, err := filepath.Abs("../../sdk/java/build/libs/integration-fixtures/h2-2.2.224.jar") + Expect(err).NotTo(HaveOccurred()) + _, err = os.Stat(dependency) + Expect(err).NotTo(HaveOccurred(), "build the packaged Java agent and its declared test fixture artifacts before integration tests") + child := exec.Command(profilesExecutable(javaHome, "java"), "-cp", directory+string(os.PathListSeparator)+dependency, "ProfilesWorkload") + child.Stderr = GinkgoWriter + stdout, err := child.StdoutPipe() + Expect(err).NotTo(HaveOccurred()) + stdin, err := child.StdinPipe() + Expect(err).NotTo(HaveOccurred()) + Expect(child.Start()).To(Succeed()) + lines := make(chan string, 16) + go func() { + defer close(lines) + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + if strings.HasPrefix(scanner.Text(), "FIXTURE ") { + lines <- scanner.Text() + } + } + }() + DeferCleanup(func() { + _ = stdin.Close() + _ = child.Process.Kill() + _ = child.Wait() + }) + workload := profilesWorkload{input: stdin, lines: lines, pid: child.Process.Pid, javaHome: javaHome} + workload.await("FIXTURE READY") + return workload +} + +func profilesExecutable(home, executable string) string { + if home == "" { + return executable + } + return filepath.Join(home, "bin", executable) +} + +func (w profilesWorkload) await(message string) { + GinkgoHelper() + var line string + Eventually(w.lines).WithTimeout(15 * time.Second).Should(Receive(&line)) + Expect(line).To(Equal(message)) +} + +func (w profilesWorkload) command(command, message string) { + GinkgoHelper() + _, err := fmt.Fprintln(w.input, command) + Expect(err).NotTo(HaveOccurred()) + w.await(message) +} + +func attachProfilesHub(workload profilesWorkload) *agentclient.Registry { + GinkgoHelper() + directory := GinkgoT().TempDir() + tokens := agentclient.NewTokenStore([]datadir.ConfigDir{{Name: datadir.ConfigGlobal, Dir: directory}}) + token, err := tokens.Mint("profile-jvm", apitoken.ScopeAgent) + Expect(err).NotTo(HaveOccurred()) + registry := agentclient.NewRegistry() + server := httptest.NewServer(agentclient.NewHub(registry, tokens)) + DeferCleanup(server.Close) + artifact, err := (agentinstall.Java{}).Artifact() + Expect(err).NotTo(HaveOccurred(), "build the packaged Java agent before integration tests") + jar := filepath.Join(directory, artifact.Name) + Expect(os.WriteFile(jar, artifact.Data, 0o600)).To(Succeed()) + config, err := json.Marshal(map[string]any{"url": server.URL, "token": token.Secret.Value(), "service": "profile-fixture", "allowMutation": true, + "resource": map[string]string{"deployment.environment.name": "profiles-integration"}}) + Expect(err).NotTo(HaveOccurred()) + configPath := filepath.Join(directory, "agent.json") + Expect(os.WriteFile(configPath, config, 0o600)).To(Succeed()) + args := (agentinstall.Java{}).Attach(agentinstall.Attachment{PID: workload.pid, Artifact: jar, Config: configPath, JavaHome: workload.javaHome}) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + attached := exec.CommandContext(ctx, args[0], args[1:]...) + attached.Stderr = GinkgoWriter + Expect(attached.Run()).To(Succeed()) + Eventually(func() int { return len(registry.Match(nil)) }).WithTimeout(10 * time.Second).Should(Equal(1)) + agent := registry.Match(nil)[0] + By("receiving an authenticated HELLO from JVM " + agent.Hello.Runtime.Version) + Expect(agent.Hello.AgentID).To(HavePrefix("hub/")) + Expect(agent.Hello.Service).To(Equal("profile-fixture")) + Expect(agent.Hello.Resource).To(HaveKeyWithValue("process.pid", strconv.Itoa(workload.pid))) + Expect(agent.Hello.Resource).To(HaveKeyWithValue("deployment.environment.name", "profiles-integration")) + Expect(agent.Hello.Resource["telltale.source.instance"]).NotTo(BeEmpty()) + DeferCleanup(agent.Close) + return registry +} + +func latestProfileRows(ref read.Ref) read.TopResult { + GinkgoHelper() + var result read.TopResult + Eventually(func(g Gomega) { + var err error + result, err = read.Top(context.Background(), ref, read.TopQuery{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(result.Rows).NotTo(BeEmpty()) + }).WithTimeout(15 * time.Second).Should(Succeed()) + return result +} + +var _ = Describe("attached JVM Profiles over authenticated sessions", func() { + DescribeTable("captures warmed HTTP and JDBC operations, updates the same IDs and clears both Top tables", func(attach func(profilesWorkload) *agentclient.Registry) { + workload := startProfilesWorkload() + registry := attach(workload) + previous := agentclient.Default + agentclient.Default = registry + DeferCleanup(func() { agentclient.Default = previous }) + agent := registry.Match(nil)[0] + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{"http_top": httptop.NewKind(registry), "sql_top": sqltop.Kind()}) + params, err := json.Marshal(map[string]any{"target": map[string]any{"seam": "agent", "agent": agent.Hello.AgentID}, "poll": "100ms"}) + Expect(err).NotTo(HaveOccurred()) + refs := map[string]read.Ref{} + for _, kind := range []string{"http_top", "sql_top"} { + session := env.Start(kind, string(params)) + tracingtest.Running(session) + info := session.Snapshot() + Expect(info.Events).NotTo(BeNil()) + refs[kind] = read.Ref{Runtime: env.Runtime, TraceRef: tracing.TraceRef{Kind: kind, Stream: info.Events.Stream}} + } + workload.command("START", "FIXTURE ACTIVE") + for _, kind := range []string{"http_top", "sql_top"} { + first := latestProfileRows(refs[kind]) + Expect(first.Frame.Source).To(HavePrefix(agent.Hello.Resource["telltale.source.instance"] + "/")) + identity := "requestId" + if kind == "sql_top" { + identity = "operationId" + Expect(first.Rows).To(ContainElement(And(HaveKeyWithValue("api", "jdbc"), HaveKeyWithValue("sql", ContainSubstring("HOLD()"))))) + } else { + Expect(first.Rows).To(ContainElement(And(HaveKeyWithValue("direction", "outgoing"), HaveKeyWithValue("method", "GET"), HaveKeyWithValue("url", HaveSuffix("/hold"))))) + } + id := first.Rows[0][identity] + elapsed := first.Rows[0]["elapsedMs"] + Eventually(func(g Gomega) { + next, err := read.Top(context.Background(), refs[kind], read.TopQuery{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(next.Frame.Tick).To(BeNumerically(">", first.Frame.Tick)) + g.Expect(next.Rows).To(ContainElement(And(HaveKeyWithValue(identity, id), HaveKey("elapsedMs")))) + g.Expect(next.Rows[0]["elapsedMs"]).To(BeNumerically(">", elapsed)) + }).WithTimeout(10 * time.Second).Should(Succeed()) + native, err := base64.StdEncoding.DecodeString(first.Frame.Summary["otlp"].(string)) + Expect(err).NotTo(HaveOccurred()) + decoded, err := profiles.Decode(native, profiles.Options{ExpectedKind: kind, ExpectedCaptureID: first.Frame.Summary["captureId"].(string), ExpectedSource: agent.Hello.Resource["telltale.source.instance"]}) + Expect(err).NotTo(HaveOccurred()) + Expect(decoded).To(HaveLen(1)) + Expect(decoded[0].Sequence).To(Equal(first.Frame.Tick)) + Expect(decoded[0].ProfileID).To(Equal(first.Frame.ID)) + Expect(decoded[0].Resource).To(SatisfyAll( + HaveKeyWithValue("service.name", agent.Hello.Service), + HaveKeyWithValue("process.pid", agent.Hello.Resource["process.pid"]), + HaveKeyWithValue("telltale.source.instance", agent.Hello.Resource["telltale.source.instance"]), + HaveKeyWithValue("deployment.environment.name", agent.Hello.Resource["deployment.environment.name"]), + )) + } + workload.command("FINISH", "FIXTURE FINISHED") + for _, ref := range refs { + Eventually(func(g Gomega) { + result, err := read.Top(context.Background(), ref, read.TopQuery{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(result.Rows).To(BeEmpty()) + g.Expect(result.Frame.Summary).To(HaveKeyWithValue("complete", true)) + frames, err := read.Frames(context.Background(), ref) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(frames).To(ContainElement(HaveField("Retained", BeNumerically(">", 0)))) + }).WithTimeout(15 * time.Second).Should(Succeed()) + } + }, Entry("over the hub", attachProfilesHub), Entry("over direct HTTP", attachProfilesHTTP), Entry("over direct HTTP after a runtime upgrade", attachUpgradedProfilesHTTP)) +}) diff --git a/pkg/agentinstall/testdata/ProfilesWorkload.java b/pkg/agentinstall/testdata/ProfilesWorkload.java new file mode 100644 index 0000000..c438e17 --- /dev/null +++ b/pkg/agentinstall/testdata/ProfilesWorkload.java @@ -0,0 +1,85 @@ +import com.sun.net.httpserver.HttpServer; +import java.io.*; +import java.net.*; +import java.sql.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicReference; + +public final class ProfilesWorkload { + private static final CountDownLatch httpEntered = new CountDownLatch(1); + private static final CountDownLatch sqlEntered = new CountDownLatch(1); + private static final CountDownLatch release = new CountDownLatch(1); + private static final AtomicReference failure = new AtomicReference<>(); + + public static int hold() throws InterruptedException { + sqlEntered.countDown(); + if (!release.await(30, TimeUnit.SECONDS)) throw new IllegalStateException("SQL release timed out"); + return 7; + } + + public static void main(String[] args) throws Exception { + Class.forName("org.h2.Driver"); + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + ExecutorService executor = Executors.newCachedThreadPool(); + server.setExecutor(executor); + server.createContext("/", exchange -> { + try { + if (exchange.getRequestURI().getPath().equals("/hold")) { + httpEntered.countDown(); + if (!release.await(30, TimeUnit.SECONDS)) throw new IllegalStateException("HTTP release timed out"); + } + byte[] body = "complete".getBytes("UTF-8"); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + } catch (Exception error) { failure.set(error); } + finally { exchange.close(); } + }); + server.start(); + String address = "http://127.0.0.1:" + server.getAddress().getPort(); + try (Connection connection = DriverManager.getConnection("jdbc:h2:mem:profiles"); + Statement statement = connection.createStatement(); + BufferedReader input = new BufferedReader(new InputStreamReader(System.in, "UTF-8"))) { + statement.execute("CREATE ALIAS HOLD FOR 'ProfilesWorkload.hold'"); + try (ResultSet warm = statement.executeQuery("SELECT 1")) { + if (!warm.next() || warm.getInt(1) != 1) throw new IllegalStateException("JDBC warmup failed"); + } + request(address + "/warm"); + System.out.println("FIXTURE READY"); + if (!"START".equals(input.readLine())) throw new IllegalStateException("expected START"); + Future http = executor.submit(() -> { + try { request(address + "/hold"); } + catch (Exception error) { failure.set(error); } + }); + Future sql = executor.submit(() -> { + try (ResultSet result = statement.executeQuery("SELECT HOLD()")) { + if (!result.next() || result.getInt(1) != 7) throw new IllegalStateException("JDBC result changed"); + } catch (Exception error) { failure.set(error); } + }); + if (!httpEntered.await(10, TimeUnit.SECONDS) || !sqlEntered.await(10, TimeUnit.SECONDS)) + throw new IllegalStateException("operations did not become active: " + failure.get()); + System.out.println("FIXTURE ACTIVE"); + if (!"FINISH".equals(input.readLine())) throw new IllegalStateException("expected FINISH"); + release.countDown(); + http.get(10, TimeUnit.SECONDS); + sql.get(10, TimeUnit.SECONDS); + if (failure.get() != null) throw new IllegalStateException("workload failed", failure.get()); + System.out.println("FIXTURE FINISHED"); + input.readLine(); + } finally { + release.countDown(); + server.stop(0); + executor.shutdownNow(); + } + } + + private static void request(String address) throws Exception { + HttpURLConnection connection = (HttpURLConnection) new URL(address).openConnection(); + connection.setConnectTimeout(5000); + connection.setReadTimeout(30000); + try (InputStream input = connection.getInputStream(); ByteArrayOutputStream body = new ByteArrayOutputStream()) { + byte[] buffer = new byte[64]; + for (int count; (count = input.read(buffer)) != -1;) body.write(buffer, 0, count); + if (!body.toString("UTF-8").equals("complete")) throw new IllegalStateException("HTTP body changed"); + } finally { connection.disconnect(); } + } +} diff --git a/pkg/agentinstall/testdata/Workload.java b/pkg/agentinstall/testdata/Workload.java index 522c510..9fd7a1e 100644 --- a/pkg/agentinstall/testdata/Workload.java +++ b/pkg/agentinstall/testdata/Workload.java @@ -4,7 +4,8 @@ public static void main(String[] args) throws Exception { long seconds=args.length==0?60:Long.parseLong(args[0]); long end=System.nanoTime()+seconds*1_000_000_000L; long value=0; - while(System.nanoTime()", 4)) + Expect(checkpoint.Blobs[0].Data[:4]).To(Equal([]byte{'F', 'L', 'R', 0})) + Expect(session.Disarm(ctx)).To(Succeed()) + } + probe, err := agent.Arm(ctx, agentclient.ArmRequest{Probe: "jvm_trace", AllowMutation: true, + Params: json.RawMessage(`{"targets":[{"class":"ProfilesWorkload","method":"hold"}],"events":10}`)}) + Expect(err).NotTo(HaveOccurred()) + Expect(probe.Disarm(ctx)).To(Succeed()) + patches, err := agent.Call(ctx, agentclient.CallRequest{Op: "jvm.classes.patches", Params: json.RawMessage(`{}`)}) + Expect(err).NotTo(HaveOccurred()) + Expect(patches).To(MatchJSON(`[]`)) +} diff --git a/pkg/agentinstall/upgrade_live_spec_test.go b/pkg/agentinstall/upgrade_live_spec_test.go new file mode 100644 index 0000000..01d3d4a --- /dev/null +++ b/pkg/agentinstall/upgrade_live_spec_test.go @@ -0,0 +1,134 @@ +package agentinstall_test + +import ( + "archive/zip" + "bytes" + "context" + "encoding/json" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "regexp" + "time" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/agentinstall" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func versionedAgent(original, version string) string { + GinkgoHelper() + data, err := os.ReadFile(original) + Expect(err).NotTo(HaveOccurred()) + input, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + Expect(err).NotTo(HaveOccurred()) + var output bytes.Buffer + archive := zip.NewWriter(&output) + for _, file := range input.File { + reader, err := file.Open() + Expect(err).NotTo(HaveOccurred()) + content, err := io.ReadAll(reader) + Expect(err).NotTo(HaveOccurred()) + Expect(reader.Close()).To(Succeed()) + if file.Name == "META-INF/MANIFEST.MF" { + pattern := regexp.MustCompile(`(?m)^Implementation-Version:[^\n]*`) + Expect(pattern.Match(content)).To(BeTrue()) + content = pattern.ReplaceAll(content, []byte("Implementation-Version: "+version+"\r")) + } + writer, err := archive.CreateHeader(&file.FileHeader) + Expect(err).NotTo(HaveOccurred()) + _, err = writer.Write(content) + Expect(err).NotTo(HaveOccurred()) + } + Expect(archive.Close()).To(Succeed()) + path := filepath.Join(GinkgoT().TempDir(), "telltale-agent.jar") + Expect(os.WriteFile(path, output.Bytes(), 0o600)).To(Succeed()) + return path +} + +func attachUpgrade(attachment agentinstall.Attachment) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + args := (agentinstall.Java{}).Attach(attachment) + return exec.CommandContext(ctx, args[0], args[1:]...).CombinedOutput() +} + +var _ = Describe("live attached agent upgrades", func() { + It("replaces repeated runtime versions in one JVM and preserves the old runtime when a capture is active", func() { + workload := startProfilesWorkload() + connected := connectProfilesHTTP(workload) + source := connected.agent.Hello.Resource["telltale.source.instance"] + attachment := connected.attachment + attachment.Artifact = versionedAgent(attachment.Artifact, "upgrade-fixture-two") + output, err := attachUpgrade(attachment) + Expect(err).To(HaveOccurred(), string(output)) + Expect(string(output)).To(ContainSubstring("--upgrade")) + attachment.Upgrade = true + session, err := connected.agent.Arm(context.Background(), agentclient.ArmRequest{Probe: "http_top", Mutates: true, AllowMutation: true}) + Expect(err).NotTo(HaveOccurred()) + output, err = attachUpgrade(attachment) + Expect(err).To(HaveOccurred(), string(output)) + Expect(string(output)).To(ContainSubstring("active captures")) + Expect(session.Disarm(context.Background())).To(Succeed()) + for _, version := range []string{"upgrade-fixture-two", "upgrade-fixture-three"} { + attachment.Artifact = versionedAgent(connected.attachment.Artifact, version) + output, err = attachUpgrade(attachment) + Expect(err).NotTo(HaveOccurred(), string(output)) + port, err := os.ReadFile(connected.portFile) + Expect(err).NotTo(HaveOccurred()) + connected.client.URL = "http://127.0.0.1:" + string(port) + health, err := connected.client.Health(context.Background()) + Expect(err).NotTo(HaveOccurred()) + Expect(health).To(And(HaveField("Status", "ok"), HaveField("RuntimeVersion", version), HaveField("UpgradeAPIVersion", 1))) + hello, err := connected.client.Hello(context.Background()) + Expect(err).NotTo(HaveOccurred()) + Expect(hello.SDKVersion).To(Equal(version)) + Expect(hello.Resource).To(And(HaveKeyWithValue("process.pid", connected.agent.Hello.Resource["process.pid"]), HaveKeyWithValue("telltale.source.instance", source))) + _, err = connected.client.Call(context.Background(), "jvm:read", agentclient.CallRequest{Op: "jvm.metrics"}) + Expect(err).NotTo(HaveOccurred()) + } + occupied, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(occupied.Close) + config, err := os.ReadFile(attachment.Config) + Expect(err).NotTo(HaveOccurred()) + var values map[string]any + Expect(json.Unmarshal(config, &values)).To(Succeed()) + values["http"].(map[string]any)["port"] = occupied.Addr().(*net.TCPAddr).Port + invalid, err := json.Marshal(values) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(attachment.Config, invalid, 0o600)).To(Succeed()) + attachment.Artifact = versionedAgent(connected.attachment.Artifact, "upgrade-fixture-failed") + output, err = attachUpgrade(attachment) + Expect(err).To(HaveOccurred(), string(output)) + Expect(string(output)).To(ContainSubstring("previous runtime was restored")) + port, err := os.ReadFile(connected.portFile) + Expect(err).NotTo(HaveOccurred()) + connected.client.URL = "http://127.0.0.1:" + string(port) + health, err := connected.client.Health(context.Background()) + Expect(err).NotTo(HaveOccurred()) + Expect(health.RuntimeVersion).To(Equal("upgrade-fixture-three")) + }) +}) + +func attachUpgradedProfilesHTTP(workload profilesWorkload) *agentclient.Registry { + GinkgoHelper() + connected := connectProfilesHTTP(workload) + attachment := connected.attachment + attachment.Upgrade = true + attachment.Artifact = versionedAgent(attachment.Artifact, "profiles-upgrade-fixture") + output, err := attachUpgrade(attachment) + Expect(err).NotTo(HaveOccurred(), string(output)) + port, err := os.ReadFile(connected.portFile) + Expect(err).NotTo(HaveOccurred()) + client := connected.client + agent, err := agentclient.DialHTTP(context.Background(), agentclient.HTTPEndpoint{ID: client.Target, URL: "http://127.0.0.1:" + string(port), Client: client.Client, Tokens: client.Tokens, Resource: map[string]string{"telltale.deployer": "profiles-fixture"}}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(agent.Close) + registry := agentclient.NewRegistry() + Expect(registry.Add(agent)).To(Succeed()) + return registry +} diff --git a/pkg/agentinstall/upgrade_spec_test.go b/pkg/agentinstall/upgrade_spec_test.go new file mode 100644 index 0000000..221ce7b --- /dev/null +++ b/pkg/agentinstall/upgrade_spec_test.go @@ -0,0 +1,23 @@ +package agentinstall_test + +import ( + "context" + "errors" + + "github.com/flanksource/telltale/pkg/agentinstall" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("agent runtime upgrades", func() { + It("passes explicit upgrade intent to the target JVM attachment", func() { + runtime := &attachHomeRuntime{} + _, err := (agentinstall.Installer{Runtime: runtime, Transport: attachHomeTransport{exe: "/opt/java/bin/java"}}).Install(context.Background(), agentinstall.Options{ + Destination: agentinstall.Destination{Kind: "kubernetes", Namespace: "shop", Pod: "orders", Container: "app"}, + PID: 321, Directory: GinkgoT().TempDir(), Service: "orders", Upgrade: true, + HTTP: &agentinstall.HTTPListener{Bind: "127.0.0.1", Port: 7070, Issuer: "fixture", Audience: "orders", JWKS: []byte(`{"keys":[]}`)}, + }) + Expect(errors.Is(err, errAttachRecorded)).To(BeTrue(), "%v", err) + Expect(runtime.attachment.Upgrade).To(BeTrue()) + }) +}) diff --git a/pkg/agentinstall/write_internal_test.go b/pkg/agentinstall/write_internal_test.go new file mode 100644 index 0000000..8592db1 --- /dev/null +++ b/pkg/agentinstall/write_internal_test.go @@ -0,0 +1,44 @@ +package agentinstall + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// recording runs each command on this host, as the local destination does, +// and keeps the argv it was given. +type recording struct { + commands [][]string +} + +func (r *recording) Run(ctx context.Context, args []string, input io.Reader) ([]byte, error) { + r.commands = append(r.commands, args) + return CommandTransport{Destination: Destination{Kind: "local"}}.Run(ctx, args, input) +} + +var _ = Describe("staging a file on the target", func() { + It("streams it as a tar archive, private, and moves it into place whole", func() { + dir := GinkgoT().TempDir() + target := filepath.Join(dir, "telltale-agent.jar") + Expect(os.WriteFile(target, []byte("an older agent"), 0o600)).To(Succeed()) + data := bytes.Repeat([]byte("telltale-agent"), 64<<10) + transport := &recording{} + + Expect(write(context.Background(), transport, target, data)).To(Succeed()) + + Expect(os.ReadFile(target)).To(Equal(data)) + info, err := os.Stat(target) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + entries, err := os.ReadDir(dir) + Expect(err).NotTo(HaveOccurred()) + Expect(entries).To(HaveLen(1), "the staged file is moved, not left beside it") + Expect(transport.commands[0][:6]).To(Equal([]string{"tar", "-x", "-m", "--no-same-owner", "-f", "-"}), "a bare dd can see end of input before a pod's exec attaches stdin") + }) +}) diff --git a/pkg/credentials/apitoken/token.go b/pkg/credentials/apitoken/token.go index 8789824..3674c7f 100644 --- a/pkg/credentials/apitoken/token.go +++ b/pkg/credentials/apitoken/token.go @@ -45,22 +45,25 @@ const ( // ScopeAPI authorizes the HTTP API. ScopeAPI Scope = "api" ScopeAgent Scope = "agent" + ScopeProxy Scope = "proxy" ) // ParseScope validates a scope selector, naming the alternatives. func ParseScope(value string) (Scope, error) { switch scope := Scope(strings.ToLower(strings.TrimSpace(value))); scope { - case ScopeGit, ScopeAPI, ScopeAgent: + case ScopeGit, ScopeAPI, ScopeAgent, ScopeProxy: return scope, nil case "": - return "", fmt.Errorf("a token scope is required; want one of: %s, %s, %s", ScopeGit, ScopeAPI, ScopeAgent) + return "", fmt.Errorf("a token scope is required; want one of: %s, %s, %s, %s", ScopeGit, ScopeAPI, ScopeAgent, ScopeProxy) default: - return "", fmt.Errorf("invalid token scope %q; want one of: %s, %s, %s", value, ScopeGit, ScopeAPI, ScopeAgent) + return "", fmt.Errorf("invalid token scope %q; want one of: %s, %s, %s, %s", value, ScopeGit, ScopeAPI, ScopeAgent, ScopeProxy) } } // Valid reports whether s names a scope. -func (s Scope) Valid() bool { return s == ScopeGit || s == ScopeAPI || s == ScopeAgent } +func (s Scope) Valid() bool { + return s == ScopeGit || s == ScopeAPI || s == ScopeAgent || s == ScopeProxy +} // nameRe is the shape a token name must take. It is the §3.2 ref-segment shape // deliberately, not by coincidence: a git-scoped token's name becomes an agent diff --git a/pkg/credentials/apitoken/verify.go b/pkg/credentials/apitoken/verify.go index 234ca78..f2d59ca 100644 --- a/pkg/credentials/apitoken/verify.go +++ b/pkg/credentials/apitoken/verify.go @@ -31,6 +31,7 @@ type Record struct { Pool bool PoolAgents []string MaxAgents int + Endpoints []string ExpiresAt *time.Time RevokedAt *time.Time } diff --git a/pkg/definition/builtin/clicky.yaml b/pkg/definition/builtin/clicky.yaml new file mode 100644 index 0000000..2a1ed82 --- /dev/null +++ b/pkg/definition/builtin/clicky.yaml @@ -0,0 +1,31 @@ +apiVersion: telltale.flanksource.com/v1alpha1 +kind: TraceDefinition +metadata: + name: clicky + annotations: + telltale.flanksource.com/summary: Record a clicky process's executions, entity operations, tasks, HTTP, logs and heap through its telltale agent + telltale.flanksource.com/examples: | + 1234 # record process 1234 until Ctrl+C + 1234 --duration 1m # record it for a minute + 1234 --every 5s # read its heap every 5 seconds +spec: + description: >- + Arms every probe github.com/flanksource/clicky/telltale registers in a clicky process: the commands it runs, its entity + operations, its tasks' transitions and progress, its commons HTTP exchanges and its commons logs. The process must run + the agent (telltale.Start, or the auto import with TELLTALE_AGENT=1); a process without one is refused. Its heap + profile is read through the agent every 30s and as the run stops. + every: 30s + collect: + clicky: + exec: + - {} + entity: + - {} + task: + - {} + http: + - features: [http] + logs: + - {} + pprof: + - profile: heap diff --git a/pkg/definition/compile_spec_test.go b/pkg/definition/compile_spec_test.go index 1ff5e11..422fe55 100644 --- a/pkg/definition/compile_spec_test.go +++ b/pkg/definition/compile_spec_test.go @@ -40,8 +40,8 @@ var _ = Describe("a compiled definition", func() { "jvm_jfr", `{"target":{"seam":"agent","selector":{"service.name":"pricing"},"wait":true,"allowMutation":true},"engine":"jfr","settings":"profile","threads":true,"maxStackDepth":64}`, ""), Entry("native thread stacks", " collect: {process: {nativeStacks: [{}]}}", "stacks", `{"target":{"seam":"process"}}`, `{"transport":"native"}`), - Entry("process resources", " collect: {process: {resources: [{interval: 1s, facets: [ports, connections]}]}}", - "process_samples", `{"target":{"seam":"process"},"interval":"1s","facets":["ports","connections"]}`, ""), + Entry("process resources", " collect: {process: {resources: [{interval: 1s, facets: [sockets, connections]}]}}", + "process", `{"target":{"seam":"process"},"interval":"1s","facets":["sockets","connections"]}`, ""), Entry("a Prometheus scrape", " collect: {prometheus: {scrape: [{endpoints: ['http://localhost:8080/metrics'], interval: 15s, labels: {env: dev}}]}}", "prometheus", `{"target":{"seam":"host"},"endpoints":["http://localhost:8080/metrics"],"interval":"15s","labels":{"env":"dev"}}`, ""), Entry("kubernetes logs", " collect: {logs: {kubernetes: [{namespace: checkout, podSelector: app=checkout, container: api}]}}", diff --git a/pkg/definition/coverage_spec_test.go b/pkg/definition/coverage_spec_test.go index 90aecd1..b8b552b 100644 --- a/pkg/definition/coverage_spec_test.go +++ b/pkg/definition/coverage_spec_test.go @@ -12,7 +12,6 @@ import ( "github.com/flanksource/telltale/pkg/definition" "github.com/flanksource/telltale/pkg/traces" - "github.com/flanksource/telltale/pkg/tracing/process" ) // path is a list's path, section.group.type. @@ -40,14 +39,17 @@ var kindProviders = []string{"recordstore", "sqlserver-xevent", "legacy-trace"} var _ = Describe("the typed lists", func() { lists := definition.Lists() - It("map every registered kind to a list", func() { - // collect.process.tree configures the run-wide process tree, which - // every command or pid run records, rather than arming a capture. - covered := map[string]bool{process.TreeKind: true} + It("map every registered kind to a list, a seam preset to one of its base kind on its seam", func() { + covered := map[string]bool{} for _, at := range lists { covered[at.Kind] = true + covered[at.Kind+"@"+string(at.Seam)] = true } for _, registration := range traces.Registrations() { + if registration.Base != "" { + Expect(covered).To(HaveKey(registration.Base+"@"+string(registration.DefaultSeam)), "no list compiles to %s, the base of preset %s, on seam %s", registration.Base, registration.Name, registration.DefaultSeam) + continue + } Expect(covered).To(HaveKey(registration.Name), "no list compiles to %s", registration.Name) } }) diff --git a/pkg/definition/crd.go b/pkg/definition/crd.go index 1fa4e9c..bed0f40 100644 --- a/pkg/definition/crd.go +++ b/pkg/definition/crd.go @@ -29,18 +29,22 @@ const dynamicValue = "Dynamic response value" // atomic (entry names are optional, so no list can be keyed), free-form // leaves preserving their fields, and no connection required, as each may be // a default. -func Schema() (map[string]any, error) { - data, err := json.Marshal(rpc.SchemaForStruct(TraceDefinition{})) +func Schema() (map[string]any, error) { return SchemaOf(TraceDefinition{}) } + +// SchemaOf is resource's openAPIV3Schema as a structural schema, reflected the +// way TraceDefinition's is: a resource of this group with a spec. +func SchemaOf(resource any) (map[string]any, error) { + data, err := json.Marshal(rpc.SchemaForStruct(resource)) if err != nil { - return nil, fmt.Errorf("reflect trace definition schema: %w", err) + return nil, fmt.Errorf("reflect %T schema: %w", resource, err) } var schema map[string]any if err := json.Unmarshal(data, &schema); err != nil { - return nil, fmt.Errorf("reflect trace definition schema: %w", err) + return nil, fmt.Errorf("reflect %T schema: %w", resource, err) } properties, _ := schema["properties"].(map[string]any) if properties == nil { - return nil, fmt.Errorf("reflect trace definition schema: no properties in %s", data) + return nil, fmt.Errorf("reflect %T schema: no properties in %s", resource, data) } delete(properties, "metadata") schema["required"] = []any{"apiVersion", "kind", "spec"} @@ -58,7 +62,7 @@ func structural(node map[string]any, path string) (map[string]any, error) { if ref, ok := node["$ref"]; ok { return nil, fmt.Errorf("schema %s refers to %v: the reflector inlines every type, so a $ref is a schema the CRD cannot carry", path, ref) } - if node["description"] == dynamicValue || strings.HasPrefix(fmt.Sprint(node["description"]), "Recursive reference to ") { + if len(node) == 0 || node["description"] == dynamicValue || strings.HasPrefix(fmt.Sprint(node["description"]), "Recursive reference to ") { return map[string]any{"x-kubernetes-preserve-unknown-fields": true}, nil } out := map[string]any{} @@ -124,13 +128,19 @@ func CRD() ([]byte, error) { if err != nil { return nil, err } + return CRDOf(Kind, Plural, "pkg/definition", schema) +} + +// CRDOf is the CustomResourceDefinition of kind in this group with schema, +// marked as generated from source. +func CRDOf(kind, plural, source string, schema map[string]any) ([]byte, error) { resource := map[string]any{ "apiVersion": "apiextensions.k8s.io/v1", "kind": "CustomResourceDefinition", - "metadata": map[string]any{"name": Plural + "." + Group}, + "metadata": map[string]any{"name": plural + "." + Group}, "spec": map[string]any{ "group": Group, - "names": map[string]any{"kind": Kind, "listKind": Kind + "List", "plural": Plural, "singular": strings.ToLower(Kind)}, + "names": map[string]any{"kind": kind, "listKind": kind + "List", "plural": plural, "singular": strings.ToLower(kind)}, "scope": "Namespaced", "versions": []any{map[string]any{ "name": Version, "served": true, "storage": true, @@ -140,7 +150,7 @@ func CRD() ([]byte, error) { } data, err := yaml.Marshal(resource) if err != nil { - return nil, fmt.Errorf("encode trace definition CRD: %w", err) + return nil, fmt.Errorf("encode %s CRD: %w", kind, err) } - return append([]byte("# Generated by `task generate` from pkg/definition; do not edit.\n"), data...), nil + return append([]byte("# Generated by `task generate` from "+source+"; do not edit.\n"), data...), nil } diff --git a/pkg/definition/definition_suite_test.go b/pkg/definition/definition_suite_test.go index 2e5aadb..84258bc 100644 --- a/pkg/definition/definition_suite_test.go +++ b/pkg/definition/definition_suite_test.go @@ -37,5 +37,11 @@ func decode(yaml string) *definition.TraceDefinition { var examples = []string{ filepath.Join("..", "..", "examples", "definitions", "checkout.yaml"), filepath.Join("..", "..", "examples", "definitions", "logs-and-queries.yaml"), + filepath.Join("..", "..", "examples", "definitions", "java-app.yaml"), + filepath.Join("..", "..", "examples", "definitions", "java-build.yaml"), + filepath.Join("..", "..", "examples", "definitions", "go-app.yaml"), + filepath.Join("..", "..", "examples", "definitions", "go-build.yaml"), + filepath.Join("..", "..", "examples", "definitions", "node-app.yaml"), + filepath.Join("..", "..", "examples", "definitions", "node-build.yaml"), filepath.Join("testdata", "all-kinds.yaml"), } diff --git a/pkg/definition/entries_host.go b/pkg/definition/entries_host.go index 14cf556..a1f0c1d 100644 --- a/pkg/definition/entries_host.go +++ b/pkg/definition/entries_host.go @@ -4,25 +4,17 @@ import ( "errors" "github.com/flanksource/commons-db/models" - "github.com/flanksource/telltale/pkg/tracing/groups" "github.com/flanksource/telltale/pkg/tracing/logs" "github.com/flanksource/telltale/pkg/tracing/opensearch" "github.com/flanksource/telltale/pkg/tracing/prometheus" "github.com/flanksource/telltale/pkg/tracing/spans" + "github.com/flanksource/telltale/pkg/tracing/sqltop" "github.com/flanksource/telltale/pkg/tracing/sqlserver/deadlock" "github.com/flanksource/telltale/pkg/tracing/sqlserver/plancache" "github.com/flanksource/telltale/pkg/tracing/sqlserver/xetrace" "github.com/flanksource/telltale/pkg/tracing/sqlserver/xevent" ) -// ProcessGroups records what each tool's processes on the host use. -type ProcessGroups struct { - Named - groups.Options -} - -func (p *ProcessGroups) options(Located) (any, any, error) { return p.Options, nil, nil } - // Scrape scrapes Prometheus endpoints into the managed Prometheus. type Scrape struct { Named @@ -98,6 +90,28 @@ func (d *DeadlockReport) options(Located) (any, any, error) { return d.Options, func (d *DeadlockReport) connection(Located) connectionSlot { return sqlServerSlot(&d.Connection) } +// SQLServerTop samples the requests a SQL Server is running. +type SQLServerTop struct { + Named + sqltop.Options +} + +func (t *SQLServerTop) options(Located) (any, any, error) { return t.Options, nil, nil } + +func (t *SQLServerTop) connection(Located) connectionSlot { return sqlServerSlot(&t.Connection) } + +// PostgresTop samples the queries a Postgres server is running. +type PostgresTop struct { + Named + sqltop.Options +} + +func (t *PostgresTop) options(Located) (any, any, error) { return t.Options, nil, nil } + +func (t *PostgresTop) connection(Located) connectionSlot { + return connectionSlot{field: &t.Connection, connType: models.ConnectionTypePostgres} +} + func sqlServerSlot(field *string) connectionSlot { return connectionSlot{field: field, connType: models.ConnectionTypeSQLServer} } diff --git a/pkg/definition/entries_jvm.go b/pkg/definition/entries_jvm.go new file mode 100644 index 0000000..4ca750d --- /dev/null +++ b/pkg/definition/entries_jvm.go @@ -0,0 +1,41 @@ +package definition + +import "github.com/flanksource/telltale/pkg/tracing/jvmtrace" + +type JVMNotifications struct { + Named + Agent + jvmtrace.NotificationOptions +} + +func (p *JVMNotifications) options(Located) (any, any, error) { + return p.NotificationOptions, nil, p.Validate() +} + +type JVMExpression struct { + Named + Agent + jvmtrace.ExpressionOptions +} + +func (p *JVMExpression) options(Located) (any, any, error) { + return p.ExpressionOptions, nil, p.Validate() +} + +type JVMMethods struct { + Named + Agent + jvmtrace.Options +} + +func (p *JVMMethods) options(Located) (any, any, error) { return p.Options, nil, p.Validate() } + +type JVMRedefine struct { + Named + Agent + jvmtrace.RedefineOptions +} + +func (p *JVMRedefine) options(Located) (any, any, error) { + return p.RedefineOptions, nil, p.Validate() +} diff --git a/pkg/definition/entries_protocols.go b/pkg/definition/entries_protocols.go new file mode 100644 index 0000000..d9acdae --- /dev/null +++ b/pkg/definition/entries_protocols.go @@ -0,0 +1,77 @@ +package definition + +import ( + "errors" + "slices" + + "github.com/flanksource/clicky/rpc" + "github.com/flanksource/telltale/pkg/tracing/httptop" + "github.com/flanksource/telltale/pkg/tracing/jms" + "github.com/flanksource/telltale/pkg/tracing/jmstop" + "github.com/flanksource/telltale/pkg/tracing/sqltop" + "github.com/flanksource/telltale/pkg/tracing/target" +) + +type JVMSQLProbe struct{ SQLProbe } + +func (p *JVMSQLProbe) options(Located) (any, any, error) { + options := p.Options + if len(options.Connections) == 0 { + options.Connections = []string{"*"} + } + if slices.Contains(options.Connections, "self") { + return nil, nil, errors.New("connections self is Go-only; use * for JVM SQL") + } + return options, nil, options.Validate() +} + +func (JVMSQLProbe) JSONSchema() map[string]any { + schema := rpc.SchemaForStruct(SQLProbe{}) + schema.Required = slices.DeleteFunc(schema.Required, func(name string) bool { return name == "connections" }) + schema.Properties["connections"].Default = []string{"*"} + schema.Properties["connections"].Description = "JVM connection identities to observe; * selects every observed connection" + return map[string]any{"type": schema.Type, "properties": schema.Properties, "required": schema.Required} +} + +type HTTPTopProbe struct { + Named + Agent + httptop.Options +} + +func (p *HTTPTopProbe) options(Located) (any, any, error) { + return p.Options, nil, p.Validate() +} + +type JMSProbe struct { + Named + Agent + jms.Options +} + +type JMSTopProbe struct { + Named + Agent + Poll string `json:"poll,omitempty"` + Top int `json:"top,omitempty"` + Destinations []string `json:"destinations,omitempty"` +} + +func (p *JMSTopProbe) options(Located) (any, any, error) { + params := jmstop.Params{Poll: p.Poll, Top: p.Top, Destinations: p.Destinations} + return params, nil, params.Validate() +} + +func (p *JMSProbe) options(Located) (any, any, error) { + return p.Options, nil, p.Validate() +} + +type JVMTop struct { + Named + Agent + sqltop.Options +} + +func (p *JVMTop) options(Located) (any, any, error) { + return p.Options, nil, (sqltop.Params{Target: target.Target{Seam: target.Agent}, Options: p.Options}).Validate() +} diff --git a/pkg/definition/entries_query.go b/pkg/definition/entries_query.go index 493d250..7e94371 100644 --- a/pkg/definition/entries_query.go +++ b/pkg/definition/entries_query.go @@ -4,7 +4,7 @@ import ( "github.com/flanksource/commons-db/models" "github.com/flanksource/commons-db/query/providers" "github.com/flanksource/commons-db/query/providers/sqlite" - tracequery "github.com/flanksource/commons-db/tracing/query" + tracequery "github.com/flanksource/telltale/pkg/tracing/query" ) // Window is when and how often a query entry reads, and how many rows one diff --git a/pkg/definition/entries_runtime.go b/pkg/definition/entries_runtime.go index 4f46f10..eb7e24c 100644 --- a/pkg/definition/entries_runtime.go +++ b/pkg/definition/entries_runtime.go @@ -4,16 +4,18 @@ import ( "errors" "fmt" + "github.com/flanksource/telltale/pkg/tracing/entity" "github.com/flanksource/telltale/pkg/tracing/exec" "github.com/flanksource/telltale/pkg/tracing/goprof" tracehttp "github.com/flanksource/telltale/pkg/tracing/http" "github.com/flanksource/telltale/pkg/tracing/jfr" "github.com/flanksource/telltale/pkg/tracing/llm" "github.com/flanksource/telltale/pkg/tracing/logs" - "github.com/flanksource/telltale/pkg/tracing/process" + "github.com/flanksource/telltale/pkg/tracing/processcapture" "github.com/flanksource/telltale/pkg/tracing/spans" tracesql "github.com/flanksource/telltale/pkg/tracing/sql" "github.com/flanksource/telltale/pkg/tracing/stacks" + "github.com/flanksource/telltale/pkg/tracing/task" "github.com/flanksource/telltale/pkg/tracing/terminal" ) @@ -113,10 +115,10 @@ func (*NativeStacks) options(Located) (any, any, error) { // Resources samples every process of the tree. type Resources struct { Named - process.SampleOptions + processcapture.Options } -func (r *Resources) options(Located) (any, any, error) { return r.SampleOptions, nil, nil } +func (r *Resources) options(Located) (any, any, error) { return r.Options, nil, nil } // FlightRecorder starts a JFR recording in a JVM. type FlightRecorder struct { @@ -180,6 +182,29 @@ func (p *ExecProbe) options(Located) (any, any, error) { return exec.Options{Name: p.Executable, TailBytes: p.TailBytes}, nil, nil } +// EntityProbe records the entity operations a clicky agent runs. +type EntityProbe struct { + Named + Agent + entity.Options +} + +func (p *EntityProbe) options(Located) (any, any, error) { return p.Options, nil, nil } + +// TaskProbe records the tasks a clicky agent runs. TaskName is the task +// kind's name glob, renamed so it does not take the entry's name. +type TaskProbe struct { + Named + Agent + TaskName string `json:"taskName,omitempty" clicky:"title=Task name,desc=Task name glob"` + Group string `json:"group,omitempty" clicky:"title=Group,desc=Parent group name glob"` + ProgressIntervalMs int `json:"progressIntervalMs,omitempty" clicky:"title=Progress interval,desc=Minimum milliseconds between progress records of a task"` +} + +func (p *TaskProbe) options(Located) (any, any, error) { + return task.Options{Name: p.TaskName, Group: p.Group, ProgressIntervalMs: p.ProgressIntervalMs}, nil, nil +} + // LLMProbe records the provider calls an agent or telltale makes. type LLMProbe struct { Named diff --git a/pkg/definition/inline.go b/pkg/definition/inline.go index a8b6788..c554086 100644 --- a/pkg/definition/inline.go +++ b/pkg/definition/inline.go @@ -29,7 +29,7 @@ type inlined interface { // Inline is a start request's ad-hoc captures as a definition, so they // compile as a stored definition's entries do: each in the one typed list of // its kind and seam whose entry carries every param it sets. Captures maps -// each entry's pointer to its capture's, /traces/. +// each entry's pointer to its capture's request pointer. func Inline(captures []telltale.RequestedCapture) (*TraceDefinition, map[string]string, []telltale.FieldError) { def := &TraceDefinition{ TypeMeta: metav1.TypeMeta{APIVersion: APIVersion, Kind: Kind}, @@ -41,10 +41,10 @@ func Inline(captures []telltale.RequestedCapture) (*TraceDefinition, map[string] for _, capture := range captures { at, err := placeCapture(&def.Spec, capture) if err != nil { - fields = append(fields, telltale.FieldError{InstancePath: "/traces/" + capture.Kind, Message: err.Error()}) + fields = append(fields, telltale.FieldError{InstancePath: capture.Pointer, Message: err.Error()}) continue } - pointers[at.Pointer()] = "/traces/" + capture.Kind + pointers[at.Pointer()] = capture.Pointer } return def, pointers, fields } @@ -69,6 +69,17 @@ func placeCapture(spec *Spec, capture telltale.RequestedCapture) (Located, error } matches = append(matches, at) } + if own := slices.DeleteFunc(slices.Clone(matches), func(at Located) bool { return at.Preset }); len(own) > 0 { + matches = own + } + if len(matches) > 1 { + preferred := slices.DeleteFunc(slices.Clone(matches), func(at Located) bool { + return (at.Section == SectionInstrument) != capture.Target.AllowMutation + }) + if len(preferred) > 0 { + matches = preferred + } + } switch { case len(tried) == 0: return Located{}, unarmable(capture) @@ -83,7 +94,7 @@ func placeCapture(spec *Spec, capture telltale.RequestedCapture) (Located, error } return Located{}, fmt.Errorf("%s each carry these params: start a trace definition that names one", strings.Join(paths, ", ")) } - appendEntry(spec, matches[0]) + matches[0].Index = appendEntry(spec, matches[0]) return matches[0], nil } @@ -170,14 +181,15 @@ func carries(want, got any) bool { return true } -// appendEntry appends at's entry to its list in spec. -func appendEntry(spec *Spec, at Located) { +// appendEntry appends at's entry to its list in spec, at the index it returns. +func appendEntry(spec *Spec, at Located) int { list := fieldByJSON(reflect.ValueOf(spec).Elem(), at.Section) if at.Group != "" { list = fieldByJSON(list, at.Group) } list = fieldByJSON(list, at.Type) list.Set(reflect.Append(list, reflect.ValueOf(at.Entry).Elem())) + return list.Len() - 1 } func fieldByJSON(value reflect.Value, name string) reflect.Value { @@ -195,6 +207,12 @@ func (p *ExecProbe) inline(map[string]any) error { return nil } +// inline takes the task kind's name glob, which decodes as the entry's name. +func (p *TaskProbe) inline(map[string]any) error { + p.TaskName, p.Name = p.Name, "" + return nil +} + // inline takes the spans kind's opensearch block. func (s *OpenSearchSpans) inline(params map[string]any) error { data, err := json.Marshal(params["opensearch"]) diff --git a/pkg/definition/inline_spec_test.go b/pkg/definition/inline_spec_test.go new file mode 100644 index 0000000..a2ba705 --- /dev/null +++ b/pkg/definition/inline_spec_test.go @@ -0,0 +1,46 @@ +package definition_test + +import ( + "strconv" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/pkg/definition" + "github.com/flanksource/telltale/pkg/tracing/target" +) + +var _ = Describe("an ad-hoc capture through an agent", func() { + agent := target.Target{Seam: target.Agent, Selector: map[string]string{"service.name": "example"}} + + DescribeTable("lands in the one list that takes it, a preset group's only when no other does", + func(kind string, params map[string]any, list string) { + _, pointers, fields := definition.Inline([]telltale.RequestedCapture{{Kind: kind, Pointer: "/traces/" + kind, Params: params, Target: agent, Seam: target.Agent}}) + Expect(fields).To(BeEmpty()) + Expect(pointers).To(Equal(map[string]string{"/spec/collect/" + list + "/0": "/traces/" + kind})) + }, + Entry("http, which collect.go also arms", "http", map[string]any{"features": []any{"http"}}, "go/http"), + Entry("exec, which collect.go also arms", "exec", map[string]any{"name": "git"}, "go/exec"), + Entry("entity, which only the clicky preset arms", "entity", map[string]any{"entity": "widget"}, "clicky/entity"), + Entry("task, which only the clicky preset arms", "task", map[string]any{"name": "deploy *"}, "clicky/task"), + ) + + It("maps each of a kind's captures to its own entry and request element", func() { + capture := func(index int, executable string) telltale.RequestedCapture { + return telltale.RequestedCapture{Kind: "exec", Pointer: "/traces/exec/" + strconv.Itoa(index), Params: map[string]any{"name": executable}, Target: agent, Seam: target.Agent} + } + def, pointers, fields := definition.Inline([]telltale.RequestedCapture{capture(0, "git"), capture(1, "make")}) + Expect(fields).To(BeEmpty()) + Expect(def.Spec.Collect.Go.Exec).To(HaveExactElements(HaveField("Executable", "git"), HaveField("Executable", "make"))) + Expect(pointers).To(Equal(map[string]string{"/spec/collect/go/exec/0": "/traces/exec/0", "/spec/collect/go/exec/1": "/traces/exec/1"})) + }) + + It("selects JVM instrumentation when a shared agent probe is allowed to mutate", func() { + mutable := agent + mutable.AllowMutation = true + _, pointers, fields := definition.Inline([]telltale.RequestedCapture{{Kind: "http", Pointer: "/traces/http", Params: map[string]any{"features": []any{"http"}}, Target: mutable, Seam: target.Agent}}) + Expect(fields).To(BeEmpty()) + Expect(pointers).To(Equal(map[string]string{"/spec/instrument/jvm/http/0": "/traces/http"})) + }) +}) diff --git a/pkg/definition/jvm_protocols_spec_test.go b/pkg/definition/jvm_protocols_spec_test.go new file mode 100644 index 0000000..58a47cd --- /dev/null +++ b/pkg/definition/jvm_protocols_spec_test.go @@ -0,0 +1,69 @@ +package definition_test + +import ( + "encoding/json" + + "github.com/flanksource/clicky/rpc" + "github.com/flanksource/telltale/pkg/definition" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("JVM protocol definitions", func() { + It("keeps the optional JVM SQL selection separate from the shared Go SQL schema", func() { + encoded, err := json.Marshal(rpc.SchemaForStruct(definition.JVMSQLProbe{})) + Expect(err).NotTo(HaveOccurred()) + var jvm struct { + Required []string `json:"required"` + Properties map[string]struct { + Default []string `json:"default"` + } `json:"properties"` + } + Expect(json.Unmarshal(encoded, &jvm)).To(Succeed()) + Expect(jvm.Required).NotTo(ContainElement("connections")) + Expect(jvm.Properties["connections"].Default).To(Equal([]string{"*"})) + Expect(rpc.SchemaForStruct(definition.SQLProbe{}).Required).To(ContainElement("connections")) + }) + + It("refuses the Go-only SQL self selector in JVM definitions", func() { + _, err := definition.Specs(decode(header+"spec: {instrument: {jvm: {sql: [{agent: {id: java/checkout}, connections: [self]}]}}}"), definition.Options{}) + Expect(err).To(MatchError(ContainSubstring("self is Go-only"))) + }) + + It("defaults omitted JVM SQL connections to every observed connection", func() { + specs, err := definition.Specs(decode(header+"spec: {instrument: {jvm: {sql: [{agent: {id: java/checkout}}]}}}"), definition.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(specs).To(HaveLen(1)) + var params struct { + Connections []string `json:"connections"` + } + Expect(json.Unmarshal(specs[0].Params, ¶ms)).To(Succeed()) + Expect(params.Connections).To(Equal([]string{"*"})) + }) + + DescribeTable("compiles an agent capture with runtime mutation enabled", + func(entry, kind string) { + specs, err := definition.Specs(decode(header+"spec: {instrument: {jvm: {"+entry+"}}}"), definition.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(specs).To(HaveLen(1)) + Expect(specs[0].Kind).To(Equal(kind)) + var params struct { + Target struct { + Seam string `json:"seam"` + Agent string `json:"agent"` + AllowMutation bool `json:"allowMutation"` + } `json:"target"` + } + Expect(json.Unmarshal(specs[0].Params, ¶ms)).To(Succeed()) + Expect(params.Target.Seam).To(Equal("agent")) + Expect(params.Target.Agent).To(Equal("java/checkout")) + Expect(params.Target.AllowMutation).To(BeTrue()) + }, + Entry("HTTP", "http: [{agent: {id: java/checkout}, features: [http]}]", "http"), + Entry("HTTP top", "httpTop: [{agent: {id: java/checkout}, direction: incoming}]", "http_top"), + Entry("SQL", "sql: [{agent: {id: java/checkout}, connections: ['*']}]", "sql"), + Entry("SQL top without a database connection", "sqlTop: [{agent: {id: java/checkout}, poll: 1s}]", "sql_top"), + Entry("JMS", "jms: [{agent: {id: java/checkout}, direction: outgoing}]", "jms"), + Entry("span mirroring", "spans: [{agent: {id: java/checkout}}]", "spans"), + ) +}) diff --git a/pkg/definition/sections.go b/pkg/definition/sections.go index f3fc30a..1c1ea4a 100644 --- a/pkg/definition/sections.go +++ b/pkg/definition/sections.go @@ -5,7 +5,9 @@ import "github.com/flanksource/telltale/pkg/traces" // Each typed list carries the kind it compiles to in its telltale tag // (kind=), and may fix the seam (seam=) or the logs backend // (backend=). A group's tag gives the seam its lists are aimed at -// unless a list fixes its own. The tags are read by the walker (walk.go); +// unless a list fixes its own. A preset group (preset) gathers lists of kinds +// other groups also arm, so an ad-hoc capture lands in it only when no other +// list takes it. The tags are read by the walker (walk.go); // the field's json name is the path segment and the default-name part. // Proxy are the proxies a command is routed through, each on its own port @@ -28,8 +30,19 @@ type InstrumentGo struct { } type InstrumentJVM struct { - FlightRecorder []FlightRecorder `json:"flightRecorder,omitempty" clicky:"title=Flight Recorder,desc=Starts a JFR recording" telltale:"kind=jvm_jfr"` - AsyncProfiler []AsyncProfiler `json:"asyncProfiler,omitempty" clicky:"title=async-profiler,desc=Attaches async-profiler" telltale:"kind=jvm_jfr"` + Notifications []JVMNotifications `json:"notifications,omitempty" clicky:"title=JMX notifications and GC" telltale:"kind=jvm_notifications,seam=agent"` + Expressions []JVMExpression `json:"expressions,omitempty" clicky:"title=Periodic OGNL" telltale:"kind=jvm_ognl,seam=agent"` + Methods []JVMMethods `json:"methods,omitempty" clicky:"title=Watch or trace methods" telltale:"kind=jvm_trace,seam=agent"` + Redefine []JVMRedefine `json:"redefine,omitempty" clicky:"title=Redefine Java code" telltale:"kind=jvm_redefine,seam=agent"` + FlightRecorder []FlightRecorder `json:"flightRecorder,omitempty" clicky:"title=Flight Recorder,desc=Starts a JFR recording" telltale:"kind=jvm_jfr"` + AsyncProfiler []AsyncProfiler `json:"asyncProfiler,omitempty" clicky:"title=async-profiler,desc=Attaches async-profiler" telltale:"kind=jvm_jfr"` + HTTP []HTTPProbe `json:"http,omitempty" clicky:"title=HTTP exchanges" telltale:"kind=http,seam=agent"` + HTTPTop []HTTPTopProbe `json:"httpTop,omitempty" clicky:"title=Active HTTP exchanges" telltale:"kind=http_top,seam=agent"` + SQL []JVMSQLProbe `json:"sql,omitempty" clicky:"title=SQL executions" telltale:"kind=sql,seam=agent"` + SQLTop []JVMTop `json:"sqlTop,omitempty" clicky:"title=Active SQL executions" telltale:"kind=sql_top,seam=agent"` + JMS []JMSProbe `json:"jms,omitempty" clicky:"title=JMS operations" telltale:"kind=jms,seam=agent"` + JMSTop []JMSTopProbe `json:"jmsTop,omitempty" clicky:"title=JMS receive handlers" telltale:"kind=jms_top,seam=agent"` + Spans []SpansProbe `json:"spans,omitempty" clicky:"title=Existing OTel spans" telltale:"kind=spans,seam=agent"` } type InstrumentSQLServer struct { @@ -40,6 +53,7 @@ type InstrumentSQLServer struct { // runtime or the servers it reaches. type Collect struct { Go CollectGo `json:"go,omitzero" clicky:"title=Go" telltale:"seam=process"` + Clicky CollectClicky `json:"clicky,omitzero" clicky:"title=Clicky" telltale:"seam=agent,preset"` Process CollectProcess `json:"process,omitzero" clicky:"title=Process" telltale:"seam=process"` Logs CollectLogs `json:"logs,omitzero" clicky:"title=Logs" telltale:"seam=host"` Query CollectQuery `json:"query,omitzero" clicky:"title=Query" telltale:"seam=host"` @@ -47,6 +61,7 @@ type Collect struct { Prometheus CollectPrometheus `json:"prometheus,omitzero" clicky:"title=Prometheus" telltale:"seam=host"` OpenSearch CollectOpenSearch `json:"opensearch,omitzero" clicky:"title=OpenSearch" telltale:"seam=host"` SQLServer CollectSQLServer `json:"sqlserver,omitzero" clicky:"title=SQL Server" telltale:"seam=connection"` + Postgres CollectPostgres `json:"postgres,omitzero" clicky:"title=Postgres" telltale:"seam=connection"` Telltale CollectTelltale `json:"telltale,omitzero" clicky:"title=Telltale" telltale:"seam=self"` } @@ -63,13 +78,26 @@ type CollectGo struct { Spans []SpansProbe `json:"spans,omitempty" clicky:"title=Spans" telltale:"kind=spans,seam=agent"` } +// CollectClicky reads the probes github.com/flanksource/clicky/telltale +// registers in a clicky process: its executions, entity operations, tasks, +// commons HTTP and commons logs, and the SDK's Go profiles. The agent is the run's pid's, or the one an +// entry names. +type CollectClicky struct { + Exec []ExecProbe `json:"exec,omitempty" clicky:"title=Executions" telltale:"kind=exec"` + Entity []EntityProbe `json:"entity,omitempty" clicky:"title=Entity operations" telltale:"kind=entity"` + Task []TaskProbe `json:"task,omitempty" clicky:"title=Tasks" telltale:"kind=task"` + HTTP []HTTPProbe `json:"http,omitempty" clicky:"title=HTTP" telltale:"kind=http"` + Logs []LogsProbe `json:"logs,omitempty" clicky:"title=Logs" telltale:"kind=logs"` + PProf []CollectPProf `json:"pprof,omitempty" clicky:"title=pprof,desc=Heap cpu allocs or goroutine profiles read through the agent at each checkpoint" telltale:"kind=goprof"` +} + // CollectProcess reads the execution's process tree. type CollectProcess struct { // Tree is the run-wide process tree every command or pid run records; it // is one object, not a capture list, and absent reads no facets. Tree *traces.TreeOptions `json:"tree,omitempty" clicky:"title=Process tree,desc=What the run-wide process tree reads of each process"` NativeStacks []NativeStacks `json:"nativeStacks,omitempty" clicky:"title=Native stacks,desc=Thread stacks of any process: sample (darwin) eu-stack or gdb (linux)" telltale:"kind=stacks"` - Resources []Resources `json:"resources,omitempty" clicky:"title=Resources,desc=CPU memory disk IO and descriptors of every process" telltale:"kind=process_samples"` + Resources []Resources `json:"resources,omitempty" clicky:"title=Processes,desc=Startup metadata resource changes and metrics" telltale:"kind=process"` } // CollectLogs reads log sources outside the process, by the backend that @@ -101,7 +129,7 @@ type CollectQuery struct { } type CollectHost struct { - ProcessGroups []ProcessGroups `json:"processGroups,omitempty" clicky:"title=Process groups" telltale:"kind=process_groups"` + Processes []Resources `json:"processes,omitempty" clicky:"title=Processes" telltale:"kind=process"` } type CollectPrometheus struct { @@ -116,6 +144,12 @@ type CollectSQLServer struct { SystemHealth []SystemHealth `json:"systemHealth,omitempty" clicky:"title=system_health,desc=Attach read-only to the system_health session" telltale:"kind=sql_xevent"` PlanCache []PlanCache `json:"planCache,omitempty" clicky:"title=Plan cache" telltale:"kind=sql_plancache"` DeadlockReport []DeadlockReport `json:"deadlockReport,omitempty" clicky:"title=Deadlock report" telltale:"kind=sql_deadlock"` + Top []SQLServerTop `json:"top,omitempty" clicky:"title=Running requests,desc=Samples sys.dm_exec_requests top style" telltale:"kind=sql_top"` +} + +// CollectPostgres reads a Postgres server over a connection. +type CollectPostgres struct { + Top []PostgresTop `json:"top,omitempty" clicky:"title=Running queries,desc=Samples pg_stat_activity top style" telltale:"kind=sql_top"` } // CollectTelltale reads telltale's own process. diff --git a/pkg/definition/start_spec_test.go b/pkg/definition/start_spec_test.go index 49198d1..9031e3d 100644 --- a/pkg/definition/start_spec_test.go +++ b/pkg/definition/start_spec_test.go @@ -16,7 +16,7 @@ import ( "github.com/flanksource/telltale/pkg/definition" "github.com/flanksource/telltale/pkg/traces" "github.com/flanksource/telltale/pkg/tracing/goprof" - "github.com/flanksource/telltale/pkg/tracing/process" + "github.com/flanksource/telltale/pkg/tracing/processcapture" "github.com/flanksource/telltale/pkg/tracing/stacks" "github.com/flanksource/telltale/pkg/tracing/tracingtest" ) @@ -63,7 +63,7 @@ var _ = Describe("the built-in definitions", func() { It("each have the summary, description and examples their command's help shows", func() { defs, err := definition.Builtins() Expect(err).ToNot(HaveOccurred()) - Expect(defs).To(HaveExactElements(HaveField("Name", "gops"), HaveField("Name", "ps"))) + Expect(defs).To(HaveExactElements(HaveField("Name", "clicky"), HaveField("Name", "gops"), HaveField("Name", "ps"))) for _, def := range defs { Expect(def.Summary()).ToNot(Equal(def.Spec.Description), def.Name) Expect(def.Spec.Description).ToNot(BeEmpty(), def.Name) @@ -79,7 +79,7 @@ var _ = Describe("the built-in definitions", func() { result, err := run.Wait() Expect(err).ToNot(HaveOccurred()) Expect(result.PID).To(Equal(target.Process.Pid)) - Expect(result.Process.Tree.Kind).To(Equal(process.TreeKind)) + Expect(result.Process.Tree.Kind).To(Equal(processcapture.Kind)) Expect(result.CheckpointErrors).To(BeEmpty()) }) diff --git a/pkg/definition/store_spec_test.go b/pkg/definition/store_spec_test.go index a410971..56cc5c7 100644 --- a/pkg/definition/store_spec_test.go +++ b/pkg/definition/store_spec_test.go @@ -48,6 +48,7 @@ var _ = Describe("the definition store", func() { HaveField("Definition.Spec.Collect.Prometheus.Scrape", HaveExactElements(HaveField("Endpoints", ConsistOf("http://cwd:8080/metrics"))))), And(HaveField("Name", "ps"), HaveField("Layer", datadir.ConfigGlobal)), And(HaveField("Name", "gops"), HaveField("Layer", definition.LayerBuiltin)), + And(HaveField("Name", "clicky"), HaveField("Layer", definition.LayerBuiltin)), )) }) diff --git a/pkg/definition/testdata/all-kinds.yaml b/pkg/definition/testdata/all-kinds.yaml index 8e876df..edc3c7f 100644 --- a/pkg/definition/testdata/all-kinds.yaml +++ b/pkg/definition/testdata/all-kinds.yaml @@ -34,6 +34,41 @@ spec: pprof: - profile: mutex jvm: + http: + - agent: {selector: {service.name: example}} + features: [http] + httpTop: + - agent: {selector: {service.name: example}} + direction: both + sql: + - agent: {selector: {service.name: example}} + sqlTop: + - agent: {selector: {service.name: example}} + poll: 5s + jms: + - agent: {selector: {service.name: example}} + direction: both + jmsTop: + - agent: {selector: {service.name: example}} + poll: 1s + destinations: [orders] + spans: + - agent: {selector: {service.name: example}} + notifications: + - agent: {selector: {service.name: example}} + mbean: 'java.lang:type=GarbageCollector,*' + expressions: + - agent: {selector: {service.name: example}} + expression: '@java.lang.Runtime@getRuntime().freeMemory()' + intervalMs: 1000 + methods: + - agent: {selector: {service.name: example}} + targets: [{class: example.Worker, method: compute}] + redefine: + - agent: {selector: {service.name: example}} + class: example.Worker + method: compute + source: 'public int compute() { return 7; }' flightRecorder: - settings: profile asyncProfiler: @@ -61,6 +96,22 @@ spec: - minLevel: warn spans: - {} + clicky: + exec: + - executable: git + entity: + - entity: widget + errorsOnly: true + task: + - taskName: "deploy *" + progressIntervalMs: 5000 + http: + - features: [http] + logs: + - minLevel: warn + pprof: + - source: pprof + profile: goroutine process: tree: facets: true @@ -69,7 +120,7 @@ spec: - {} resources: - interval: 1s - facets: [ports] + facets: [sockets] logs: kubernetes: - namespace: checkout @@ -125,8 +176,8 @@ spec: params: {namespace: checkout} every: 30s host: - processGroups: - - groups: [claude] + processes: + - facets: [files, sockets, connections] prometheus: scrape: - endpoints: [http://localhost:8080/metrics] @@ -142,6 +193,13 @@ spec: - poll: 30s deadlockReport: - {} + top: + - minElapsed: 500ms + top: 20 + postgres: + top: + - poll: 2s + idle: true telltale: http: - features: [http] diff --git a/pkg/definition/walk.go b/pkg/definition/walk.go index 13fd4ef..6df75d0 100644 --- a/pkg/definition/walk.go +++ b/pkg/definition/walk.go @@ -29,6 +29,9 @@ type Located struct { Seam target.Seam // Backend fixes a logs list's cloud backend; Provider a query list's. Backend, Provider string + // Preset is a list of a preset group, which an ad-hoc capture another + // list also takes does not land in. + Preset bool // Title and Description are the list's, as its form shows them. Title, Description string // Name is the entry's name, or its default once Resolve filled it. @@ -71,6 +74,7 @@ func (l Located) DefaultName(definition string) string { // listTag is a typed list's telltale tag, or a group's (seam alone). type listTag struct { kind, seam, backend, provider string + preset bool } func parseTag(field reflect.StructField) listTag { @@ -86,6 +90,8 @@ func parseTag(field reflect.StructField) listTag { tag.backend = value case "provider": tag.provider = value + case "preset": + tag.preset = true } } return tag @@ -147,7 +153,7 @@ func walkGroup(section, group string, groupTag listTag, value reflect.Value) []L } found = append(found, Located{ Section: section, Group: group, Type: jsonName(field), Index: index, Count: list.Len(), - Kind: tag.kind, Seam: target.Seam(seam), Backend: tag.backend, Provider: tag.provider, + Kind: tag.kind, Seam: target.Seam(seam), Backend: tag.backend, Provider: tag.provider, Preset: groupTag.preset, Title: clickyTag(field, "title"), Description: clickyTag(field, "desc"), Name: entry.EntryName(), Entry: entry, }) diff --git a/pkg/deploy/attach.go b/pkg/deploy/attach.go new file mode 100644 index 0000000..53bd295 --- /dev/null +++ b/pkg/deploy/attach.go @@ -0,0 +1,134 @@ +package deploy + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/flanksource/commons-db/connection" + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/models" + + "github.com/flanksource/telltale/pkg/agentinstall" +) + +// KindAttach copies the agent into the container and attaches it to a +// running JVM with kubectl exec, starting its HTTP listener. +const KindAttach = "attach" + +// AttachSpec is an attach javaContainer. url is a template rendered per +// target to the listener's base URL, e.g. +// portforward://{{.Pod.Name}}.{{.Pod.Namespace}}:{{.Port "telltale"}}?kind=pod. +type AttachSpec struct { + Kind string `json:"kind"` + URL string `json:"url"` + PID int `json:"pid"` + Directory string `json:"directory"` + Bind string `json:"bind"` + Port int `json:"port"` + Service string `json:"service"` + JavaHome string `json:"javaHome,omitempty"` + ExtensionsDir string `json:"extensionsDir,omitempty"` + AllowMutation bool `json:"allowMutation,omitempty"` +} + +func init() { + RegisterJavaContainer(KindAttach, func(raw json.RawMessage) (JavaContainer, error) { + var spec AttachSpec + if err := decodeStrict(raw, &spec); err != nil { + return nil, err + } + return newAttach(spec) + }) +} + +type attach struct{ spec AttachSpec } + +func newAttach(spec AttachSpec) (*attach, error) { + switch { + case spec.URL == "": + return nil, errors.New("url (the listener URL template) is required") + case spec.PID < 0: + return nil, errors.New("pid of the JVM in the container must be non-negative; omit it to discover the JVM") + case !strings.HasPrefix(spec.Directory, "/"): + return nil, fmt.Errorf("directory %q must be absolute", spec.Directory) + case spec.Bind == "": + return nil, errors.New("bind (the listener address in the container) is required") + case spec.Port <= 0 || spec.Port > 65535: + return nil, fmt.Errorf("port %d must be a fixed listener port the url reaches", spec.Port) + case spec.Service == "": + return nil, errors.New("service is required") + } + return &attach{spec: spec}, nil +} + +// Client reaches the listener with plain HTTP; the token is the auth. +func (a *attach) Client(dbcontext.Context, string) (*http.Client, error) { + return listenerClient(), nil +} + +// listenerClient reaches a listener directly, over a port-forward, or through +// the apiserver proxy, whose credentials the connection transport adds. +func listenerClient() *http.Client { + return &http.Client{Transport: connection.HTTPConnection{}.Transport(), Timeout: time.Minute} +} + +// settings are what the attached agent is configured with beside its +// listener's auth; they count in the digest, as the agent cannot change +// them without a JVM restart. +func (a *attach) settings() map[string]string { + return map[string]string{"bind": a.spec.Bind, "port": strconv.Itoa(a.spec.Port), "pid": strconv.Itoa(a.spec.PID), "service": a.spec.Service, + "allowMutation": strconv.FormatBool(a.spec.AllowMutation), "extensionsDir": a.spec.ExtensionsDir, "directory": a.spec.Directory} +} + +func (a *attach) Deploy(ctx dbcontext.Context, target Target, provision Provision) TargetStatus { + endpoint, err := RenderURL(a.spec.URL, target) + if err != nil { + return failed("", err) + } + resolved, err := ctx.ResolveConnectionURL(endpoint, models.ConnectionTypeHTTP, target.Pod.Namespace) + if err != nil { + return failed(endpoint, fmt.Errorf("resolve %s: %w", endpoint, err)) + } + config, err := adapterConfig(target, provision, a.settings()) + if err != nil { + return failed(endpoint, err) + } + client := listenerClient() + found, health, _ := inspect(ctx, client, target, resolved, config["configDigest"]) + switch found { + case current: + if !provision.Upgrade { + return TargetStatus{State: StateDeployed, AgentAPIVersion: health.APIVersion, Endpoint: endpoint} + } + case outdated: + if !provision.Upgrade { + return outdatedStatus(endpoint, health) + } + case stale: + if !provision.Upgrade { + return failed(endpoint, errors.New("an agent attached with another provision answers; use --upgrade to replace its runtime")) + } + } + if err := a.install(ctx, target, config, provision); err != nil { + return failed(endpoint, err) + } + return confirm(ctx, client, target, endpoint, func() (string, error) { + return ctx.ResolveConnectionURL(endpoint, models.ConnectionTypeHTTP, target.Pod.Namespace) + }, config["configDigest"]) +} + +func (a *attach) install(ctx dbcontext.Context, target Target, config map[string]string, provision Provision) error { + listener := agentinstall.HTTPListener{Bind: a.spec.Bind, Port: a.spec.Port, Issuer: config["issuer"], Audience: config["audience"], JWKS: json.RawMessage(config["jwks"]), JWKSURL: config["jwksUrl"], ConfigDigest: config["configDigest"]} + destination := agentinstall.Destination{Kind: "kubernetes", Namespace: target.Pod.Namespace, Pod: target.Pod.Name, Container: target.Container.Name} + installer := agentinstall.Installer{Runtime: agentinstall.Java{}, Transport: provision.Exec(target)} + _, err := installer.Install(ctx, agentinstall.Options{Destination: destination, PID: a.spec.PID, Directory: a.spec.Directory, Service: a.spec.Service, AllowMutation: a.spec.AllowMutation, ExtensionsDir: a.spec.ExtensionsDir, JavaHome: a.spec.JavaHome, HTTP: &listener, Upgrade: provision.Upgrade}) + if err != nil { + return fmt.Errorf("attach agent to pid %d in %s: %w", a.spec.PID, target.ID, err) + } + return nil +} diff --git a/pkg/deploy/attach_spec_test.go b/pkg/deploy/attach_spec_test.go new file mode 100644 index 0000000..9480750 --- /dev/null +++ b/pkg/deploy/attach_spec_test.go @@ -0,0 +1,199 @@ +package deploy_test + +import ( + "archive/tar" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "path" + "strconv" + "sync" + + dbcontext "github.com/flanksource/commons-db/context" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/agentinstall" + "github.com/flanksource/telltale/pkg/deploy" +) + +// listenerStub is an attached agent's /health: absent until the fake +// container attaches an agent, then answering with its configured digest. +type listenerStub struct { + mu sync.Mutex + apiVersion int + digest string +} + +func (l *listenerStub) ServeHTTP(w http.ResponseWriter, r *http.Request) { + l.mu.Lock() + defer l.mu.Unlock() + if r.URL.Path != "/health" || l.digest == "" { + http.NotFound(w, r) + return + } + status, code := "ok", http.StatusOK + if l.apiVersion != agentclient.AgentAPIVersion { + status, code = "outdated", http.StatusServiceUnavailable + } + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(map[string]any{"status": status, "apiVersion": l.apiVersion, "configDigest": l.digest}) +} + +// container plays the JVM container kubectl exec reaches: it keeps what is +// written, answers the checksum of it, and attaching starts the listener +// with the written agent configuration. +type container struct { + mu sync.Mutex + listener *listenerStub + files map[string][]byte + commands []string + config map[string]any + upgrade bool +} + +func (c *container) Run(_ context.Context, args []string, input io.Reader) ([]byte, error) { + c.mu.Lock() + defer c.mu.Unlock() + // env -u NAME… runs the rest of the command without those variables. + if args[0] == "env" { + args = args[1:] + for len(args) > 1 && args[0] == "-u" { + args = args[2:] + } + } + // The helper runs as /bin/java: a command is named by its base. + c.commands = append(c.commands, path.Base(args[0])) + switch path.Base(args[0]) { + case "sh": + return []byte("__TELLTALE_PRESENT__=true\nJAVA_VERSION=\"1.8.0_512\"\n__TELLTALE_TOOLS__=true\n"), nil + case "cat": + return []byte("1 (java) S 0 1 1 0 -1 4194560 950 0 0 0 1 2 0 0 20 0 41 0 2318867 5000000\n"), nil + case "uname": + return []byte("Linux\n"), nil + case "readlink": + return []byte("/opt/java/openjdk/bin/java\n"), nil + case "tar": + // tar -x … -f - -C : every file of the archive lands in dir. + dir := args[len(args)-1] + archive := tar.NewReader(input) + for { + header, err := archive.Next() + if err == io.EOF { + return nil, nil + } + if err != nil { + return nil, err + } + data, err := io.ReadAll(archive) + if err != nil { + return nil, err + } + c.files[path.Join(dir, header.Name)] = data + } + case "mv": + c.files[args[3]] = c.files[args[2]] + return nil, nil + case "sha256sum": + sum := sha256.Sum256(c.files[args[1]]) + return []byte(hex.EncodeToString(sum[:]) + " " + args[1] + "\n"), nil + case "java": + upgrade, err := strconv.ParseBool(args[len(args)-1]) + Expect(err).NotTo(HaveOccurred()) + c.upgrade = upgrade + Expect(json.Unmarshal(c.files[args[len(args)-2]], &c.config)).To(Succeed()) + listener := c.config["http"].(map[string]any) + c.listener.mu.Lock() + c.listener.digest = listener["configDigest"].(string) + c.listener.mu.Unlock() + return nil, nil + } + return nil, nil +} + +func listenerPod(name string, stub *listenerStub) *corev1.Pod { + server := httptest.NewServer(stub) + DeferCleanup(server.Close) + _, port, err := net.SplitHostPort(server.Listener.Addr().String()) + Expect(err).ToNot(HaveOccurred()) + number, err := strconv.Atoi(port) + Expect(err).ToNot(HaveOccurred()) + return pod(name, map[string]string{"app": "orders"}, corev1.PodRunning, corev1.ContainerPort{Name: "telltale", ContainerPort: int32(number)}) +} + +func attachDeployer() deploy.Deployer { + deployer := tomcatDeployer() + spec, err := deploy.NewJavaContainerSpec(map[string]any{"kind": "attach", "url": `http://127.0.0.1:{{.Port "telltale"}}`, "pid": 1, "directory": "/opt/telltale", + "bind": "0.0.0.0", "port": 7070, "service": "orders", "allowMutation": true}) + Expect(err).ToNot(HaveOccurred()) + deployer.Spec.JavaContainer = spec + return deployer +} + +var _ = Describe("attach java containers", func() { + It("reattaches a healthy target when an upgrade is explicitly requested", func() { + listener := &listenerStub{apiVersion: agentclient.AgentAPIVersion} + jvmContainer := &container{listener: listener, files: map[string][]byte{}} + options := deploy.Options{Kubernetes: fake.NewClientset(listenerPod("orders-0", listener)), Issuer: issuer(), Exec: func(deploy.Target) agentinstall.Transport { return jvmContainer }} + _, err := deploy.Deploy(dbcontext.New(), attachDeployer(), options) + Expect(err).NotTo(HaveOccurred()) + Expect(jvmContainer.upgrade).To(BeFalse()) + before := len(jvmContainer.commands) + options.Upgrade = true + status, err := deploy.Deploy(dbcontext.New(), attachDeployer(), options) + Expect(err).NotTo(HaveOccurred()) + Expect(status.Targets).To(ConsistOf(HaveField("State", deploy.StateDeployed))) + Expect(len(jvmContainer.commands)).To(BeNumerically(">", before)) + Expect(jvmContainer.upgrade).To(BeTrue()) + }) + + It("attaches the agent with a hub-less listener once and leaves a current one alone", func() { + listener := &listenerStub{apiVersion: agentclient.AgentAPIVersion} + jvmContainer := &container{listener: listener, files: map[string][]byte{}} + signer := issuer() + options := deploy.Options{Kubernetes: fake.NewClientset(listenerPod("orders-0", listener)), Issuer: signer, Exec: func(deploy.Target) agentinstall.Transport { return jvmContainer }} + status, err := deploy.Deploy(dbcontext.New(), attachDeployer(), options) + Expect(err).ToNot(HaveOccurred()) + Expect(status.Targets).To(ConsistOf(And(HaveField("State", deploy.StateDeployed), HaveField("AgentAPIVersion", agentclient.AgentAPIVersion)))) + Expect(jvmContainer.commands).To(ContainElement("java")) + Expect(jvmContainer.config).ToNot(HaveKey("url"), "a hub-less agent dials no hub") + Expect(jvmContainer.config["http"]).To(And(HaveKeyWithValue("bind", "0.0.0.0"), HaveKeyWithValue("port", 7070.0), HaveKeyWithValue("issuer", issuerRef), + HaveKeyWithValue("audience", deploy.TargetID("example", namespace, "orders-0")), HaveKey("jwks"))) + + commands := len(jvmContainer.commands) + again, err := deploy.Deploy(dbcontext.New(), attachDeployer(), options) + Expect(err).ToNot(HaveOccurred()) + Expect(again).To(Equal(status)) + Expect(jvmContainer.commands).To(HaveLen(commands), "a current agent is not attached again") + }) + + It("selects the targets of a dry run without an issuer and without touching their containers", func() { + listener := &listenerStub{apiVersion: agentclient.AgentAPIVersion} + jvmContainer := &container{listener: listener, files: map[string][]byte{}} + status, err := deploy.Deploy(dbcontext.New(), attachDeployer(), deploy.Options{DryRun: true, + Kubernetes: fake.NewClientset(listenerPod("orders-1", listener), listenerPod("orders-0", listener)), Exec: func(deploy.Target) agentinstall.Transport { return jvmContainer }}) + Expect(err).ToNot(HaveOccurred()) + Expect(status.Targets).To(Equal([]deploy.TargetStatus{ + {Target: deploy.TargetID("example", namespace, "orders-0"), Namespace: namespace, Pod: "orders-0", Container: jvm, State: deploy.StateSelected}, + {Target: deploy.TargetID("example", namespace, "orders-1"), Namespace: namespace, Pod: "orders-1", Container: jvm, State: deploy.StateSelected}, + })) + Expect(jvmContainer.commands).To(BeEmpty()) + }) + + It("reports outdated, without attaching, when the JVM runs another agent API", func() { + listener := &listenerStub{apiVersion: agentclient.AgentAPIVersion + 1, digest: "any"} + jvmContainer := &container{listener: listener, files: map[string][]byte{}} + status, err := deploy.Deploy(dbcontext.New(), attachDeployer(), deploy.Options{Kubernetes: fake.NewClientset(listenerPod("orders-0", listener)), Issuer: issuer(), Exec: func(deploy.Target) agentinstall.Transport { return jvmContainer }}) + Expect(err).To(MatchError(ContainSubstring("outdated"))) + Expect(status.Targets).To(ConsistOf(HaveField("State", deploy.StateOutdated))) + Expect(jvmContainer.commands).To(BeEmpty()) + }) +}) diff --git a/pkg/deploy/auth/auth_suite_test.go b/pkg/deploy/auth/auth_suite_test.go new file mode 100644 index 0000000..d75e674 --- /dev/null +++ b/pkg/deploy/auth/auth_suite_test.go @@ -0,0 +1,13 @@ +package auth_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestAuth(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Deployer auth") +} diff --git a/pkg/deploy/auth/issuer.go b/pkg/deploy/auth/issuer.go new file mode 100644 index 0000000..cc75ba5 --- /dev/null +++ b/pkg/deploy/auth/issuer.go @@ -0,0 +1,241 @@ +// Package auth mints the short-lived JWTs an execution presents to a deployed +// agent. Tokens are RS256: the agent side must run on Java 8, which has RSA +// signatures but no Ed25519. The agent verifies them against the issuer's +// JWKS, provisioned inline at deploy time and, when the issuer runtime serves +// it, refreshed from a jwksUrl on an unknown kid. +package auth + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "math/big" + "net/http" + "slices" + "strings" + "sync" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +// Scope is an access level an agent grants per operation and capture kind. +// The agent's catalog names the scope of every operation and probe; the agent +// is the one table, a minter only asks for what the catalog says. +type Scope string + +const ( + ScopeRead Scope = "jvm:read" + ScopeTrace Scope = "jvm:trace" + ScopeInstrument Scope = "jvm:instrument" + ScopeEval Scope = "jvm:eval" + ScopeWrite Scope = "jvm:write" +) + +// Scopes are every scope an agent knows. +var Scopes = []Scope{ScopeRead, ScopeTrace, ScopeInstrument, ScopeEval, ScopeWrite} + +// MaxTTL bounds a token's lifetime. +const MaxTTL = 5 * time.Minute + +// minKeyBits is the smallest RSA key an issuer signs with. +const minKeyBits = 2048 + +// Claims are what one execution request is allowed to do. +type Claims struct { + Audience string + Subject string + Scopes []Scope +} + +func (c Claims) validate(ttl time.Duration) error { + switch { + case c.Audience == "": + return errors.New("mint token: audience (the target id) is required") + case c.Subject == "": + return errors.New("mint token: subject (the principal) is required") + case len(c.Scopes) == 0: + return errors.New("mint token: at least one scope is required") + case ttl <= 0 || ttl > MaxTTL: + return fmt.Errorf("mint token: ttl %s must be positive and at most %s", ttl, MaxTTL) + } + for _, scope := range c.Scopes { + if !slices.Contains(Scopes, scope) { + return fmt.Errorf("mint token: unknown scope %q", scope) + } + } + return nil +} + +// JWK is an RSA public key in JWKS form. +type JWK struct { + KeyType string `json:"kty"` + KeyID string `json:"kid"` + Use string `json:"use"` + Algorithm string `json:"alg"` + N string `json:"n"` + E string `json:"e"` +} + +// JWKS is the public half of an issuer's keys, the signing key first. +type JWKS struct { + Keys []JWK `json:"keys"` +} + +type signingKey struct { + private *rsa.PrivateKey + public JWK +} + +// Issuer signs tokens with its current key and publishes every key it has +// signed with, so tokens minted before a rotation stay verifiable. +type Issuer struct { + name string + mu sync.RWMutex + keys []signingKey +} + +// NewIssuer is an issuer named name (the tokens' iss) signing with key. +func NewIssuer(name string, key *rsa.PrivateKey) (*Issuer, error) { + if name == "" { + return nil, errors.New("issuer name is required") + } + signing, err := newSigningKey(key) + if err != nil { + return nil, err + } + return &Issuer{name: name, keys: []signingKey{signing}}, nil +} + +func newSigningKey(key *rsa.PrivateKey) (signingKey, error) { + if key == nil { + return signingKey{}, errors.New("issuer signing key is required") + } + if key.N.BitLen() < minKeyBits { + return signingKey{}, fmt.Errorf("issuer signing key has %d bits, want at least %d", key.N.BitLen(), minKeyBits) + } + n := base64.RawURLEncoding.EncodeToString(key.N.Bytes()) + e := base64.RawURLEncoding.EncodeToString(big.NewInt(int64(key.E)).Bytes()) + return signingKey{private: key, public: JWK{KeyType: "RSA", KeyID: thumbprint(n, e), Use: "sig", Algorithm: "RS256", N: n, E: e}}, nil +} + +// thumbprint is the RFC 7638 JWK thumbprint, the key's id. +func thumbprint(n, e string) string { + sum := sha256.Sum256([]byte(`{"e":"` + e + `","kty":"RSA","n":"` + n + `"}`)) + return base64.RawURLEncoding.EncodeToString(sum[:]) +} + +// Name is the issuer's iss claim. +func (i *Issuer) Name() string { return i.name } + +// Mint signs claims valid for ttl. +func (i *Issuer) Mint(claims Claims, ttl time.Duration) (string, error) { + if err := claims.validate(ttl); err != nil { + return "", err + } + now := time.Now() + scopes := make([]string, len(claims.Scopes)) + for index, scope := range claims.Scopes { + scopes[index] = string(scope) + } + i.mu.RLock() + current := i.keys[0] + i.mu.RUnlock() + token := jwt.NewWithClaims(jwt.SigningMethodRS256, jwt.MapClaims{ + "iss": i.name, + "aud": claims.Audience, + "sub": claims.Subject, + "iat": now.Unix(), + "exp": now.Add(ttl).Unix(), + "scope": strings.Join(scopes, " "), + }) + token.Header["kid"] = current.public.KeyID + signed, err := token.SignedString(current.private) + if err != nil { + return "", fmt.Errorf("mint token: %w", err) + } + return signed, nil +} + +// Rotate makes key the signing key; the previous keys stay published. +func (i *Issuer) Rotate(key *rsa.PrivateKey) error { + signing, err := newSigningKey(key) + if err != nil { + return err + } + i.mu.Lock() + defer i.mu.Unlock() + i.keys = append([]signingKey{signing}, i.keys...) + return nil +} + +// JWKS is every published public key, the signing key first. +func (i *Issuer) JWKS() JWKS { + i.mu.RLock() + defer i.mu.RUnlock() + keys := make([]JWK, len(i.keys)) + for index, key := range i.keys { + keys[index] = key.public + } + return JWKS{Keys: keys} +} + +// ServeHTTP serves the JWKS, the endpoint an agent's jwksUrl names. +func (i *Issuer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.Header().Set("Allow", http.MethodGet) + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "max-age=60") + _ = json.NewEncoder(w).Encode(i.JWKS()) +} + +// GenerateSigningKey is a new 3072-bit RSA key as PKCS#8 PEM, the form +// ParsePrivateKey reads. +func GenerateSigningKey() ([]byte, error) { + key, err := rsa.GenerateKey(rand.Reader, 3072) + if err != nil { + return nil, fmt.Errorf("signing key: %w", err) + } + der, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return nil, fmt.Errorf("signing key: %w", err) + } + return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), nil +} + +// ParsePrivateKey reads a PKCS#8 or PKCS#1 PEM RSA private key. +func ParsePrivateKey(data []byte) (*rsa.PrivateKey, error) { + block, _ := pem.Decode(data) + if block == nil { + return nil, errors.New("signing key: no PEM block") + } + switch block.Type { + case "RSA PRIVATE KEY": + key, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("signing key: %w", err) + } + return key, nil + case "PRIVATE KEY": + parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("signing key: %w", err) + } + key, ok := parsed.(*rsa.PrivateKey) + if !ok { + return nil, fmt.Errorf("signing key: want an RSA key for RS256, got %T", parsed) + } + return key, nil + default: + return nil, fmt.Errorf("signing key: unsupported PEM block %q", block.Type) + } +} diff --git a/pkg/deploy/auth/issuer_spec_test.go b/pkg/deploy/auth/issuer_spec_test.go new file mode 100644 index 0000000..c6cc35a --- /dev/null +++ b/pkg/deploy/auth/issuer_spec_test.go @@ -0,0 +1,139 @@ +package auth_test + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "math/big" + "net/http/httptest" + "time" + + "github.com/golang-jwt/jwt/v5" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/telltale/pkg/deploy/auth" +) + +const ( + issuerName = "https://telltale.example/issuer" + targetID = "deployer:example/orders/orders-0" + principal = "operator@example" +) + +func rsaKey() *rsa.PrivateKey { + key, err := rsa.GenerateKey(rand.Reader, 2048) + Expect(err).ToNot(HaveOccurred()) + return key +} + +// verify parses token with the public keys of jwks, keyed by kid, as an +// agent does: RS256 only, with iss and aud required. +func verify(token string, jwks auth.JWKS) (jwt.MapClaims, string) { + claims := jwt.MapClaims{} + parsed, err := jwt.ParseWithClaims(token, claims, func(t *jwt.Token) (any, error) { + kid, _ := t.Header["kid"].(string) + for _, key := range jwks.Keys { + if key.KeyID == kid { + n, err := base64.RawURLEncoding.DecodeString(key.N) + Expect(err).ToNot(HaveOccurred()) + e, err := base64.RawURLEncoding.DecodeString(key.E) + Expect(err).ToNot(HaveOccurred()) + return &rsa.PublicKey{N: new(big.Int).SetBytes(n), E: int(new(big.Int).SetBytes(e).Int64())}, nil + } + } + return nil, jwt.ErrTokenUnverifiable + }, jwt.WithValidMethods([]string{"RS256"}), jwt.WithIssuer(issuerName), jwt.WithAudience(targetID), jwt.WithExpirationRequired()) + Expect(err).ToNot(HaveOccurred()) + return claims, parsed.Header["kid"].(string) +} + +var _ = Describe("a generated signing key", func() { + It("is a PKCS#8 PEM RSA key an issuer signs with", func() { + data, err := auth.GenerateSigningKey() + Expect(err).ToNot(HaveOccurred()) + block, _ := pem.Decode(data) + Expect(block).NotTo(BeNil()) + Expect(block.Type).To(Equal("PRIVATE KEY")) + key, err := auth.ParsePrivateKey(data) + Expect(err).ToNot(HaveOccurred()) + Expect(key.N.BitLen()).To(Equal(3072)) + _, err = auth.NewIssuer(issuerName, key) + Expect(err).ToNot(HaveOccurred()) + }) +}) + +var _ = Describe("Issuer", func() { + It("mints an RS256 token an agent verifies with the published JWKS", func() { + issuer, err := auth.NewIssuer(issuerName, rsaKey()) + Expect(err).ToNot(HaveOccurred()) + token, err := issuer.Mint(auth.Claims{Audience: targetID, Subject: principal, Scopes: []auth.Scope{auth.ScopeRead, auth.ScopeTrace}}, time.Minute) + Expect(err).ToNot(HaveOccurred()) + claims, kid := verify(token, issuer.JWKS()) + Expect(kid).To(Equal(issuer.JWKS().Keys[0].KeyID)) + Expect(claims).To(HaveKeyWithValue("sub", principal)) + Expect(claims).To(HaveKeyWithValue("scope", "jvm:read jvm:trace")) + exp, err := claims.GetExpirationTime() + Expect(err).ToNot(HaveOccurred()) + Expect(exp.Time).To(BeTemporally("~", time.Now().Add(time.Minute), 2*time.Second)) + }) + + DescribeTable("refuses a token it should not mint", + func(claims auth.Claims, ttl time.Duration, message string) { + issuer, err := auth.NewIssuer(issuerName, rsaKey()) + Expect(err).ToNot(HaveOccurred()) + _, err = issuer.Mint(claims, ttl) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("no audience", auth.Claims{Subject: principal, Scopes: []auth.Scope{auth.ScopeRead}}, time.Minute, "audience"), + Entry("no subject", auth.Claims{Audience: targetID, Scopes: []auth.Scope{auth.ScopeRead}}, time.Minute, "subject"), + Entry("no scope", auth.Claims{Audience: targetID, Subject: principal}, time.Minute, "scope"), + Entry("an unknown scope", auth.Claims{Audience: targetID, Subject: principal, Scopes: []auth.Scope{"jvm:root"}}, time.Minute, `unknown scope "jvm:root"`), + Entry("a long-lived token", auth.Claims{Audience: targetID, Subject: principal, Scopes: []auth.Scope{auth.ScopeRead}}, auth.MaxTTL+time.Second, "ttl"), + ) + + It("keeps the retired key in the JWKS after a rotation and signs with the new one", func() { + issuer, err := auth.NewIssuer(issuerName, rsaKey()) + Expect(err).ToNot(HaveOccurred()) + before, err := issuer.Mint(auth.Claims{Audience: targetID, Subject: principal, Scopes: []auth.Scope{auth.ScopeRead}}, time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(issuer.Rotate(rsaKey())).To(Succeed()) + after, err := issuer.Mint(auth.Claims{Audience: targetID, Subject: principal, Scopes: []auth.Scope{auth.ScopeRead}}, time.Minute) + Expect(err).ToNot(HaveOccurred()) + jwks := issuer.JWKS() + Expect(jwks.Keys).To(HaveLen(2)) + _, oldKid := verify(before, jwks) + _, newKid := verify(after, jwks) + Expect(newKid).ToNot(Equal(oldKid)) + Expect(jwks.Keys[0].KeyID).To(Equal(newKid), "the signing key is listed first") + }) + + It("serves its JWKS for agents configured with a jwksUrl", func() { + issuer, err := auth.NewIssuer(issuerName, rsaKey()) + Expect(err).ToNot(HaveOccurred()) + recorder := httptest.NewRecorder() + issuer.ServeHTTP(recorder, httptest.NewRequest("GET", "/.well-known/jwks.json", nil)) + Expect(recorder.Code).To(Equal(200)) + Expect(recorder.Header().Get("Content-Type")).To(Equal("application/json")) + var served auth.JWKS + Expect(json.Unmarshal(recorder.Body.Bytes(), &served)).To(Succeed()) + Expect(served).To(Equal(issuer.JWKS())) + Expect(served.Keys[0]).To(And(HaveField("KeyType", "RSA"), HaveField("Algorithm", "RS256"), HaveField("Use", "sig"))) + }) + + It("loads a PKCS#8 or PKCS#1 PEM signing key and refuses anything else", func() { + key := rsaKey() + pkcs8, err := x509.MarshalPKCS8PrivateKey(key) + Expect(err).ToNot(HaveOccurred()) + for _, block := range []*pem.Block{{Type: "PRIVATE KEY", Bytes: pkcs8}, {Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}} { + parsed, err := auth.ParsePrivateKey(pem.EncodeToMemory(block)) + Expect(err).ToNot(HaveOccurred()) + Expect(parsed.Equal(key)).To(BeTrue()) + } + _, err = auth.ParsePrivateKey([]byte("not a key")) + Expect(err).To(MatchError(ContainSubstring("PEM"))) + }) +}) diff --git a/pkg/deploy/auth/tokens.go b/pkg/deploy/auth/tokens.go new file mode 100644 index 0000000..afd2ac8 --- /dev/null +++ b/pkg/deploy/auth/tokens.go @@ -0,0 +1,23 @@ +package auth + +import ( + "context" + "errors" + "time" +) + +// Tokens mints one token per request for a fixed principal, the +// agentclient.TokenSource a direct transport asks before each call. +type Tokens struct { + Issuer *Issuer + Subject string + TTL time.Duration +} + +// Token is a token for audience carrying scope. +func (t Tokens) Token(_ context.Context, audience, scope string) (string, error) { + if t.Issuer == nil { + return "", errors.New("token source has no issuer") + } + return t.Issuer.Mint(Claims{Audience: audience, Subject: t.Subject, Scopes: []Scope{Scope(scope)}}, t.TTL) +} diff --git a/pkg/deploy/cluster.go b/pkg/deploy/cluster.go new file mode 100644 index 0000000..19982ee --- /dev/null +++ b/pkg/deploy/cluster.go @@ -0,0 +1,68 @@ +package deploy + +import ( + dbcontext "github.com/flanksource/commons-db/context" + dbkubernetes "github.com/flanksource/commons-db/kubernetes" + + "github.com/flanksource/telltale/pkg/agentinstall" +) + +// The annotations a saved deployer keeps its cluster in. They are the +// deploying client's choice, not part of the resource a cluster applies. +const ( + AnnotationKubeconfig = "telltale.flanksource.com/kubeconfig" + AnnotationKubeContext = "telltale.flanksource.com/kubecontext" +) + +// Cluster is the kubeconfig and kube context a deploy reached its pods with; +// both unset is kubectl's and client-go's default. A later execution of a +// saved deployer reconnects through the same cluster. +type Cluster struct { + Kubeconfig string `json:"kubeconfig,omitempty"` + Context string `json:"context,omitempty"` +} + +// ClusterOf is the cluster a saved deployer was deployed to. +func ClusterOf(deployer Deployer) Cluster { + return Cluster{Kubeconfig: deployer.Annotations[AnnotationKubeconfig], Context: deployer.Annotations[AnnotationKubeContext]} +} + +// Annotate records c on deployer, removing what c leaves unset. +func (c Cluster) Annotate(deployer *Deployer) { + set := func(key, value string) { + if value == "" { + delete(deployer.Annotations, key) + return + } + if deployer.Annotations == nil { + deployer.Annotations = map[string]string{} + } + deployer.Annotations[key] = value + } + set(AnnotationKubeconfig, c.Kubeconfig) + set(AnnotationKubeContext, c.Context) + if len(deployer.Annotations) == 0 { + deployer.Annotations = nil + } +} + +// Bind makes ctx's Kubernetes client, which port-forward URLs resolve +// through, this cluster's. The default cluster leaves ctx as it is. +func (c Cluster) Bind(ctx dbcontext.Context) (dbcontext.Context, error) { + if c == (Cluster{}) { + return ctx, nil + } + client, err := ctx.KubeContextClient(dbkubernetes.Options{Kubeconfig: c.Kubeconfig, Kubecontext: c.Context}) + if err != nil { + return ctx, err + } + return ctx.WithLocalKubernetes(client), nil +} + +// Exec runs commands in a target's container with kubectl against c. +func (c Cluster) Exec() ExecFunc { + return func(target Target) agentinstall.Transport { + return agentinstall.CommandTransport{Destination: agentinstall.Destination{Kind: "kubernetes", Kubeconfig: c.Kubeconfig, Context: c.Context, + Namespace: target.Pod.Namespace, Pod: target.Pod.Name, Container: target.Container.Name}} + } +} diff --git a/pkg/deploy/cluster_spec_test.go b/pkg/deploy/cluster_spec_test.go new file mode 100644 index 0000000..58c83df --- /dev/null +++ b/pkg/deploy/cluster_spec_test.go @@ -0,0 +1,48 @@ +package deploy_test + +import ( + dbcontext "github.com/flanksource/commons-db/context" + dbkubernetes "github.com/flanksource/commons-db/kubernetes" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/telltale/pkg/agentinstall" + "github.com/flanksource/telltale/pkg/deploy" +) + +var _ = Describe("the cluster a deployer was deployed to", func() { + uat := deploy.Cluster{Kubeconfig: "/home/example/.kube/uat", Context: "uat"} + + It("is kept on the saved deployer and read back", func() { + deployer := tomcatDeployer() + uat.Annotate(&deployer) + Expect(deploy.ClusterOf(deployer)).To(Equal(uat)) + deploy.Cluster{}.Annotate(&deployer) + Expect(deploy.ClusterOf(deployer)).To(Equal(deploy.Cluster{})) + Expect(deployer.Annotations).To(BeEmpty()) + }) + + It("runs kubectl in a target's container against that cluster", func() { + pod := listenerPod("orders-0", &listenerStub{}) + transport := uat.Exec()(deploy.Target{ID: "deployer/orders/shop/orders-0", Pod: pod, Container: pod.Spec.Containers[0]}) + Expect(transport).To(Equal(agentinstall.CommandTransport{Destination: agentinstall.Destination{Kind: "kubernetes", + Kubeconfig: uat.Kubeconfig, Context: uat.Context, Namespace: pod.Namespace, Pod: pod.Name, Container: pod.Spec.Containers[0].Name}})) + }) + + It("binds a context to that cluster's client, and leaves the default cluster alone", func() { + var asked []dbkubernetes.Options + chosen := &dbkubernetes.Client{} + ctx := dbcontext.New().WithKubeContextClients(func(options dbkubernetes.Options) (*dbkubernetes.Client, error) { + asked = append(asked, options) + return chosen, nil + }) + bound, err := uat.Bind(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(asked).To(Equal([]dbkubernetes.Options{{Kubeconfig: uat.Kubeconfig, Kubecontext: uat.Context}})) + Expect(bound.LocalKubernetes()).To(BeIdenticalTo(chosen)) + + _, err = deploy.Cluster{}.Bind(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(asked).To(HaveLen(1), "the default cluster asks for no client") + }) +}) diff --git a/pkg/deploy/confirm_internal_test.go b/pkg/deploy/confirm_internal_test.go new file mode 100644 index 0000000..dba0d9b --- /dev/null +++ b/pkg/deploy/confirm_internal_test.go @@ -0,0 +1,41 @@ +package deploy + +import ( + "context" + "encoding/json" + "net" + "net/http" + "net/http/httptest" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/telltale/pkg/agentclient" +) + +var _ = Describe("confirming a fresh agent", func() { + It("resolves its endpoint again on each attempt, so a port-forward torn down before the listener came up is reopened", func() { + const digest = "provision-1" + live := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{"status": "ok", "apiVersion": agentclient.AgentAPIVersion, "configDigest": digest}) + })) + DeferCleanup(live.Close) + // A local port nobody listens on, as a port-forward is once its first connection was refused. + closed, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + dead := "http://" + closed.Addr().String() + Expect(closed.Close()).To(Succeed()) + + var resolutions int + resolve := func() (string, error) { + resolutions++ + if resolutions == 1 { + return dead, nil + } + return live.URL, nil + } + status := confirm(context.Background(), live.Client(), Target{ID: "deployer/orders/shop/orders-0"}, "portforward://orders-0.shop:7070?kind=pod", resolve, digest) + Expect(status).To(Equal(TargetStatus{State: StateDeployed, AgentAPIVersion: agentclient.AgentAPIVersion, Endpoint: "portforward://orders-0.shop:7070?kind=pod"})) + Expect(resolutions).To(Equal(2)) + }) +}) diff --git a/pkg/deploy/connect.go b/pkg/deploy/connect.go new file mode 100644 index 0000000..600457f --- /dev/null +++ b/pkg/deploy/connect.go @@ -0,0 +1,102 @@ +package deploy + +import ( + "errors" + "fmt" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/models" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/deploy/auth" +) + +// The resource attributes a connected target carries, which a capture's +// target selector matches. +const ( + AttributeDeployer = "telltale.deployer" + AttributeTarget = "telltale.target.id" + AttributeNamespace = "k8s.namespace.name" + AttributePod = "k8s.pod.name" + AttributeContainer = "k8s.container.name" +) + +// SigningKeySecret is the named secret of the spec.auth.issuer connection +// that holds the RSA signing key, PEM encoded. +const SigningKeySecret = "signingKey" + +// NewIssuer is the issuer spec.auth names: its connection's signingKey +// secret signs, and its reference is the tokens' iss. +func NewIssuer(ctx dbcontext.Context, spec AuthSpec) (*auth.Issuer, error) { + if spec.Issuer == "" { + return nil, errors.New("spec.auth.issuer is required") + } + connection, err := ctx.HydrateConnectionByURL(spec.Issuer) + if err != nil { + return nil, fmt.Errorf("issuer %s: %w", spec.Issuer, err) + } + if connection == nil { + return nil, fmt.Errorf("issuer %s: connection not found", spec.Issuer) + } + pem, err := connection.Secret(SigningKeySecret) + if err != nil { + return nil, fmt.Errorf("issuer %s: %w", spec.Issuer, err) + } + key, err := auth.ParsePrivateKey([]byte(pem)) + if err != nil { + return nil, fmt.Errorf("issuer %s: %w", spec.Issuer, err) + } + return auth.NewIssuer(spec.Issuer, key) +} + +// Connect dials every target of status over direct HTTP and adds it to +// registry, named by its target id and carrying its deployer, namespace, pod +// and container, so a capture selects a deployed target and never a URL. +// Every target must be deployed. +func Connect(ctx dbcontext.Context, deployer Deployer, status Status, registry *agentclient.Registry, tokens agentclient.TokenSource) ([]*agentclient.Agent, error) { + container, err := deployer.Spec.JavaContainer.Container() + if err != nil { + return nil, fmt.Errorf("deployer %s: %w", deployer.Name, err) + } + var agents []*agentclient.Agent + for _, target := range status.Targets { + agent, err := connect(ctx, deployer.Name, container, target, tokens) + if err == nil { + err = registry.Add(agent) + } + if err != nil { + return nil, errors.Join(fmt.Errorf("connect %s: %w", target.Target, err), closeAll(agents)) + } + agents = append(agents, agent) + } + return agents, nil +} + +func connect(ctx dbcontext.Context, deployer string, container JavaContainer, target TargetStatus, tokens agentclient.TokenSource) (*agentclient.Agent, error) { + if target.State != StateDeployed { + return nil, fmt.Errorf("target is %s: %s", target.State, target.Error) + } + client, err := container.Client(ctx, target.Namespace) + if err != nil { + return nil, err + } + url, err := ctx.ResolveConnectionURL(target.Endpoint, models.ConnectionTypeHTTP, target.Namespace) + if err != nil { + return nil, fmt.Errorf("resolve %s: %w", target.Endpoint, err) + } + resource := targetResource(deployer, target) + return agentclient.DialHTTP(ctx, agentclient.HTTPEndpoint{ID: target.Target, URL: url, Client: client, Tokens: tokens, Resource: resource}) +} + +func targetResource(deployer string, target TargetStatus) map[string]string { + return map[string]string{AttributeDeployer: deployer, AttributeTarget: target.Target, AttributeNamespace: target.Namespace, + AttributePod: target.Pod, AttributeContainer: target.Container} +} + +func closeAll(agents []*agentclient.Agent) error { + var errs []error + for _, agent := range agents { + errs = append(errs, agent.Close()) + } + return errors.Join(errs...) +} diff --git a/pkg/deploy/crd.go b/pkg/deploy/crd.go new file mode 100644 index 0000000..647fe37 --- /dev/null +++ b/pkg/deploy/crd.go @@ -0,0 +1,38 @@ +package deploy + +import ( + "fmt" + + "github.com/flanksource/telltale/pkg/definition" +) + +// CRDFile is where `task generate` writes the Deployer CRD, relative to the +// repository root. +const CRDFile = "crds/" + definition.Group + "_" + Plural + ".yaml" + +// Schema is the Deployer's structural openAPIV3Schema. A javaContainer keeps +// the fields of its kind, which the kind's registered factory validates. +func Schema() (map[string]any, error) { + schema, err := definition.SchemaOf(Deployer{}) + if err != nil { + return nil, err + } + spec, _ := schema["properties"].(map[string]any)["spec"].(map[string]any) + properties, _ := spec["properties"].(map[string]any) + container, ok := properties["javaContainer"].(map[string]any) + if !ok { + return nil, fmt.Errorf("deployer schema has no spec.javaContainer") + } + container["x-kubernetes-preserve-unknown-fields"] = true + container["required"] = []any{"kind"} + return schema, nil +} + +// CRD is the Deployer CustomResourceDefinition. +func CRD() ([]byte, error) { + schema, err := Schema() + if err != nil { + return nil, err + } + return definition.CRDOf(Kind, Plural, "pkg/deploy", schema) +} diff --git a/pkg/deploy/crd_spec_test.go b/pkg/deploy/crd_spec_test.go new file mode 100644 index 0000000..a942fe9 --- /dev/null +++ b/pkg/deploy/crd_spec_test.go @@ -0,0 +1,70 @@ +package deploy_test + +import ( + "bytes" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning" + "k8s.io/apiextensions-apiserver/pkg/apiserver/validation" + "sigs.k8s.io/yaml" + + "github.com/flanksource/telltale/pkg/deploy" +) + +var ( + committedCRD = filepath.Join("..", "..", deploy.CRDFile) + example = filepath.Join("..", "..", "examples", "deployers", "orders.yaml") +) + +func crdSchema() *apiextensions.JSONSchemaProps { + data, err := os.ReadFile(committedCRD) + Expect(err).ToNot(HaveOccurred()) + var crd apiextensionsv1.CustomResourceDefinition + Expect(yaml.UnmarshalStrict(data, &crd)).To(Succeed()) + Expect(crd.Spec.Versions).To(HaveLen(1)) + var internal apiextensions.JSONSchemaProps + Expect(apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(crd.Spec.Versions[0].Schema.OpenAPIV3Schema, &internal, nil)).To(Succeed()) + return &internal +} + +var _ = Describe("the Deployer CRD", func() { + It("is the one `task generate` writes from the types", func() { + generated, err := deploy.CRD() + Expect(err).ToNot(HaveOccurred()) + committed, err := os.ReadFile(committedCRD) + Expect(err).ToNot(HaveOccurred()) + Expect(string(committed)).To(Equal(string(generated)), "run `task generate`") + }) + + It("is a structural schema the API server accepts and keeps a javaContainer's own fields", func() { + structural, err := structuralschema.NewStructural(crdSchema()) + Expect(err).ToNot(HaveOccurred()) + Expect(structuralschema.ValidateStructural(nil, structural)).To(BeEmpty()) + data, err := os.ReadFile(example) + Expect(err).ToNot(HaveOccurred()) + for _, document := range bytes.Split(data, []byte("\n---\n")) { + var object map[string]any + Expect(yaml.Unmarshal(document, &object)).To(Succeed()) + validator, _, err := validation.NewSchemaValidator(crdSchema()) + Expect(err).ToNot(HaveOccurred()) + Expect(validation.ValidateCustomResource(nil, object, validator)).To(BeEmpty()) + pruned, err := yaml.Marshal(object) + Expect(err).ToNot(HaveOccurred()) + pruning.Prune(object, structural, true) + after, err := yaml.Marshal(object) + Expect(err).ToNot(HaveOccurred()) + Expect(string(after)).To(Equal(string(pruned)), "nothing is pruned") + + deployer, err := deploy.Decode(document) + Expect(err).ToNot(HaveOccurred()) + _, err = deployer.Spec.JavaContainer.Container() + Expect(err).ToNot(HaveOccurred(), "the example's %s javaContainer is valid", deployer.Spec.JavaContainer.Kind) + } + }) +}) diff --git a/pkg/deploy/deploy.go b/pkg/deploy/deploy.go new file mode 100644 index 0000000..2773b42 --- /dev/null +++ b/pkg/deploy/deploy.go @@ -0,0 +1,201 @@ +package deploy + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + + dbcontext "github.com/flanksource/commons-db/context" + "k8s.io/client-go/kubernetes" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/agentinstall" + "github.com/flanksource/telltale/pkg/deploy/auth" +) + +// Provision is what every deployed agent is given to verify its executions, +// and how a deploy runs commands in a target's container. +type Provision struct { + Issuer string + JWKS auth.JWKS + JWKSURL string + Exec ExecFunc + Upgrade bool +} + +// ExecFunc runs commands in a target's JVM container. +type ExecFunc func(Target) agentinstall.Transport + +// Options are a deploy's cluster and signing key. +type Options struct { + Kubernetes kubernetes.Interface + Issuer *auth.Issuer + // Cluster is the cluster Kubernetes reaches; nil Exec runs kubectl exec + // against it. + Cluster Cluster + // Exec runs commands in a target's container; nil is Cluster.Exec(). + Exec ExecFunc + // DryRun only selects the targets, each StateSelected: it installs + // nothing and needs no Issuer. + DryRun bool + Upgrade bool +} + +// Deploy installs the agent into every target the deployer selects and +// reports each one. It is idempotent: a target already running this agent API +// with this provision is left alone. The error names every target that is +// not deployed. A dry run reports the targets it would deploy to. +func Deploy(ctx dbcontext.Context, deployer Deployer, options Options) (Status, error) { + if options.DryRun { + return selected(ctx, deployer, options.Kubernetes) + } + container, provision, err := prepare(deployer, options) + if err != nil { + return Status{}, err + } + targets, err := Resolve(ctx, options.Kubernetes, deployer.Name, deployer.Spec.Selector) + if err != nil { + return Status{}, fmt.Errorf("deployer %s: %w", deployer.Name, err) + } + // Published only once the selector picks pods to provision with it. + if provision.JWKS, err = AgentJWKS(ctx, options.Kubernetes, deployer, options.Issuer); err != nil { + return Status{}, fmt.Errorf("deployer %s: %w", deployer.Name, err) + } + var status Status + var failures []error + for _, target := range targets { + result := container.Deploy(ctx, target, provision) + result.Target, result.Namespace, result.Pod, result.Container = target.ID, target.Pod.Namespace, target.Pod.Name, target.Container.Name + status.Targets = append(status.Targets, result) + if result.State != StateDeployed { + failures = append(failures, fmt.Errorf("%s %s: %s", target.ID, result.State, result.Error)) + } + } + if len(failures) > 0 { + return status, fmt.Errorf("deployer %s: %w", deployer.Name, errors.Join(failures...)) + } + return status, nil +} + +// selected is every target the deployer selects, each StateSelected, once its +// java container is known to be deployable. +func selected(ctx dbcontext.Context, deployer Deployer, client kubernetes.Interface) (Status, error) { + if client == nil { + return Status{}, fmt.Errorf("deployer %s: a kubernetes client is required", deployer.Name) + } + if _, err := deployer.Spec.JavaContainer.Container(); err != nil { + return Status{}, fmt.Errorf("deployer %s: %w", deployer.Name, err) + } + if source := deployer.Spec.Auth.JWKS; source != nil { + if err := source.validate(); err != nil { + return Status{}, fmt.Errorf("deployer %s: %w", deployer.Name, err) + } + } + targets, err := Resolve(ctx, client, deployer.Name, deployer.Spec.Selector) + if err != nil { + return Status{}, fmt.Errorf("deployer %s: %w", deployer.Name, err) + } + var status Status + for _, target := range targets { + status.Targets = append(status.Targets, TargetStatus{Target: target.ID, Namespace: target.Pod.Namespace, Pod: target.Pod.Name, Container: target.Container.Name, State: StateSelected}) + } + return status, nil +} + +func prepare(deployer Deployer, options Options) (JavaContainer, Provision, error) { + switch { + case options.Kubernetes == nil: + return nil, Provision{}, fmt.Errorf("deployer %s: a kubernetes client is required", deployer.Name) + case options.Issuer == nil: + return nil, Provision{}, fmt.Errorf("deployer %s: an issuer is required", deployer.Name) + case deployer.Spec.Auth.Issuer == "": + return nil, Provision{}, fmt.Errorf("deployer %s: spec.auth.issuer is required", deployer.Name) + case options.Issuer.Name() != deployer.Spec.Auth.Issuer: + return nil, Provision{}, fmt.Errorf("deployer %s: issuer %q does not sign for spec.auth.issuer %q", deployer.Name, options.Issuer.Name(), deployer.Spec.Auth.Issuer) + } + container, err := deployer.Spec.JavaContainer.Container() + if err != nil { + return nil, Provision{}, fmt.Errorf("deployer %s: %w", deployer.Name, err) + } + exec := options.Exec + if exec == nil { + exec = options.Cluster.Exec() + } + if options.Upgrade && deployer.Spec.JavaContainer.Kind != KindAttach { + return nil, Provision{}, errors.New("--upgrade replaces attached Java runtimes; select an attach deployer") + } + return container, Provision{Issuer: options.Issuer.Name(), JWKS: options.Issuer.JWKS(), JWKSURL: deployer.Spec.Auth.JWKSURL, Exec: exec, Upgrade: options.Upgrade}, nil +} + +// adapterConfig is the agent configuration a target is provisioned with, +// with its digest: the agent echoes the digest, so a deploy can tell whether +// the agent it finds was given this provision. +func adapterConfig(target Target, provision Provision, extra map[string]string) (map[string]string, error) { + jwks, err := json.Marshal(provision.JWKS) + if err != nil { + return nil, err + } + config := map[string]string{"issuer": provision.Issuer, "audience": target.ID, "jwks": string(jwks)} + if provision.JWKSURL != "" { + config["jwksUrl"] = provision.JWKSURL + } + for key, value := range extra { + config[key] = value + } + canonical, err := json.Marshal(config) + if err != nil { + return nil, err + } + sum := sha256.Sum256(canonical) + config["configDigest"] = hex.EncodeToString(sum[:]) + return config, nil +} + +// presence is what a target's health says about the agent already there. +type presence int + +const ( + absent presence = iota // nothing answers at the endpoint + current // this agent API with this provision + stale // this agent API with another provision + outdated // another agent API +) + +// inspect reads a target's /health against the provision's digest. +func inspect(ctx context.Context, client *http.Client, target Target, endpoint, digest string) (presence, agentclient.Health, error) { + health, err := agentclient.HTTPClient{Target: target.ID, URL: endpoint, Client: client, Tokens: noTokens{}}.Health(ctx) + switch { + case err != nil: + return absent, health, err + case health.Status == "outdated" || health.APIVersion != agentclient.AgentAPIVersion: + return outdated, health, nil + case health.Status != "ok": + return absent, health, fmt.Errorf("agent %s health status %q", target.ID, health.Status) + case health.ConfigDigest != digest: + return stale, health, nil + } + return current, health, nil +} + +// noTokens backs the unauthenticated health check; asking it is a bug. +type noTokens struct{} + +func (noTokens) Token(context.Context, string, string) (string, error) { + return "", errors.New("the health check carries no token") +} + +func outdatedStatus(endpoint string, health agentclient.Health) TargetStatus { + message := health.Error + if message == "" { + message = fmt.Sprintf("agent API %d is loaded; this deployer requires %d; restart the JVM to replace it", health.APIVersion, agentclient.AgentAPIVersion) + } + return TargetStatus{State: StateOutdated, AgentAPIVersion: health.APIVersion, Endpoint: endpoint, Error: message} +} + +func failed(endpoint string, err error) TargetStatus { + return TargetStatus{State: StateFailed, Endpoint: endpoint, Error: err.Error()} +} diff --git a/pkg/deploy/deploy_suite_test.go b/pkg/deploy/deploy_suite_test.go new file mode 100644 index 0000000..160ffb7 --- /dev/null +++ b/pkg/deploy/deploy_suite_test.go @@ -0,0 +1,13 @@ +package deploy_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestDeploy(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Deployers") +} diff --git a/pkg/deploy/deployer.go b/pkg/deploy/deployer.go new file mode 100644 index 0000000..e206868 --- /dev/null +++ b/pkg/deploy/deployer.go @@ -0,0 +1,144 @@ +// Package deploy is the Deployer resource: it puts telltale's JVM agent into +// the Java containers a selector picks and provisions the key their executions +// are verified with. Deployment and execution are separate: a deployer +// installs and owns auth, an execution only reaches a deployed target with a +// short-lived token the deployer's issuer minted. +package deploy + +import ( + "bytes" + "encoding/json" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" + + "github.com/flanksource/telltale/pkg/definition" +) + +// The resource's kind, beside TraceDefinition in the same group. +const ( + Kind = "Deployer" + Plural = "deployers" +) + +// Deployer selects JVMs and the container kind that receives the agent. +type Deployer struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec Spec `json:"spec"` + Status Status `json:"status,omitzero"` +} + +// Spec is what a deployer installs where, and who signs its executions. +type Spec struct { + Selector Selector `json:"selector" clicky:"title=Selector,desc=Pods that receive the agent"` + JavaContainer JavaContainerSpec `json:"javaContainer" clicky:"title=Java container,desc=How the agent gets into the JVM, keyed by kind"` + Auth AuthSpec `json:"auth" clicky:"title=Auth,desc=The issuer of execution tokens"` +} + +// WorkloadKind is what a selector's labels match. +type WorkloadKind string + +const ( + WorkloadDeployment WorkloadKind = "Deployment" + WorkloadStatefulSet WorkloadKind = "StatefulSet" + WorkloadPod WorkloadKind = "Pod" +) + +// Selector picks the running pods of the one workload it names, or of the +// workloads its labels match. +type Selector struct { + Kind WorkloadKind `json:"kind" clicky:"title=Kind,desc=Deployment StatefulSet or Pod"` + Namespace string `json:"namespace" clicky:"title=Namespace"` + // Name selects one workload (or pod) of Kind, in place of Labels. + Name string `json:"name,omitempty" clicky:"title=Name,desc=The one workload (or pod) to select instead of labels"` + Labels map[string]string `json:"labels,omitempty" clicky:"title=Labels,desc=Labels of the workloads (or pods) to select"` + // Container is the JVM container; unset, each pod's only container. + Container string `json:"container,omitempty" clicky:"title=Container,desc=The container running the JVM (default: the pod's only container)"` +} + +// AuthSpec names the connection whose signingKey secret signs execution +// tokens; its reference is the tokens' issuer. +type AuthSpec struct { + Issuer string `json:"issuer" clicky:"title=Issuer,desc=connection:// reference holding the signingKey secret (RSA PEM)"` + // JWKSURL is where the issuer runtime serves its JWKS. Agents refresh + // from it on an unknown key id, so a rotation needs no redeploy. + JWKSURL string `json:"jwksUrl,omitempty" clicky:"title=JWKS URL,desc=Where the issuer runtime serves its JWKS"` + // JWKS is where the keys agents verify tokens with come from; unset, they + // are the issuer's own, provisioned inline. + JWKS *JWKSSource `json:"jwks,omitempty" clicky:"title=JWKS,desc=A Secret or ConfigMap the agents' keys are read from or published to"` +} + +// JWKSMode is how a deploy uses its JWKS object. +type JWKSMode string + +const ( + // JWKSRead provisions the JWKS the object already holds, such as one a + // shared issuer publishes; the issuer's signing key must be among them. + JWKSRead JWKSMode = "read" + // JWKSPublish writes the issuer's JWKS to the object on every deploy, then + // provisions it. + JWKSPublish JWKSMode = "publish" +) + +// JWKSSource is a Secret or ConfigMap key in the selector's namespace holding +// a JWKS document. +type JWKSSource struct { + Mode JWKSMode `json:"mode" clicky:"title=Mode,desc=read or publish"` + Kind string `json:"kind" clicky:"title=Kind,desc=ConfigMap or Secret"` + Name string `json:"name" clicky:"title=Name"` + // Key is the data key; jwks.json when unset. + Key string `json:"key,omitempty" clicky:"title=Key,desc=Data key holding the JWKS (default jwks.json)"` +} + +// State is where one target's deployment stands. +type State string + +const ( + // StateSelected is a target a dry run selected and did not deploy to. + StateSelected State = "selected" + StateDeployed State = "deployed" + StateOutdated State = "outdated" + StateFailed State = "failed" +) + +// Status is every selected target's state. +type Status struct { + Targets []TargetStatus `json:"targets,omitempty"` +} + +// TargetStatus is one selected pod's deployment. +type TargetStatus struct { + Target string `json:"target"` + Namespace string `json:"namespace"` + Pod string `json:"pod"` + Container string `json:"container"` + State State `json:"state"` + AgentAPIVersion int `json:"agentAPIVersion,omitempty"` + // Endpoint is the rendered execution URL before workload resolution, so + // a later process resolves it (a port-forward) again. + Endpoint string `json:"endpoint,omitempty"` + Error string `json:"error,omitempty"` +} + +// Decode reads a deployer from YAML or JSON, refusing unknown fields. +func Decode(data []byte) (*Deployer, error) { + raw, err := yaml.YAMLToJSON(data) + if err != nil { + return nil, fmt.Errorf("decode deployer: %w", err) + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + var deployer Deployer + if err := decoder.Decode(&deployer); err != nil { + return nil, fmt.Errorf("decode deployer: %w", err) + } + if deployer.APIVersion != definition.APIVersion || deployer.Kind != Kind { + return nil, fmt.Errorf("decode deployer: want apiVersion %s and kind %s, got %q and %q", definition.APIVersion, Kind, deployer.APIVersion, deployer.Kind) + } + if deployer.Name == "" { + return nil, fmt.Errorf("decode deployer: metadata.name is required") + } + return &deployer, nil +} diff --git a/pkg/deploy/issuer.go b/pkg/deploy/issuer.go new file mode 100644 index 0000000..e98bd82 --- /dev/null +++ b/pkg/deploy/issuer.go @@ -0,0 +1,79 @@ +package deploy + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/flanksource/commons-db/connection/store" + "sigs.k8s.io/yaml" + + "github.com/flanksource/telltale/pkg/deploy/auth" +) + +// DefaultIssuer is the issuer a Deployer built from flags signs with when +// none is named; EnsureIssuer creates it on first use. +const DefaultIssuer = "connection://telltale/issuer" + +var dnsLabel = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`) + +// EnsureIssuer creates reference's connection, with a new signingKey, in the +// connections every Flanksource tool shares (/flanksource/connections) +// unless the store already defines it, and reports whether it did. An +// existing issuer keeps its key, so agents provisioned with it stay valid. +func EnsureIssuer(connections *store.Store, reference string) (bool, error) { + namespace, name, ok := strings.Cut(strings.TrimPrefix(reference, "connection://"), "/") + if !strings.HasPrefix(reference, "connection://") || !ok || namespace == "" || name == "" || strings.Contains(name, "/") { + return false, fmt.Errorf("issuer %q: want connection:///", reference) + } + existing, err := connections.Find(reference) + if err != nil || existing != nil { + return false, err + } + for _, layer := range connections.Layers() { + if layer.Name == store.LayerGlobal { + _, err := WriteIssuer(layer.Dir, namespace, name) + return err == nil, err + } + } + return false, fmt.Errorf("issuer %s: the connection store has no %s layer to create it in", reference, store.LayerGlobal) +} + +// WriteIssuer writes dir/.yaml: a connection whose signingKey secret is +// a new RSA key, for a Deployer's spec.auth.issuer. The key never leaves the +// file, which is private; agents get only its public JWKS. An existing file +// is never replaced. +func WriteIssuer(dir, namespace, name string) (string, error) { + if !dnsLabel.MatchString(name) || (namespace != "" && !dnsLabel.MatchString(namespace)) { + return "", fmt.Errorf("issuer connection %s/%s: namespace and name must be lowercase DNS labels", namespace, name) + } + key, err := auth.GenerateSigningKey() + if err != nil { + return "", err + } + metadata := map[string]string{"name": name} + if namespace != "" { + metadata["namespace"] = namespace + } + data, err := yaml.Marshal(map[string]any{"apiVersion": "mission-control.flanksource.com/v1", "kind": "Connection", "metadata": metadata, + // http with no url: an issuer is a JWKS publisher, served where spec.auth.jwksUrl says, if anywhere; + // the connection store refuses a connection that declares no kind. + "spec": map[string]any{"http": map[string]any{}, "secrets": map[string]any{SigningKeySecret: map[string]string{"value": string(key)}}}}) + if err != nil { + return "", err + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return "", err + } + path := filepath.Join(dir, name+".yaml") + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return "", fmt.Errorf("write %s (an existing connection is never replaced): %w", path, err) + } + if _, err := file.Write(data); err != nil { + return "", fmt.Errorf("write %s: %w", path, err) + } + return path, file.Close() +} diff --git a/pkg/deploy/issuer_spec_test.go b/pkg/deploy/issuer_spec_test.go new file mode 100644 index 0000000..21c3ab2 --- /dev/null +++ b/pkg/deploy/issuer_spec_test.go @@ -0,0 +1,53 @@ +package deploy_test + +import ( + "os" + "path/filepath" + + "github.com/flanksource/commons-db/connection/store" + dbcontext "github.com/flanksource/commons-db/context" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/telltale/pkg/deploy" +) + +var _ = Describe("the default issuer", func() { + var connections *store.Store + var global string + + BeforeEach(func() { + dir := GinkgoT().TempDir() + global = filepath.Join(dir, "flanksource", "connections") + var err error + connections, err = store.New(store.Layer{Name: store.LayerGlobal, Dir: global}, store.Layer{Name: store.LayerCWD, Dir: filepath.Join(dir, ".connections")}) + Expect(err).ToNot(HaveOccurred()) + }) + + It("is created once, private, in the shared connections, with a key NewIssuer signs with", func() { + created, err := deploy.EnsureIssuer(connections, deploy.DefaultIssuer) + Expect(err).ToNot(HaveOccurred()) + Expect(created).To(BeTrue()) + info, err := os.Stat(filepath.Join(global, "issuer.yaml")) + Expect(err).ToNot(HaveOccurred()) + Expect(info.Mode().Perm()).To(Equal(os.FileMode(0o600))) + first, err := os.ReadFile(filepath.Join(global, "issuer.yaml")) + Expect(err).ToNot(HaveOccurred()) + + issuer, err := deploy.NewIssuer(connections.Attach(dbcontext.New()), deploy.AuthSpec{Issuer: deploy.DefaultIssuer}) + Expect(err).ToNot(HaveOccurred()) + Expect(issuer.Name()).To(Equal(deploy.DefaultIssuer)) + + created, err = deploy.EnsureIssuer(connections, deploy.DefaultIssuer) + Expect(err).ToNot(HaveOccurred()) + Expect(created).To(BeFalse(), "an existing issuer keeps its key") + again, err := os.ReadFile(filepath.Join(global, "issuer.yaml")) + Expect(err).ToNot(HaveOccurred()) + Expect(again).To(Equal(first)) + }) + + It("refuses a reference that names no namespace and connection", func() { + _, err := deploy.EnsureIssuer(connections, "https://issuer.example") + Expect(err).To(MatchError(ContainSubstring("connection:///"))) + }) +}) diff --git a/pkg/deploy/javaagent.go b/pkg/deploy/javaagent.go new file mode 100644 index 0000000..5079e03 --- /dev/null +++ b/pkg/deploy/javaagent.go @@ -0,0 +1,17 @@ +package deploy + +import ( + "encoding/json" + "fmt" +) + +// KindJavaAgent is a JVM that pre-loads the agent (an init container or +// JAVA_TOOL_OPTIONS). It is registered so a deployer naming it fails with +// what is missing rather than as an unknown kind. +const KindJavaAgent = "javaagent" + +func init() { + RegisterJavaContainer(KindJavaAgent, func(json.RawMessage) (JavaContainer, error) { + return nil, fmt.Errorf("kind %s is not implemented yet: use %s to attach the agent at runtime or %s to deploy the WAR", KindJavaAgent, KindAttach, KindTomcat) + }) +} diff --git a/pkg/deploy/jwks.go b/pkg/deploy/jwks.go new file mode 100644 index 0000000..fc1617a --- /dev/null +++ b/pkg/deploy/jwks.go @@ -0,0 +1,135 @@ +package deploy + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + "github.com/flanksource/telltale/pkg/deploy/auth" +) + +const defaultJWKSKey = "jwks.json" + +func (s JWKSSource) validate() error { + switch { + case s.Mode != JWKSRead && s.Mode != JWKSPublish: + return fmt.Errorf("jwks mode %q is not %s or %s", s.Mode, JWKSRead, JWKSPublish) + case s.Kind != "ConfigMap" && s.Kind != "Secret": + return fmt.Errorf("jwks kind %q is not ConfigMap or Secret", s.Kind) + case s.Name == "": + return errors.New("jwks name is required") + } + return nil +} + +func (s JWKSSource) key() string { + if s.Key == "" { + return defaultJWKSKey + } + return s.Key +} + +// AgentJWKS is the JWKS a deployer's agents verify tokens with: the issuer's +// own, or the one its Secret or ConfigMap holds once read, or published to it. +func AgentJWKS(ctx context.Context, client kubernetes.Interface, deployer Deployer, issuer *auth.Issuer) (auth.JWKS, error) { + source := deployer.Spec.Auth.JWKS + if source == nil { + return issuer.JWKS(), nil + } + if err := source.validate(); err != nil { + return auth.JWKS{}, err + } + namespace := deployer.Spec.Selector.Namespace + if source.Mode == JWKSPublish { + return issuer.JWKS(), publishJWKS(ctx, client, namespace, deployer.Name, *source, issuer.JWKS()) + } + data, err := readJWKS(ctx, client, namespace, *source) + if err != nil { + return auth.JWKS{}, err + } + var jwks auth.JWKS + if err := json.Unmarshal(data, &jwks); err != nil { + return auth.JWKS{}, fmt.Errorf("%s %s/%s key %s is not a JWKS: %w", source.Kind, namespace, source.Name, source.key(), err) + } + signing := issuer.JWKS().Keys[0].KeyID + if !slices.ContainsFunc(jwks.Keys, func(key auth.JWK) bool { return key.KeyID == signing }) { + return auth.JWKS{}, fmt.Errorf("%s %s/%s does not hold the signing key %s of %s, so agents would refuse its tokens", source.Kind, namespace, source.Name, signing, issuer.Name()) + } + return jwks, nil +} + +func readJWKS(ctx context.Context, client kubernetes.Interface, namespace string, source JWKSSource) ([]byte, error) { + var data []byte + var found bool + switch source.Kind { + case "ConfigMap": + object, err := client.CoreV1().ConfigMaps(namespace).Get(ctx, source.Name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("read ConfigMap %s/%s: %w", namespace, source.Name, err) + } + var text string + text, found = object.Data[source.key()] + data = []byte(text) + default: + object, err := client.CoreV1().Secrets(namespace).Get(ctx, source.Name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("read Secret %s/%s: %w", namespace, source.Name, err) + } + data, found = object.Data[source.key()] + } + if !found { + return nil, fmt.Errorf("%s %s/%s has no key %q", source.Kind, namespace, source.Name, source.key()) + } + return data, nil +} + +// publishJWKS writes jwks to the source's key, creating the object labelled +// as telltale's when it does not exist yet. +func publishJWKS(ctx context.Context, client kubernetes.Interface, namespace, deployer string, source JWKSSource, jwks auth.JWKS) error { + data, err := json.Marshal(jwks) + if err != nil { + return err + } + meta := metav1.ObjectMeta{Name: source.Name, Namespace: namespace, Labels: map[string]string{"app.kubernetes.io/managed-by": "telltale", "telltale.flanksource.com/deployer": deployer}} + if source.Kind == "ConfigMap" { + configMaps := client.CoreV1().ConfigMaps(namespace) + existing, err := configMaps.Get(ctx, source.Name, metav1.GetOptions{}) + switch { + case apierrors.IsNotFound(err): + _, err = configMaps.Create(ctx, &corev1.ConfigMap{ObjectMeta: meta, Data: map[string]string{source.key(): string(data)}}, metav1.CreateOptions{}) + case err == nil: + if existing.Data == nil { + existing.Data = map[string]string{} + } + existing.Data[source.key()] = string(data) + _, err = configMaps.Update(ctx, existing, metav1.UpdateOptions{}) + } + if err != nil { + return fmt.Errorf("publish JWKS to ConfigMap %s/%s: %w", namespace, source.Name, err) + } + return nil + } + secrets := client.CoreV1().Secrets(namespace) + existing, err := secrets.Get(ctx, source.Name, metav1.GetOptions{}) + switch { + case apierrors.IsNotFound(err): + _, err = secrets.Create(ctx, &corev1.Secret{ObjectMeta: meta, Data: map[string][]byte{source.key(): data}}, metav1.CreateOptions{}) + case err == nil: + if existing.Data == nil { + existing.Data = map[string][]byte{} + } + existing.Data[source.key()] = data + _, err = secrets.Update(ctx, existing, metav1.UpdateOptions{}) + } + if err != nil { + return fmt.Errorf("publish JWKS to Secret %s/%s: %w", namespace, source.Name, err) + } + return nil +} diff --git a/pkg/deploy/jwks_spec_test.go b/pkg/deploy/jwks_spec_test.go new file mode 100644 index 0000000..8c54615 --- /dev/null +++ b/pkg/deploy/jwks_spec_test.go @@ -0,0 +1,100 @@ +package deploy_test + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "encoding/json" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/flanksource/telltale/pkg/deploy" + "github.com/flanksource/telltale/pkg/deploy/auth" +) + +var _ = Describe("deployer JWKS", func() { + const issuerName = "connection://shop/signing" + var issuer *auth.Issuer + var other auth.JWK + + BeforeEach(func() { + key, err := rsa.GenerateKey(rand.Reader, 2048) + Expect(err).NotTo(HaveOccurred()) + issuer, err = auth.NewIssuer(issuerName, key) + Expect(err).NotTo(HaveOccurred()) + otherKey, err := rsa.GenerateKey(rand.Reader, 2048) + Expect(err).NotTo(HaveOccurred()) + shared, err := auth.NewIssuer("connection://shop/shared", otherKey) + Expect(err).NotTo(HaveOccurred()) + other = shared.JWKS().Keys[0] + }) + + deployer := func(source *deploy.JWKSSource) deploy.Deployer { + return deploy.Deployer{Spec: deploy.Spec{Selector: deploy.Selector{Namespace: "shop"}, Auth: deploy.AuthSpec{Issuer: issuerName, JWKS: source}}} + } + document := func(keys ...auth.JWK) string { + data, err := json.Marshal(auth.JWKS{Keys: keys}) + Expect(err).NotTo(HaveOccurred()) + return string(data) + } + + It("is the issuer's own keys when no object is named", func() { + jwks, err := deploy.AgentJWKS(context.Background(), fake.NewClientset(), deployer(nil), issuer) + Expect(err).NotTo(HaveOccurred()) + Expect(jwks).To(Equal(issuer.JWKS())) + }) + + It("publishes the issuer's keys to a ConfigMap it creates, then to the same one on the next deploy", func() { + client := fake.NewClientset() + source := &deploy.JWKSSource{Mode: deploy.JWKSPublish, Kind: "ConfigMap", Name: "telltale-jwks"} + for range 2 { + jwks, err := deploy.AgentJWKS(context.Background(), client, deployer(source), issuer) + Expect(err).NotTo(HaveOccurred()) + Expect(jwks).To(Equal(issuer.JWKS())) + } + published, err := client.CoreV1().ConfigMaps("shop").Get(context.Background(), "telltale-jwks", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(published.Data).To(Equal(map[string]string{"jwks.json": document(issuer.JWKS().Keys...)})) + Expect(published.Labels).To(HaveKeyWithValue("app.kubernetes.io/managed-by", "telltale")) + }) + + It("publishes to a Secret key it names", func() { + client := fake.NewClientset() + _, err := deploy.AgentJWKS(context.Background(), client, deployer(&deploy.JWKSSource{Mode: deploy.JWKSPublish, Kind: "Secret", Name: "keys", Key: "agents.jwks"}), issuer) + Expect(err).NotTo(HaveOccurred()) + secret, err := client.CoreV1().Secrets("shop").Get(context.Background(), "keys", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(string(secret.Data["agents.jwks"])).To(Equal(document(issuer.JWKS().Keys...))) + }) + + It("reads the keys another issuer already published, when they include this issuer's signing key", func() { + client := fake.NewClientset(&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "shared-jwks", Namespace: "shop"}, Data: map[string][]byte{"jwks.json": []byte(document(other, issuer.JWKS().Keys[0]))}}) + jwks, err := deploy.AgentJWKS(context.Background(), client, deployer(&deploy.JWKSSource{Mode: deploy.JWKSRead, Kind: "Secret", Name: "shared-jwks"}), issuer) + Expect(err).NotTo(HaveOccurred()) + Expect(jwks.Keys).To(Equal([]auth.JWK{other, issuer.JWKS().Keys[0]})) + }) + + DescribeTable("refuses a JWKS source it cannot provision", + func(source deploy.JWKSSource, objects []corev1.ConfigMap, message string) { + client := fake.NewClientset() + for i := range objects { + _, err := client.CoreV1().ConfigMaps("shop").Create(context.Background(), &objects[i], metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + } + _, err := deploy.AgentJWKS(context.Background(), client, deployer(&source), issuer) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("an unknown mode", deploy.JWKSSource{Mode: "sync", Kind: "ConfigMap", Name: "k"}, nil, `jwks mode "sync"`), + Entry("an unknown kind", deploy.JWKSSource{Mode: deploy.JWKSRead, Kind: "Vault", Name: "k"}, nil, `jwks kind "Vault"`), + Entry("no name", deploy.JWKSSource{Mode: deploy.JWKSRead, Kind: "ConfigMap"}, nil, "jwks name is required"), + Entry("a missing object", deploy.JWKSSource{Mode: deploy.JWKSRead, Kind: "ConfigMap", Name: "absent"}, nil, "read ConfigMap shop/absent"), + Entry("a missing key", deploy.JWKSSource{Mode: deploy.JWKSRead, Kind: "ConfigMap", Name: "keys"}, + []corev1.ConfigMap{{ObjectMeta: metav1.ObjectMeta{Name: "keys", Namespace: "shop"}, Data: map[string]string{"other": "{}"}}}, `ConfigMap shop/keys has no key "jwks.json"`), + Entry("keys without the issuer's", deploy.JWKSSource{Mode: deploy.JWKSRead, Kind: "ConfigMap", Name: "keys"}, + []corev1.ConfigMap{{ObjectMeta: metav1.ObjectMeta{Name: "keys", Namespace: "shop"}, Data: map[string]string{"jwks.json": `{"keys":[]}`}}}, "does not hold the signing key"), + ) +}) diff --git a/pkg/deploy/registry.go b/pkg/deploy/registry.go new file mode 100644 index 0000000..bb818a1 --- /dev/null +++ b/pkg/deploy/registry.go @@ -0,0 +1,102 @@ +package deploy + +import ( + "bytes" + "encoding/json" + "fmt" + "maps" + "net/http" + "slices" + "strings" + "sync" + + dbcontext "github.com/flanksource/commons-db/context" +) + +// JavaContainer gets the agent into one target's JVM. +type JavaContainer interface { + // Deploy installs the agent into target, or confirms it is already + // there with this provision, and reports where it executes. + Deploy(ctx dbcontext.Context, target Target, provision Provision) TargetStatus + // Client is the HTTP client executions reach a deployed target with. + Client(ctx dbcontext.Context, namespace string) (*http.Client, error) +} + +// Factory decodes one kind's javaContainer spec. +type Factory func(spec json.RawMessage) (JavaContainer, error) + +var ( + containersMu sync.RWMutex + containers = map[string]Factory{} +) + +// RegisterJavaContainer adds a javaContainer kind; a kind registers once. +func RegisterJavaContainer(kind string, factory Factory) { + containersMu.Lock() + defer containersMu.Unlock() + if kind == "" || factory == nil { + panic("deploy: RegisterJavaContainer needs a kind and a factory") + } + if _, exists := containers[kind]; exists { + panic(fmt.Sprintf("deploy: javaContainer kind %q registered twice", kind)) + } + containers[kind] = factory +} + +// JavaContainerSpec is a javaContainer object: its kind picks the factory +// that decodes the rest of it. +type JavaContainerSpec struct { + Kind string `json:"kind" clicky:"title=Kind,desc=tomcat attach or javaagent"` + raw json.RawMessage +} + +// NewJavaContainerSpec is the spec of kind with the given fields. +func NewJavaContainerSpec(spec any) (JavaContainerSpec, error) { + data, err := json.Marshal(spec) + if err != nil { + return JavaContainerSpec{}, err + } + var decoded JavaContainerSpec + return decoded, json.Unmarshal(data, &decoded) +} + +func (s *JavaContainerSpec) UnmarshalJSON(data []byte) error { + var head struct { + Kind string `json:"kind"` + } + if err := json.Unmarshal(data, &head); err != nil { + return err + } + s.Kind, s.raw = head.Kind, bytes.Clone(data) + return nil +} + +func (s JavaContainerSpec) MarshalJSON() ([]byte, error) { + if s.raw == nil { + return json.Marshal(map[string]string{"kind": s.Kind}) + } + return s.raw, nil +} + +// Container builds the registered java container for the spec. +func (s JavaContainerSpec) Container() (JavaContainer, error) { + containersMu.RLock() + factory, ok := containers[s.Kind] + kinds := slices.Sorted(maps.Keys(containers)) + containersMu.RUnlock() + if !ok { + return nil, fmt.Errorf("javaContainer kind %q is not registered (registered: %s)", s.Kind, strings.Join(kinds, ", ")) + } + container, err := factory(s.raw) + if err != nil { + return nil, fmt.Errorf("javaContainer %s: %w", s.Kind, err) + } + return container, nil +} + +// decodeStrict decodes a kind's spec, refusing a field the kind does not know. +func decodeStrict(data json.RawMessage, into any) error { + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + return decoder.Decode(into) +} diff --git a/pkg/deploy/resolve.go b/pkg/deploy/resolve.go new file mode 100644 index 0000000..4954dde --- /dev/null +++ b/pkg/deploy/resolve.go @@ -0,0 +1,81 @@ +package deploy + +import ( + "context" + "errors" + "fmt" + "strings" + + dbcontext "github.com/flanksource/commons-db/context" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/deploy/auth" +) + +// TargetPrefix starts every target id, so a registry hands those ids to a +// Resolver and every other id to local discovery. +const TargetPrefix = "deployer/" + +// ParseTargetID is the deployer, namespace and pod a target id names. +func ParseTargetID(id string) (deployer, namespace, pod string, err error) { + parts := strings.Split(strings.TrimPrefix(id, TargetPrefix), "/") + if !strings.HasPrefix(id, TargetPrefix) || len(parts) != 3 || parts[0] == "" || parts[1] == "" || parts[2] == "" { + return "", "", "", fmt.Errorf("target id %q: want deployer///", id) + } + return parts[0], parts[1], parts[2], nil +} + +// Resolver connects a stored deployer's target the first time an execution +// names it, minting each request's token with the deployer's issuer. +type Resolver struct { + Store Store + // Subject is the principal the tokens name, such as operator@host. + Subject string + // Context is the context connection references (the issuer's, a + // port-forward's) resolve in. + Context func(context.Context) (dbcontext.Context, error) + // Issuer is the issuer spec.auth names; nil is NewIssuer. + Issuer func(dbcontext.Context, AuthSpec) (*auth.Issuer, error) +} + +// Resolve connects target id into registry; it is an agentclient.Resolver. +func (r Resolver) Resolve(ctx context.Context, registry *agentclient.Registry, id string) error { + name, _, _, err := ParseTargetID(id) + if err != nil { + return err + } + if r.Subject == "" || r.Context == nil { + return errors.New("deployer resolver: a subject and a context are required") + } + deployer, err := r.Store.Load(name) + if err != nil { + return err + } + var target *TargetStatus + for i := range deployer.Status.Targets { + if deployer.Status.Targets[i].Target == id { + target = &deployer.Status.Targets[i] + } + } + if target == nil { + return fmt.Errorf("deployer %s has no deployed target %s: run telltale jvm deploy for its current pods", name, id) + } + dbctx, err := r.Context(ctx) + if err != nil { + return err + } + if dbctx, err = ClusterOf(*deployer).Bind(dbctx); err != nil { + return fmt.Errorf("deployer %s: %w", name, err) + } + newIssuer := r.Issuer + if newIssuer == nil { + newIssuer = NewIssuer + } + issuer, err := newIssuer(dbctx, deployer.Spec.Auth) + if err != nil { + return fmt.Errorf("deployer %s: %w", name, err) + } + tokens := auth.Tokens{Issuer: issuer, Subject: r.Subject, TTL: auth.MaxTTL} + _, err = Connect(dbctx, *deployer, Status{Targets: []TargetStatus{*target}}, registry, tokens) + return err +} diff --git a/pkg/deploy/selector.go b/pkg/deploy/selector.go new file mode 100644 index 0000000..5ed4fbd --- /dev/null +++ b/pkg/deploy/selector.go @@ -0,0 +1,181 @@ +package deploy + +import ( + "context" + "errors" + "fmt" + "maps" + "slices" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" +) + +// Target is one selected pod's JVM container. +type Target struct { + // ID names the target to its agent: the audience of its tokens and the + // identity a capture selects it by. + ID string + Pod *corev1.Pod + Container corev1.Container +} + +// TargetID is the target id of a pod a deployer selected. +func TargetID(deployer, namespace, pod string) string { + return "deployer/" + deployer + "/" + namespace + "/" + pod +} + +func (s Selector) validate() error { + switch { + case s.Namespace == "": + return errors.New("selector namespace is required") + case len(s.Labels) == 0 && s.Name == "": + return errors.New("selector labels or a workload name is required") + case len(s.Labels) > 0 && s.Name != "": + return errors.New("selector names a workload or matches labels, not both") + } + return nil +} + +// Resolve is the running pods selector picks, sorted by name. +func Resolve(ctx context.Context, client kubernetes.Interface, deployer string, selector Selector) ([]Target, error) { + if err := selector.validate(); err != nil { + return nil, err + } + if selector.Name != "" && selector.Kind == WorkloadPod { + return namedPod(ctx, client, deployer, selector) + } + podSelectors, err := podSelectors(ctx, client, selector) + if err != nil { + return nil, err + } + pods := map[string]*corev1.Pod{} + for _, podSelector := range podSelectors { + list, err := client.CoreV1().Pods(selector.Namespace).List(ctx, metav1.ListOptions{LabelSelector: podSelector}) + if err != nil { + return nil, fmt.Errorf("list pods in %s matching %s: %w", selector.Namespace, podSelector, err) + } + for i := range list.Items { + if list.Items[i].Status.Phase == corev1.PodRunning { + pods[list.Items[i].Name] = &list.Items[i] + } + } + } + if len(pods) == 0 { + return nil, noRunningPods(selector) + } + return targets(deployer, selector.Container, pods) +} + +func noRunningPods(selector Selector) error { + which := selector.Name + if which == "" { + which = labels.SelectorFromSet(selector.Labels).String() + } + return fmt.Errorf("selector %s %s in %s matched no running pods", selector.Kind, which, selector.Namespace) +} + +// namedPod is the one pod a Pod selector names, when it runs. +func namedPod(ctx context.Context, client kubernetes.Interface, deployer string, selector Selector) ([]Target, error) { + pod, err := client.CoreV1().Pods(selector.Namespace).Get(ctx, selector.Name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("get Pod %s/%s: %w", selector.Namespace, selector.Name, err) + } + if pod.Status.Phase != corev1.PodRunning { + return nil, noRunningPods(selector) + } + return targets(deployer, selector.Container, map[string]*corev1.Pod{pod.Name: pod}) +} + +func targets(deployer, container string, pods map[string]*corev1.Pod) ([]Target, error) { + var result []Target + for _, name := range slices.Sorted(maps.Keys(pods)) { + pod := pods[name] + index := slices.IndexFunc(pod.Spec.Containers, func(c corev1.Container) bool { return c.Name == container }) + switch { + case container == "" && len(pod.Spec.Containers) == 1: + index = 0 + case container == "": + names := make([]string, len(pod.Spec.Containers)) + for i, c := range pod.Spec.Containers { + names[i] = c.Name + } + return nil, fmt.Errorf("pod %s/%s runs containers %s: name the JVM container", pod.Namespace, pod.Name, strings.Join(names, ", ")) + case index < 0: + return nil, fmt.Errorf("pod %s/%s has no container %q", pod.Namespace, pod.Name, container) + } + result = append(result, Target{ID: TargetID(deployer, pod.Namespace, pod.Name), Pod: pod, Container: pod.Spec.Containers[index]}) + } + return result, nil +} + +// podSelectors are the label selectors of the pods the workloads own. +func podSelectors(ctx context.Context, client kubernetes.Interface, selector Selector) ([]string, error) { + workloads := metav1.ListOptions{LabelSelector: labels.SelectorFromSet(selector.Labels).String()} + if selector.Name != "" { + return namedWorkloadSelector(ctx, client, selector) + } + switch selector.Kind { + case WorkloadPod: + return []string{workloads.LabelSelector}, nil + case WorkloadDeployment: + list, err := client.AppsV1().Deployments(selector.Namespace).List(ctx, workloads) + if err != nil { + return nil, fmt.Errorf("list deployments: %w", err) + } + return workloadSelectors(len(list.Items), func(i int) (string, *metav1.LabelSelector) { return list.Items[i].Name, list.Items[i].Spec.Selector }) + case WorkloadStatefulSet: + list, err := client.AppsV1().StatefulSets(selector.Namespace).List(ctx, workloads) + if err != nil { + return nil, fmt.Errorf("list statefulsets: %w", err) + } + return workloadSelectors(len(list.Items), func(i int) (string, *metav1.LabelSelector) { return list.Items[i].Name, list.Items[i].Spec.Selector }) + default: + return nil, fmt.Errorf("unsupported selector kind %q (supported: %s)", selector.Kind, strings.Join([]string{string(WorkloadDeployment), string(WorkloadStatefulSet), string(WorkloadPod)}, ", ")) + } +} + +// namedWorkloadSelector is the pod selector of the one Deployment or +// StatefulSet a selector names. +func namedWorkloadSelector(ctx context.Context, client kubernetes.Interface, selector Selector) ([]string, error) { + var podSelector *metav1.LabelSelector + var err error + switch selector.Kind { + case WorkloadDeployment: + var workload *appsv1.Deployment + if workload, err = client.AppsV1().Deployments(selector.Namespace).Get(ctx, selector.Name, metav1.GetOptions{}); err == nil { + podSelector = workload.Spec.Selector + } + case WorkloadStatefulSet: + var workload *appsv1.StatefulSet + if workload, err = client.AppsV1().StatefulSets(selector.Namespace).Get(ctx, selector.Name, metav1.GetOptions{}); err == nil { + podSelector = workload.Spec.Selector + } + default: + return nil, fmt.Errorf("unsupported selector kind %q (supported: %s)", selector.Kind, strings.Join([]string{string(WorkloadDeployment), string(WorkloadStatefulSet), string(WorkloadPod)}, ", ")) + } + if err != nil { + return nil, fmt.Errorf("get %s %s/%s: %w", selector.Kind, selector.Namespace, selector.Name, err) + } + return workloadSelectors(1, func(int) (string, *metav1.LabelSelector) { return selector.Name, podSelector }) +} + +func workloadSelectors(count int, item func(int) (string, *metav1.LabelSelector)) ([]string, error) { + result := make([]string, 0, count) + for i := range count { + name, selector := item(i) + parsed, err := metav1.LabelSelectorAsSelector(selector) + if err != nil { + return nil, fmt.Errorf("workload %s pod selector: %w", name, err) + } + if parsed.Empty() { + return nil, fmt.Errorf("workload %s has an empty pod selector", name) + } + result = append(result, parsed.String()) + } + return result, nil +} diff --git a/pkg/deploy/selector_spec_test.go b/pkg/deploy/selector_spec_test.go new file mode 100644 index 0000000..113b26d --- /dev/null +++ b/pkg/deploy/selector_spec_test.go @@ -0,0 +1,109 @@ +package deploy_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + + "github.com/flanksource/telltale/pkg/deploy" +) + +const ( + namespace = "orders" + jvm = "app" +) + +func pod(name string, labels map[string]string, phase corev1.PodPhase, ports ...corev1.ContainerPort) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace, Labels: labels}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "sidecar"}, {Name: jvm, Ports: ports}}}, + Status: corev1.PodStatus{Phase: phase, PodIP: "10.0.0.1"}, + } +} + +func cluster() *fake.Clientset { + web := map[string]string{"app": "orders-web", "pod-template-hash": "abc"} + objects := []runtime.Object{ + &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "orders-web", Namespace: namespace, Labels: map[string]string{"tier": "web"}}, + Spec: appsv1.DeploymentSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "orders-web"}}}}, + &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "orders-db", Namespace: namespace, Labels: map[string]string{"tier": "web"}}, + Spec: appsv1.StatefulSetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "orders-db"}}}}, + pod("orders-web-b", web, corev1.PodRunning), + pod("orders-web-a", web, corev1.PodRunning), + pod("orders-web-pending", web, corev1.PodPending), + pod("orders-db-0", map[string]string{"app": "orders-db"}, corev1.PodRunning), + pod("billing", map[string]string{"app": "billing", "tier": "web"}, corev1.PodRunning), + } + return fake.NewClientset(objects...) +} + +func selected(targets []deploy.Target) []string { + names := make([]string, len(targets)) + for i, target := range targets { + names[i] = target.Pod.Name + } + return names +} + +var _ = Describe("Selector", func() { + DescribeTable("resolves exactly the running pods of the workloads its labels match", + func(kind deploy.WorkloadKind, labels map[string]string, want []string) { + targets, err := deploy.Resolve(context.Background(), cluster(), "example", deploy.Selector{Kind: kind, Namespace: namespace, Labels: labels, Container: jvm}) + Expect(err).ToNot(HaveOccurred()) + Expect(selected(targets)).To(Equal(want)) + }, + Entry("a Deployment's pods, sorted, without the pending one", deploy.WorkloadDeployment, map[string]string{"tier": "web"}, []string{"orders-web-a", "orders-web-b"}), + Entry("a StatefulSet's pods", deploy.WorkloadStatefulSet, map[string]string{"tier": "web"}, []string{"orders-db-0"}), + Entry("pods by their own labels", deploy.WorkloadPod, map[string]string{"tier": "web"}, []string{"billing"}), + ) + + DescribeTable("resolves the running pods of the one workload it names", + func(kind deploy.WorkloadKind, name string, want []string) { + targets, err := deploy.Resolve(context.Background(), cluster(), "example", deploy.Selector{Kind: kind, Namespace: namespace, Name: name, Container: jvm}) + Expect(err).ToNot(HaveOccurred()) + Expect(selected(targets)).To(Equal(want)) + }, + Entry("a Deployment by name, though its own labels match nothing of its pods", deploy.WorkloadDeployment, "orders-web", []string{"orders-web-a", "orders-web-b"}), + Entry("a StatefulSet by name", deploy.WorkloadStatefulSet, "orders-db", []string{"orders-db-0"}), + Entry("a pod by name", deploy.WorkloadPod, "billing", []string{"billing"}), + ) + + It("names each target by deployer, namespace and pod", func() { + targets, err := deploy.Resolve(context.Background(), cluster(), "example", deploy.Selector{Kind: deploy.WorkloadStatefulSet, Namespace: namespace, Labels: map[string]string{"tier": "web"}, Container: jvm}) + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(HaveLen(1)) + Expect(targets[0].ID).To(Equal("deployer/example/orders/orders-db-0")) + Expect(targets[0].Container.Name).To(Equal(jvm)) + }) + + It("takes a pod's only container when the selector names none", func() { + single := pod("cycle-0", map[string]string{"app": "cycle"}, corev1.PodRunning) + single.Spec.Containers = single.Spec.Containers[1:] + targets, err := deploy.Resolve(context.Background(), fake.NewClientset(single), "example", deploy.Selector{Kind: deploy.WorkloadPod, Namespace: namespace, Name: "cycle-0"}) + Expect(err).ToNot(HaveOccurred()) + Expect(targets).To(HaveLen(1)) + Expect(targets[0].Container.Name).To(Equal(jvm)) + }) + + DescribeTable("fails loud on a selector that cannot select", + func(selector deploy.Selector, message string) { + _, err := deploy.Resolve(context.Background(), cluster(), "example", selector) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("an unknown kind", deploy.Selector{Kind: "DaemonSet", Namespace: namespace, Labels: map[string]string{"tier": "web"}, Container: jvm}, `unsupported selector kind "DaemonSet"`), + Entry("neither labels nor a name", deploy.Selector{Kind: deploy.WorkloadPod, Namespace: namespace, Container: jvm}, "selector labels or a workload name is required"), + Entry("both labels and a name", deploy.Selector{Kind: deploy.WorkloadDeployment, Namespace: namespace, Name: "orders-web", Labels: map[string]string{"tier": "web"}, Container: jvm}, "selector names a workload or matches labels, not both"), + Entry("a named workload that does not exist", deploy.Selector{Kind: deploy.WorkloadDeployment, Namespace: namespace, Name: "orders-api", Container: jvm}, `get Deployment orders/orders-api`), + Entry("a pending named pod", deploy.Selector{Kind: deploy.WorkloadPod, Namespace: namespace, Name: "orders-web-pending", Container: jvm}, "selector Pod orders-web-pending in orders matched no running pods"), + Entry("no namespace", deploy.Selector{Kind: deploy.WorkloadPod, Labels: map[string]string{"tier": "web"}, Container: jvm}, "namespace"), + Entry("no container, on a pod running several", deploy.Selector{Kind: deploy.WorkloadPod, Namespace: namespace, Labels: map[string]string{"tier": "web"}}, `pod orders/billing runs containers sidecar, app: name the JVM container`), + Entry("a container a pod lacks", deploy.Selector{Kind: deploy.WorkloadPod, Namespace: namespace, Labels: map[string]string{"tier": "web"}, Container: "jvm"}, `pod orders/billing has no container "jvm"`), + Entry("no match", deploy.Selector{Kind: deploy.WorkloadPod, Namespace: namespace, Labels: map[string]string{"tier": "none"}, Container: jvm}, "matched no running pods"), + ) +}) diff --git a/pkg/deploy/store.go b/pkg/deploy/store.go new file mode 100644 index 0000000..11d353b --- /dev/null +++ b/pkg/deploy/store.go @@ -0,0 +1,120 @@ +package deploy + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + + "sigs.k8s.io/yaml" + + "github.com/flanksource/telltale/pkg/agentclient" +) + +// Store keeps applied deployers with the status their last deploy reported, +// so a later process connects to their targets without deploying again. +type Store struct{ Dir string } + +// deployerName is a Kubernetes object name, which is also a file name. +var deployerName = regexp.MustCompile(`^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$`) + +func (s Store) path(name string) (string, error) { + if s.Dir == "" { + return "", errors.New("deployer store: a directory is required") + } + if !deployerName.MatchString(name) { + return "", fmt.Errorf("invalid deployer name %q", name) + } + return filepath.Join(s.Dir, name+".yaml"), nil +} + +// Save writes deployer, replacing the one stored under its name. +func (s Store) Save(deployer Deployer) error { + path, err := s.path(deployer.Name) + if err != nil { + return err + } + data, err := yaml.Marshal(deployer) + if err != nil { + return fmt.Errorf("save deployer %s: %w", deployer.Name, err) + } + if err := os.MkdirAll(s.Dir, 0o700); err != nil { + return fmt.Errorf("save deployer %s: %w", deployer.Name, err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + return fmt.Errorf("save deployer %s: %w", deployer.Name, err) + } + return nil +} + +// Load is the deployer stored under name. +func (s Store) Load(name string) (*Deployer, error) { + path, err := s.path(name) + if err != nil { + return nil, err + } + data, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("deployer %q has not been deployed: run telltale jvm deploy -f ", name) + } + if err != nil { + return nil, err + } + deployer, err := Decode(data) + if err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + if deployer.Name != name { + return nil, fmt.Errorf("%s holds deployer %q, not %q", path, deployer.Name, name) + } + return deployer, nil +} + +// List is every stored deployer, by name. +func (s Store) List() ([]Deployer, error) { + entries, err := os.ReadDir(s.Dir) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + var deployers []Deployer + for _, entry := range entries { + name, isDeployer := strings.CutSuffix(entry.Name(), ".yaml") + if entry.IsDir() || !isDeployer { + continue + } + deployer, err := s.Load(name) + if err != nil { + return nil, err + } + deployers = append(deployers, *deployer) + } + slices.SortFunc(deployers, func(a, b Deployer) int { return strings.Compare(a.Name, b.Name) }) + return deployers, nil +} + +// Known is every deployed target of the stored deployers, as agents a lookup +// can find by deployer, pod or namespace/pod before they are connected. +func (s Store) Known() ([]agentclient.Known, error) { + deployers, err := s.List() + if err != nil { + return nil, err + } + var known []agentclient.Known + for _, deployer := range deployers { + for _, target := range deployer.Status.Targets { + if target.State != StateDeployed { + continue + } + known = append(known, agentclient.Known{ID: target.Target, Runtime: "java", Names: []string{deployer.Name, target.Pod, target.Namespace + "/" + target.Pod}, + Resource: targetResource(deployer.Name, target), State: string(target.State), Endpoint: target.Endpoint}) + } + } + return known, nil +} diff --git a/pkg/deploy/store_spec_test.go b/pkg/deploy/store_spec_test.go new file mode 100644 index 0000000..e62fe05 --- /dev/null +++ b/pkg/deploy/store_spec_test.go @@ -0,0 +1,144 @@ +package deploy_test + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/golang-jwt/jwt/v5" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/deploy" + "github.com/flanksource/telltale/pkg/deploy/auth" +) + +// verifyingAgent is a deployed agent's /hello and /call that, like the real +// listener, admits only a token its issuer signed for this target and scope. +type verifyingAgent struct { + key *rsa.PublicKey + target string + mu sync.Mutex + grants []string +} + +func (v *verifyingAgent) admit(scope string, next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + raw, found := strings.CutPrefix(r.Header.Get(agentclient.TokenHeader), "Bearer ") + claims := jwt.MapClaims{} + _, err := jwt.ParseWithClaims(raw, claims, func(*jwt.Token) (any, error) { return v.key, nil }, + jwt.WithValidMethods([]string{"RS256"}), jwt.WithAudience(v.target), jwt.WithIssuer(issuerRef), jwt.WithExpirationRequired()) + if !found || err != nil || claims["scope"] != scope { + w.WriteHeader(http.StatusUnauthorized) + return + } + v.mu.Lock() + v.grants = append(v.grants, claims["sub"].(string)+" "+scope) + v.mu.Unlock() + next(w, r) + } +} + +func (v *verifyingAgent) handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /hello", v.admit("jvm:read", func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{"apiVersion": agentclient.AgentAPIVersion, "protocol": 1, "started": "2026-01-02T03:04:05Z", "runtime": map[string]string{"lang": "java"}, + "ops": []map[string]any{{"name": "jvm.metrics", "mutates": false, "scope": "jvm:read"}}}) + })) + mux.HandleFunc("POST /call", v.admit("jvm:read", func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"heapUsed":42}`) + })) + return mux +} + +var _ = Describe("deployer store and target resolver", func() { + target := deploy.TargetID("example", namespace, "orders-0") + + // deployed is the attach deployer as `jvm deploy` stores it: deployed to + // one target whose listener the endpoint reaches. + deployed := func(endpoint string) deploy.Deployer { + deployer := attachDeployer() + deployer.Status = deploy.Status{Targets: []deploy.TargetStatus{{Target: target, Namespace: namespace, Pod: "orders-0", Container: jvm, State: deploy.StateDeployed, AgentAPIVersion: agentclient.AgentAPIVersion, Endpoint: endpoint}}} + return deployer + } + + It("keeps applied deployers with their status, by name", func() { + store := deploy.Store{Dir: GinkgoT().TempDir()} + Expect(store.Save(deployed("http://127.0.0.1:7070"))).To(Succeed()) + loaded, err := store.Load("example") + Expect(err).ToNot(HaveOccurred()) + Expect(loaded.Status).To(Equal(deployed("http://127.0.0.1:7070").Status)) + Expect(loaded.Spec.JavaContainer.Kind).To(Equal("attach")) + listed, err := store.List() + Expect(err).ToNot(HaveOccurred()) + Expect(listed).To(HaveLen(1)) + _, err = store.Load("missing") + Expect(err).To(MatchError(ContainSubstring(`deployer "missing" has not been deployed`))) + _, err = store.Load("../escape") + Expect(err).To(MatchError(ContainSubstring("invalid deployer name"))) + }) + + It("lists deployed target metadata for agent inventory and excludes failed targets", func() { + store := deploy.Store{Dir: GinkgoT().TempDir()} + value := deployed("http://127.0.0.1:7070") + value.Status.Targets = append(value.Status.Targets, deploy.TargetStatus{Target: deploy.TargetID("example", namespace, "orders-1"), + Namespace: namespace, Pod: "orders-1", Container: jvm, State: deploy.StateFailed, Error: "not deployed"}) + Expect(store.Save(value)).To(Succeed()) + known, err := store.Known() + Expect(err).NotTo(HaveOccurred()) + Expect(known).To(Equal([]agentclient.Known{{ID: target, Runtime: "java", Names: []string{"example", "orders-0", namespace + "/orders-0"}, + State: string(deploy.StateDeployed), Endpoint: "http://127.0.0.1:7070", Resource: map[string]string{deploy.AttributeDeployer: "example", + deploy.AttributeTarget: target, deploy.AttributeNamespace: namespace, deploy.AttributePod: "orders-0", deploy.AttributeContainer: jvm}}})) + }) + + It("connects a stored target on its first use, with tokens its issuer mints for it", func() { + key, err := rsa.GenerateKey(rand.Reader, 2048) + Expect(err).ToNot(HaveOccurred()) + agent := &verifyingAgent{key: &key.PublicKey, target: target} + server := httptest.NewServer(agent.handler()) + DeferCleanup(server.Close) + store := deploy.Store{Dir: GinkgoT().TempDir()} + Expect(store.Save(deployed(server.URL))).To(Succeed()) + signer, err := auth.NewIssuer(issuerRef, key) + Expect(err).ToNot(HaveOccurred()) + resolver := deploy.Resolver{Store: store, Subject: "operator@example", Context: func(context.Context) (dbcontext.Context, error) { return dbcontext.New(), nil }, + Issuer: func(_ dbcontext.Context, spec deploy.AuthSpec) (*auth.Issuer, error) { + Expect(spec.Issuer).To(Equal(issuerRef)) + return signer, nil + }} + registry := agentclient.NewRegistry() + registry.SetResolver(deploy.TargetPrefix, resolver.Resolve) + + claimed, err := registry.Resolve(context.Background(), target) + Expect(claimed, err).To(BeTrue()) + connected := registry.ByID(target) + Expect(connected.Hello.Resource).To(HaveKeyWithValue(deploy.AttributeDeployer, "example")) + value, err := connected.Call(context.Background(), agentclient.CallRequest{Op: "jvm.metrics", Params: json.RawMessage(`{}`)}) + Expect(err).ToNot(HaveOccurred()) + Expect(value).To(MatchJSON(`{"heapUsed":42}`)) + Expect(agent.grants).To(Equal([]string{"operator@example jvm:read", "operator@example jvm:read"})) + + claimed, err = registry.Resolve(context.Background(), "pid/1234") + Expect(claimed, err).To(BeFalse()) + }) + + DescribeTable("refuses a target it cannot connect, naming what to do", + func(id, message string) { + store := deploy.Store{Dir: GinkgoT().TempDir()} + Expect(store.Save(deployed("http://127.0.0.1:1"))).To(Succeed()) + resolver := deploy.Resolver{Store: store, Subject: "operator@example", Context: func(context.Context) (dbcontext.Context, error) { return dbcontext.New(), nil }} + Expect(resolver.Resolve(context.Background(), agentclient.NewRegistry(), id)).To(MatchError(ContainSubstring(message))) + }, + Entry("a malformed id", "deployer/example/orders", "want deployer///"), + Entry("a deployer never deployed", deploy.TargetID("other", namespace, "orders-0"), "telltale jvm deploy"), + Entry("a pod the deployer did not deploy", deploy.TargetID("example", namespace, "orders-9"), "has no deployed target"), + ) +}) diff --git a/pkg/deploy/template.go b/pkg/deploy/template.go new file mode 100644 index 0000000..82a2c9f --- /dev/null +++ b/pkg/deploy/template.go @@ -0,0 +1,52 @@ +package deploy + +import ( + "errors" + "fmt" + "strings" + "text/template" +) + +// PodData is what a URL template sees of the selected pod. +type PodData struct { + Name string + Namespace string + Labels map[string]string + IP string +} + +// templateData binds .Pod, .Container and .Port "". +type templateData struct { + Pod PodData + Container string + target Target +} + +// Port is the container port named name, an error when there is none. +func (d templateData) Port(name string) (int32, error) { + for _, port := range d.target.Container.Ports { + if port.Name == name { + return port.ContainerPort, nil + } + } + return 0, fmt.Errorf("container %q of pod %s/%s has no port named %q", d.Container, d.Pod.Namespace, d.Pod.Name, name) +} + +// RenderURL renders a javaContainer URL template for target. A binding the +// target lacks is an error, never ``. +func RenderURL(text string, target Target) (string, error) { + if strings.TrimSpace(text) == "" { + return "", errors.New("url template is required") + } + parsed, err := template.New("url").Option("missingkey=error").Parse(text) + if err != nil { + return "", fmt.Errorf("parse url template %q: %w", text, err) + } + pod := target.Pod + data := templateData{Pod: PodData{Name: pod.Name, Namespace: pod.Namespace, Labels: pod.Labels, IP: pod.Status.PodIP}, Container: target.Container.Name, target: target} + var out strings.Builder + if err := parsed.Execute(&out, data); err != nil { + return "", fmt.Errorf("render url template %q for %s: %w", text, target.ID, err) + } + return out.String(), nil +} diff --git a/pkg/deploy/template_spec_test.go b/pkg/deploy/template_spec_test.go new file mode 100644 index 0000000..3beed2c --- /dev/null +++ b/pkg/deploy/template_spec_test.go @@ -0,0 +1,34 @@ +package deploy_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + + "github.com/flanksource/telltale/pkg/deploy" +) + +func target() deploy.Target { + running := pod("orders-0", map[string]string{"app": "orders"}, corev1.PodRunning, corev1.ContainerPort{Name: "http", ContainerPort: 8080}) + return deploy.Target{ID: "deployer/example/orders/orders-0", Pod: running, Container: running.Spec.Containers[1]} +} + +var _ = Describe("URL templates", func() { + It("renders the pod, its namespace, labels and a named container port", func() { + rendered, err := deploy.RenderURL(`proxy://{{.Pod.Name}}.{{.Pod.Namespace}}:{{.Port "http"}}/{{.Pod.Labels.app}}?kind=pod`, target()) + Expect(err).ToNot(HaveOccurred()) + Expect(rendered).To(Equal("proxy://orders-0.orders:8080/orders?kind=pod")) + }) + + DescribeTable("fails loud on a binding the target does not have, never rendering ", + func(template, message string) { + rendered, err := deploy.RenderURL(template, target()) + Expect(err).To(MatchError(ContainSubstring(message))) + Expect(rendered).To(BeEmpty()) + }, + Entry("a missing port", `http://{{.Pod.Name}}:{{.Port "admin"}}`, `container "app" of pod orders/orders-0 has no port named "admin"`), + Entry("a missing label", `http://{{.Pod.Labels.tier}}`, `map has no entry for key "tier"`), + Entry("an unknown field", `http://{{.Node}}`, `can't evaluate field Node`), + Entry("an empty template", ``, "url template is required"), + ) +}) diff --git a/pkg/deploy/tomcat.go b/pkg/deploy/tomcat.go new file mode 100644 index 0000000..372edd9 --- /dev/null +++ b/pkg/deploy/tomcat.go @@ -0,0 +1,274 @@ +package deploy + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + "github.com/flanksource/commons-db/connection" + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/models" + + "github.com/flanksource/telltale/pkg/agentclient" + javaagent "github.com/flanksource/telltale/pkg/agentinstall/java" +) + +// KindTomcat hot-deploys telltale-diag.war through the Tomcat manager. +const KindTomcat = "tomcat" + +// The paths a tomcat javaContainer takes when it names none. +const ( + DefaultManagerPath = "/manager" + DefaultContextPath = "/telltale-diag" +) + +// TomcatSpec is a tomcat javaContainer. Its embedded HTTPConnection is the +// manager login (a connection reference, basic auth, bearer, OAuth and TLS). +// +// url is a template rendered per target to the Tomcat base URL the deploy +// uses: the manager upload and the /health probe. It must reach Tomcat with +// its Authorization header intact, so it is a portforward:// or direct +// http(s) URL, never the apiserver proxy, which deletes Authorization from +// every request it proxies. +// +// executionUrl is the Tomcat base URL executions use, defaulting to url. It +// may be a proxy:// URL: executions carry their token in +// X-Telltale-Authorization, which the apiserver forwards. +type TomcatSpec struct { + Kind string `json:"kind"` + connection.HTTPConnection + ExecutionURL string `json:"executionUrl,omitempty"` + ManagerPath string `json:"managerPath,omitempty"` + ContextPath string `json:"contextPath,omitempty"` + RuntimeDir string `json:"runtimeDir"` + ExtensionsDir string `json:"extensionsDir,omitempty"` + AllowMutation bool `json:"allowMutation,omitempty"` +} + +func init() { + RegisterJavaContainer(KindTomcat, func(raw json.RawMessage) (JavaContainer, error) { + var spec TomcatSpec + if err := decodeStrict(raw, &spec); err != nil { + return nil, err + } + return newTomcat(spec) + }) +} + +type tomcat struct{ spec TomcatSpec } + +// apiServerProxyPath is an apiserver services/proxy or pods/proxy URL path. +var apiServerProxyPath = regexp.MustCompile(`/api/v1/namespaces/[^/]+/(pods|services)/[^/]+/proxy`) + +// refuseAPIServerProxy refuses a manager URL the apiserver would proxy. +func refuseAPIServerProxy(template string) error { + if !strings.HasPrefix(strings.TrimSpace(template), "proxy://") && !apiServerProxyPath.MatchString(template) { + return nil + } + return fmt.Errorf("url %q goes through the apiserver proxy, which deletes the Authorization header from every request it proxies, so the manager login never reaches Tomcat: deploy over portforward://.:?kind=pod (or a direct http(s) URL) and put the proxy:// URL in executionUrl", template) +} + +func newTomcat(spec TomcatSpec) (*tomcat, error) { + if spec.ManagerPath == "" { + spec.ManagerPath = DefaultManagerPath + } + if spec.ContextPath == "" { + spec.ContextPath = DefaultContextPath + } + login := spec.HTTPConnection + switch { + case spec.URL == "": + return nil, errors.New("url (the Tomcat base URL template the deploy uses) is required") + case !strings.HasPrefix(spec.ContextPath, "/") || spec.ContextPath == "/" || strings.HasSuffix(spec.ContextPath, "/"): + return nil, fmt.Errorf("contextPath %q must be a non-root path such as %s", spec.ContextPath, DefaultContextPath) + case !strings.HasPrefix(spec.ManagerPath, "/"): + return nil, fmt.Errorf("managerPath %q must start with /", spec.ManagerPath) + case !strings.HasPrefix(spec.RuntimeDir, "/"): + return nil, fmt.Errorf("runtimeDir %q must be an absolute directory in the JVM's container", spec.RuntimeDir) + case login.ConnectionName == "" && login.HTTPBasicAuth.IsEmpty() && login.Bearer.IsEmpty() && login.OAuth.IsEmpty(): + return nil, errors.New("a manager login is required: connection, username/password, bearer or oauth") + } + if err := refuseAPIServerProxy(spec.URL); err != nil { + return nil, err + } + return &tomcat{spec: spec}, nil +} + +// login is the hydrated manager connection; its URL is the template's, not +// the referenced connection's. +func (t *tomcat) login(ctx dbcontext.Context, namespace string) (connection.HTTPConnection, error) { + login := t.spec.HTTPConnection + login.URL = "" + hydrated, err := login.Hydrate(ctx, namespace) + if err != nil { + return connection.HTTPConnection{}, fmt.Errorf("hydrate manager login: %w", err) + } + return *hydrated, nil +} + +// Client reaches the WAR with the manager connection's TLS and no login: an +// execution authenticates with its own token. The connection transport adds +// the apiserver's credentials to an apiserver proxy URL. +func (t *tomcat) Client(ctx dbcontext.Context, namespace string) (*http.Client, error) { + login, err := t.login(ctx, namespace) + if err != nil { + return nil, err + } + return &http.Client{Transport: connection.HTTPConnection{TLS: login.TLS}.Transport(), Timeout: time.Minute}, nil +} + +func (t *tomcat) Deploy(ctx dbcontext.Context, target Target, provision Provision) TargetStatus { + endpoint, err := t.executionEndpoint(target) + if err != nil { + return failed("", err) + } + base, err := RenderURL(t.spec.URL, target) + if err != nil { + return failed(endpoint, err) + } + config, err := adapterConfig(target, provision, t.adapterSettings()) + if err != nil { + return failed(endpoint, err) + } + health, err := resolvePath(ctx, base, t.spec.ContextPath, target) + if err != nil { + return failed(endpoint, err) + } + client, err := t.Client(ctx, target.Pod.Namespace) + if err != nil { + return failed(endpoint, err) + } + if found, current := t.current(ctx, client, target, health, config); found { + return TargetStatus{State: StateDeployed, AgentAPIVersion: current, Endpoint: endpoint} + } + if err := t.upload(ctx, target, base, config); err != nil { + return failed(endpoint, err) + } + return confirm(ctx, client, target, endpoint, func() (string, error) { return resolvePath(ctx, base, t.spec.ContextPath, target) }, config["configDigest"]) +} + +func (t *tomcat) current(ctx context.Context, client *http.Client, target Target, health string, config map[string]string) (bool, int) { + found, report, _ := inspect(ctx, client, target, health, config["configDigest"]) + return found == current, report.APIVersion +} + +// executionEndpoint is the rendered, unresolved execution URL with the WAR's +// context path: what the status records and Connect resolves again. +func (t *tomcat) executionEndpoint(target Target) (string, error) { + template := t.spec.ExecutionURL + if template == "" { + template = t.spec.URL + } + rendered, err := RenderURL(template, target) + if err != nil { + return "", err + } + return joinPath(rendered, t.spec.ContextPath) +} + +func (t *tomcat) adapterSettings() map[string]string { + settings := map[string]string{"runtimeDir": t.spec.RuntimeDir, "allowMutation": strconv.FormatBool(t.spec.AllowMutation)} + if t.spec.ExtensionsDir != "" { + settings["extensionsDir"] = t.spec.ExtensionsDir + } + return settings +} + +// upload deploys the target's configured WAR with the manager text API, +// replacing any WAR at the context path. +func (t *tomcat) upload(ctx dbcontext.Context, target Target, base string, config map[string]string) error { + war, err := javaagent.War() + if err != nil { + return fmt.Errorf("telltale-diag.war is not embedded; run make build: %w", err) + } + if war, err = configureWAR(war, config); err != nil { + return err + } + login, err := t.login(ctx, target.Pod.Namespace) + if err != nil { + return err + } + manager, err := resolvePath(ctx, base, t.spec.ManagerPath+"/text/deploy", target) + if err != nil { + return err + } + deploy := manager + "?" + url.Values{"path": {t.spec.ContextPath}, "update": {"true"}}.Encode() + request, err := http.NewRequestWithContext(ctx, http.MethodPut, deploy, bytes.NewReader(war)) + if err != nil { + return err + } + request.Header.Set("Content-Type", "application/octet-stream") + response, err := (&http.Client{Transport: login.Transport(), Timeout: 5 * time.Minute}).Do(request) + if err != nil { + return fmt.Errorf("tomcat manager deploy to %s: %w", target.ID, err) + } + defer func() { _ = response.Body.Close() }() + body, _ := io.ReadAll(io.LimitReader(response.Body, 64<<10)) + reply := strings.TrimSpace(string(body)) + if response.StatusCode != http.StatusOK || !strings.HasPrefix(reply, "OK") { + return fmt.Errorf("tomcat manager deploy to %s: HTTP %d: %s", target.ID, response.StatusCode, reply) + } + return nil +} + +// joinPath appends path to raw's path, ahead of its query, so a workload URL +// such as portforward://pod.ns:8080?kind=pod keeps its query. +func joinPath(raw, path string) (string, error) { + parsed, err := url.Parse(raw) + if err != nil { + return "", fmt.Errorf("parse %q: %w", raw, err) + } + parsed.Path = strings.TrimRight(parsed.Path, "/") + path + return parsed.String(), nil +} + +// resolvePath is base with path appended, resolved to a URL this process +// can reach (a port-forward or an apiserver proxy URL). +func resolvePath(ctx dbcontext.Context, base, path string, target Target) (string, error) { + joined, err := joinPath(base, path) + if err != nil { + return "", err + } + resolved, err := ctx.ResolveConnectionURL(joined, models.ConnectionTypeHTTP, target.Pod.Namespace) + if err != nil { + return "", fmt.Errorf("resolve %s: %w", joined, err) + } + return resolved, nil +} + +// confirm reads the target's health after a deploy until its agent answers. +// Each attempt resolves the endpoint again: a port-forward opened before the +// agent listened is torn down by its first refused connection, and resolving +// opens a new one in its place. +func confirm(ctx context.Context, client *http.Client, target Target, endpoint string, resolve func() (string, error), digest string) TargetStatus { + deadline := time.Now().Add(30 * time.Second) + for { + found, health, err := absent, agentclient.Health{}, error(nil) + executions, resolveErr := resolve() + if resolveErr != nil { + err = resolveErr + } else { + found, health, err = inspect(ctx, client, target, executions, digest) + } + switch { + case found == current: + return TargetStatus{State: StateDeployed, AgentAPIVersion: health.APIVersion, Endpoint: endpoint} + case found == outdated: + return outdatedStatus(endpoint, health) + case found == stale: + return TargetStatus{State: StateFailed, AgentAPIVersion: health.APIVersion, Endpoint: endpoint, Error: fmt.Sprintf("agent reports config digest %q after deploy, want %q", health.ConfigDigest, digest)} + case time.Now().After(deadline) || ctx.Err() != nil: + return failed(endpoint, fmt.Errorf("agent did not become healthy after deploy: %w", errors.Join(err, ctx.Err()))) + } + time.Sleep(200 * time.Millisecond) + } +} diff --git a/pkg/deploy/tomcat_spec_test.go b/pkg/deploy/tomcat_spec_test.go new file mode 100644 index 0000000..ef6df54 --- /dev/null +++ b/pkg/deploy/tomcat_spec_test.go @@ -0,0 +1,222 @@ +package deploy_test + +import ( + "archive/zip" + "bytes" + "crypto/rand" + "crypto/rsa" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + + dbcontext "github.com/flanksource/commons-db/context" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/deploy" + "github.com/flanksource/telltale/pkg/deploy/auth" +) + +const ( + issuerRef = "connection://example/signing" + managerUser = "manager" + managerPassword = "example" +) + +// tomcatStub is the Tomcat manager text API plus the deployed WAR's /health: +// it records each deploy and answers health with the uploaded config digest +// and its JVM's agent API. +type tomcatStub struct { + mu sync.Mutex + apiVersion int + deploys int + config map[string]string +} + +func (t *tomcatStub) ServeHTTP(w http.ResponseWriter, r *http.Request) { + t.mu.Lock() + defer t.mu.Unlock() + switch { + case r.Method == http.MethodPut && r.URL.Path == "/manager/text/deploy": + t.deploy(w, r) + case r.Method == http.MethodGet && r.URL.Path == deploy.DefaultContextPath+"/health": + if t.config == nil { + http.NotFound(w, r) + return + } + status, code := "ok", http.StatusOK + if t.apiVersion != agentclient.AgentAPIVersion { + status, code = "outdated", http.StatusServiceUnavailable + } + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(map[string]any{"status": status, "apiVersion": t.apiVersion, "configDigest": t.config["configDigest"]}) + default: + http.NotFound(w, r) + } +} + +func (t *tomcatStub) deploy(w http.ResponseWriter, r *http.Request) { + user, password, ok := r.BasicAuth() + if !ok || user != managerUser || password != managerPassword { + w.WriteHeader(http.StatusUnauthorized) + return + } + Expect(r.URL.Query().Get("path")).To(Equal(deploy.DefaultContextPath)) + Expect(r.URL.Query().Get("update")).To(Equal("true")) + war, err := io.ReadAll(r.Body) + Expect(err).ToNot(HaveOccurred()) + t.config = warConfig(war) + t.deploys++ + _, _ = io.WriteString(w, "OK - Deployed application at context path ["+deploy.DefaultContextPath+"]\n") +} + +// warConfig is the properties the deployer embedded in the uploaded WAR. +func warConfig(war []byte) map[string]string { + reader, err := zip.NewReader(bytes.NewReader(war), int64(len(war))) + Expect(err).ToNot(HaveOccurred()) + file, err := reader.Open("WEB-INF/telltale-diag.properties") + Expect(err).ToNot(HaveOccurred()) + data, err := io.ReadAll(file) + Expect(err).ToNot(HaveOccurred()) + unescape := strings.NewReplacer(`\:`, ":", `\=`, "=", `\#`, "#", `\!`, "!", `\\`, `\`) + config := map[string]string{} + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + key, value, found := strings.Cut(line, "=") + Expect(found).To(BeTrue(), line) + config[key] = unescape.Replace(value) + } + return config +} + +func stubPod(name string, stub *tomcatStub) *corev1.Pod { + server := httptest.NewServer(stub) + DeferCleanup(server.Close) + _, port, err := net.SplitHostPort(server.Listener.Addr().String()) + Expect(err).ToNot(HaveOccurred()) + number, err := strconv.Atoi(port) + Expect(err).ToNot(HaveOccurred()) + return pod(name, map[string]string{"app": "orders"}, corev1.PodRunning, corev1.ContainerPort{Name: "http", ContainerPort: int32(number)}) +} + +func tomcatDeployer() deploy.Deployer { + deployer, err := deploy.Decode([]byte(fmt.Sprintf(` +apiVersion: telltale.flanksource.com/v1alpha1 +kind: Deployer +metadata: {name: example} +spec: + selector: {kind: Pod, namespace: %s, labels: {app: orders}, container: %s} + javaContainer: + kind: tomcat + url: 'http://127.0.0.1:{{.Port "http"}}' + username: {value: %s} + password: {value: %s} + runtimeDir: /opt/telltale/runtime + auth: {issuer: %s} +`, namespace, jvm, managerUser, managerPassword, issuerRef))) + Expect(err).ToNot(HaveOccurred()) + return *deployer +} + +func issuer() *auth.Issuer { + key, err := rsa.GenerateKey(rand.Reader, 2048) + Expect(err).ToNot(HaveOccurred()) + signer, err := auth.NewIssuer(issuerRef, key) + Expect(err).ToNot(HaveOccurred()) + return signer +} + +var _ = Describe("tomcat java containers", func() { + It("deploys a WAR provisioned for each target once, and leaves a current one alone", func() { + first, second := &tomcatStub{apiVersion: agentclient.AgentAPIVersion}, &tomcatStub{apiVersion: agentclient.AgentAPIVersion} + options := deploy.Options{Kubernetes: fake.NewClientset(stubPod("orders-0", first), stubPod("orders-1", second)), Issuer: issuer()} + status, err := deploy.Deploy(dbcontext.New(), tomcatDeployer(), options) + Expect(err).ToNot(HaveOccurred()) + Expect(status.Targets).To(HaveLen(2)) + for _, target := range status.Targets { + Expect(target).To(And(HaveField("State", deploy.StateDeployed), HaveField("AgentAPIVersion", agentclient.AgentAPIVersion), HaveField("Endpoint", HaveSuffix(deploy.DefaultContextPath)))) + } + jwks, err := json.Marshal(options.Issuer.JWKS()) + Expect(err).ToNot(HaveOccurred()) + Expect(first.config).To(And(HaveKeyWithValue("audience", deploy.TargetID("example", namespace, "orders-0")), HaveKeyWithValue("issuer", issuerRef), + HaveKeyWithValue("jwks", string(jwks)), HaveKeyWithValue("runtimeDir", "/opt/telltale/runtime"), HaveKeyWithValue("allowMutation", "false"))) + Expect(second.config).To(HaveKeyWithValue("audience", deploy.TargetID("example", namespace, "orders-1"))) + Expect(first.config).ToNot(HaveKey(ContainSubstring("token")), "no static token is provisioned") + + again, err := deploy.Deploy(dbcontext.New(), tomcatDeployer(), options) + Expect(err).ToNot(HaveOccurred()) + Expect(again).To(Equal(status)) + Expect([]int{first.deploys, second.deploys}).To(Equal([]int{1, 1}), "a current target is not redeployed") + }) + + It("deploys over url and records the executionUrl, with the context path ahead of its query, as the endpoint", func() { + stub := &tomcatStub{apiVersion: agentclient.AgentAPIVersion} + deployer := tomcatDeployer() + spec, err := deploy.NewJavaContainerSpec(map[string]any{"kind": "tomcat", "url": `http://127.0.0.1:{{.Port "http"}}`, "executionUrl": `proxy://{{.Pod.Name}}.{{.Pod.Namespace}}:{{.Port "http"}}?kind=pod`, + "username": map[string]string{"value": managerUser}, "password": map[string]string{"value": managerPassword}, "runtimeDir": "/opt/telltale/runtime"}) + Expect(err).ToNot(HaveOccurred()) + deployer.Spec.JavaContainer = spec + running := stubPod("orders-0", stub) + status, err := deploy.Deploy(dbcontext.New(), deployer, deploy.Options{Kubernetes: fake.NewClientset(running), Issuer: issuer()}) + Expect(err).ToNot(HaveOccurred()) + port := running.Spec.Containers[1].Ports[0].ContainerPort + Expect(status.Targets).To(ConsistOf(And(HaveField("State", deploy.StateDeployed), HaveField("Endpoint", fmt.Sprintf("proxy://orders-0.orders:%d%s?kind=pod", port, deploy.DefaultContextPath))))) + Expect(stub.deploys).To(Equal(1)) + }) + + It("redeploys a target provisioned with another key", func() { + stub := &tomcatStub{apiVersion: agentclient.AgentAPIVersion} + kubernetes := fake.NewClientset(stubPod("orders-0", stub)) + _, err := deploy.Deploy(dbcontext.New(), tomcatDeployer(), deploy.Options{Kubernetes: kubernetes, Issuer: issuer()}) + Expect(err).ToNot(HaveOccurred()) + _, err = deploy.Deploy(dbcontext.New(), tomcatDeployer(), deploy.Options{Kubernetes: kubernetes, Issuer: issuer()}) + Expect(err).ToNot(HaveOccurred()) + Expect(stub.deploys).To(Equal(2)) + }) + + It("reports outdated when the JVM keeps another agent API after a redeploy", func() { + stub := &tomcatStub{apiVersion: agentclient.AgentAPIVersion - 1} + status, err := deploy.Deploy(dbcontext.New(), tomcatDeployer(), deploy.Options{Kubernetes: fake.NewClientset(stubPod("orders-0", stub)), Issuer: issuer()}) + Expect(err).To(MatchError(ContainSubstring("outdated"))) + Expect(status.Targets).To(ConsistOf(And(HaveField("State", deploy.StateOutdated), HaveField("AgentAPIVersion", agentclient.AgentAPIVersion-1), HaveField("Error", ContainSubstring("restart the JVM"))))) + }) + + It("fails a target whose manager refuses the login", func() { + deployer := tomcatDeployer() + spec, err := deploy.NewJavaContainerSpec(map[string]any{"kind": "tomcat", "url": `http://127.0.0.1:{{.Port "http"}}`, "username": map[string]string{"value": managerUser}, "password": map[string]string{"value": "wrong"}, "runtimeDir": "/opt/telltale/runtime"}) + Expect(err).ToNot(HaveOccurred()) + deployer.Spec.JavaContainer = spec + status, err := deploy.Deploy(dbcontext.New(), deployer, deploy.Options{Kubernetes: fake.NewClientset(stubPod("orders-0", &tomcatStub{apiVersion: agentclient.AgentAPIVersion})), Issuer: issuer()}) + Expect(err).To(HaveOccurred()) + Expect(status.Targets).To(ConsistOf(And(HaveField("State", deploy.StateFailed), HaveField("Error", ContainSubstring("HTTP 401"))))) + }) + + DescribeTable("refuses a javaContainer it cannot run", + func(spec map[string]any, message string) { + deployer := tomcatDeployer() + container, err := deploy.NewJavaContainerSpec(spec) + Expect(err).ToNot(HaveOccurred()) + deployer.Spec.JavaContainer = container + _, err = deploy.Deploy(dbcontext.New(), deployer, deploy.Options{Kubernetes: fake.NewClientset(), Issuer: issuer()}) + Expect(err).To(MatchError(ContainSubstring(message))) + }, + Entry("an unregistered kind", map[string]any{"kind": "jetty"}, `javaContainer kind "jetty" is not registered (registered: attach, javaagent, tomcat)`), + Entry("javaagent, not implemented yet", map[string]any{"kind": "javaagent"}, "not implemented"), + Entry("a tomcat without a manager login", map[string]any{"kind": "tomcat", "url": "http://example", "runtimeDir": "/opt/telltale"}, "manager login is required"), + Entry("a tomcat field it does not know", map[string]any{"kind": "tomcat", "url": "http://example", "runtimeDir": "/opt/telltale", "token": "x"}, `unknown field "token"`), + Entry("a manager url through the apiserver proxy, which strips Authorization", + map[string]any{"kind": "tomcat", "url": `proxy://{{.Pod.Name}}.{{.Pod.Namespace}}:8080?kind=pod`, "username": map[string]string{"value": managerUser}, "runtimeDir": "/opt/telltale"}, + "deletes the Authorization header"), + Entry("a literal apiserver proxy manager url, refused naming portforward", + map[string]any{"kind": "tomcat", "url": "https://apiserver.example/api/v1/namespaces/orders/pods/orders-0:8080/proxy", "username": map[string]string{"value": managerUser}, "runtimeDir": "/opt/telltale"}, + "deploy over portforward://"), + ) +}) diff --git a/pkg/deploy/war.go b/pkg/deploy/war.go new file mode 100644 index 0000000..d1ba3e1 --- /dev/null +++ b/pkg/deploy/war.go @@ -0,0 +1,67 @@ +package deploy + +import ( + "archive/zip" + "bytes" + "fmt" + "io" + "maps" + "slices" + "strings" +) + +// warConfigEntry is where telltale-diag.war reads an embedded configuration. +const warConfigEntry = "WEB-INF/telltale-diag.properties" + +// configureWAR is war with config embedded as its properties file, so a +// manager upload needs no file on the target's filesystem. +func configureWAR(war []byte, config map[string]string) ([]byte, error) { + reader, err := zip.NewReader(bytes.NewReader(war), int64(len(war))) + if err != nil { + return nil, fmt.Errorf("read telltale-diag.war: %w", err) + } + var out bytes.Buffer + writer := zip.NewWriter(&out) + for _, file := range reader.File { + if file.Name == warConfigEntry { + return nil, fmt.Errorf("telltale-diag.war already carries %s", warConfigEntry) + } + if err := copyEntry(writer, file); err != nil { + return nil, fmt.Errorf("copy %s: %w", file.Name, err) + } + } + entry, err := writer.Create(warConfigEntry) + if err != nil { + return nil, err + } + if _, err := io.WriteString(entry, properties(config)); err != nil { + return nil, err + } + if err := writer.Close(); err != nil { + return nil, err + } + return out.Bytes(), nil +} + +// copyEntry copies a file's compressed bytes as they are; a directory entry, +// which carries none, is re-created from its header. +func copyEntry(writer *zip.Writer, file *zip.File) error { + if !file.FileInfo().IsDir() { + return writer.Copy(file) + } + header := file.FileHeader + header.Method = zip.Store + header.CompressedSize64, header.UncompressedSize64, header.CRC32 = 0, 0, 0 + _, err := writer.CreateHeader(&header) + return err +} + +// properties is config as a java.util.Properties file, keys sorted. +func properties(config map[string]string) string { + escape := strings.NewReplacer(`\`, `\\`, "\n", `\n`, "\r", `\r`, "\t", `\t`, "=", `\=`, ":", `\:`, "#", `\#`, "!", `\!`) + var out strings.Builder + for _, key := range slices.Sorted(maps.Keys(config)) { + out.WriteString(escape.Replace(key) + "=" + escape.Replace(config[key]) + "\n") + } + return out.String() +} diff --git a/pkg/deploy/war_e2e_internal_test.go b/pkg/deploy/war_e2e_internal_test.go new file mode 100644 index 0000000..c13d436 --- /dev/null +++ b/pkg/deploy/war_e2e_internal_test.go @@ -0,0 +1,285 @@ +package deploy + +import ( + "bufio" + "context" + "crypto/rand" + "crypto/rsa" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + dutyKubernetes "github.com/flanksource/commons-db/kubernetes" + "github.com/golang-jwt/jwt/v5" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + + "github.com/flanksource/telltale/pkg/agentclient" + javaagent "github.com/flanksource/telltale/pkg/agentinstall/java" + "github.com/flanksource/telltale/pkg/deploy/auth" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/jvmtrace" + "github.com/flanksource/telltale/pkg/tracing/tracingtest" +) + +// harnessJar is the embedded Tomcat that `gradlew tomcatHarnessJar` builds. +var harnessJar = filepath.Join("..", "..", "sdk", "java", "build", "libs", "telltale-tomcat-harness.jar") + +const harnessWorkload = "com.flanksource.telltale.harness.HarnessWorkload" + +func signingKey() *rsa.PrivateKey { + key, err := rsa.GenerateKey(rand.Reader, 2048) + Expect(err).ToNot(HaveOccurred()) + return key +} + +// startHarness runs telltale-diag.war, provisioned for target, in a real +// Tomcat JVM and returns its execution URL. +func startHarness(target Target, provision Provision) string { + _, err := os.Stat(harnessJar) + Expect(err).ToNot(HaveOccurred(), "build the harness: ./sdk/java/gradlew -p sdk/java war tomcatHarnessJar") + directory := GinkgoT().TempDir() + container := &tomcat{spec: TomcatSpec{ContextPath: DefaultContextPath, RuntimeDir: filepath.Join(directory, "runtime"), AllowMutation: true}} + config, err := adapterConfig(target, provision, container.adapterSettings()) + Expect(err).ToNot(HaveOccurred()) + war, err := javaagent.War() + Expect(err).ToNot(HaveOccurred(), "build the embedded WAR: ./sdk/java/gradlew -p sdk/java war") + war, err = configureWAR(war, config) + Expect(err).ToNot(HaveOccurred()) + path := filepath.Join(directory, "telltale-diag.war") + Expect(os.WriteFile(path, war, 0o600)).To(Succeed()) + harness := exec.Command("java", "-jar", harnessJar, path, DefaultContextPath) + stdin, err := harness.StdinPipe() + Expect(err).ToNot(HaveOccurred()) + stdout, err := harness.StdoutPipe() + Expect(err).ToNot(HaveOccurred()) + harness.Stderr = GinkgoWriter + Expect(harness.Start()).To(Succeed()) + DeferCleanup(func() { _ = stdin.Close(); _ = harness.Wait() }) + ready := make(chan string, 1) + go func() { + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + if port, found := strings.CutPrefix(scanner.Text(), "READY "); found { + ready <- port + } + } + }() + var port string + Eventually(ready, time.Minute).Should(Receive(&port)) + endpoint := "http://127.0.0.1:" + port + DefaultContextPath + found, health, err := inspect(context.Background(), listenerClient(), target, endpoint, config["configDigest"]) + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(Equal(current), "the WAR echoes the provisioned digest: %+v", health) + return endpoint +} + +func harnessTarget() Target { + return Target{ID: TargetID("e2e", "local", "harness"), Pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "harness", Namespace: "local"}}, Container: corev1.Container{Name: "tomcat"}} +} + +const apiServerIdentity = "apiserver-identity" + +// apiServerProxy plays the apiserver's pods/proxy: it authenticates its own +// identity, then deletes Authorization (as k8s.io/apiserver's authentication +// filter does) and forwards everything else to the pod. +type apiServerProxy struct { + base string + mu sync.Mutex + // forwarded counts requests that authenticated as the apiserver identity + // and carried an execution token in X-Telltale-Authorization. + forwarded int +} + +func (a *apiServerProxy) forwardedTokens() int { + a.mu.Lock() + defer a.mu.Unlock() + return a.forwarded +} + +func fakeAPIServer(backend string) *apiServerProxy { + target, err := url.Parse(backend) + Expect(err).ToNot(HaveOccurred()) + const prefix = "/api/v1/namespaces/local/pods/harness:8080/proxy" + proxy := &apiServerProxy{} + forward := httputil.NewSingleHostReverseProxy(target) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer "+apiServerIdentity { + w.WriteHeader(http.StatusUnauthorized) + _, _ = io.WriteString(w, `{"kind":"Status","code":401,"message":"Unauthorized"}`) + return + } + if !strings.HasPrefix(r.URL.Path, prefix) { + http.NotFound(w, r) + return + } + proxy.mu.Lock() + if r.Header.Get(agentclient.TokenHeader) != "" { + proxy.forwarded++ + } + proxy.mu.Unlock() + r.Header.Del("Authorization") + r.URL.Path = strings.TrimPrefix(r.URL.Path, prefix) + forward.ServeHTTP(w, r) + })) + DeferCleanup(server.Close) + proxy.base = server.URL + prefix + Expect(dutyKubernetes.RegisterAPIServerProxy(proxy.base, &rest.Config{Host: server.URL, BearerToken: apiServerIdentity})).To(Succeed()) + return proxy +} + +// catalogStatus is the HTTP status of GET /catalog with bearer token. +func catalogStatus(endpoint, token string) int { + request, err := http.NewRequest(http.MethodGet, endpoint+"/catalog", nil) + Expect(err).ToNot(HaveOccurred()) + if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + response, err := http.DefaultClient.Do(request) + Expect(err).ToNot(HaveOccurred()) + Expect(response.Body.Close()).To(Succeed()) + return response.StatusCode +} + +// forged signs claims with key under kid, as no honest issuer would. +func forged(key *rsa.PrivateKey, kid string, claims jwt.MapClaims) string { + token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims) + token.Header["kid"] = kid + signed, err := token.SignedString(key) + Expect(err).ToNot(HaveOccurred()) + return signed +} + +var _ = Describe("telltale-diag.war over direct HTTP", Label("jvm"), func() { + It("verifies every execution token and picks up a rotated key from the issuer's JWKS", func() { + key := signingKey() + issuer, err := auth.NewIssuer("e2e-issuer", key) + Expect(err).ToNot(HaveOccurred()) + jwks := httptest.NewServer(issuer) + DeferCleanup(jwks.Close) + target := harnessTarget() + endpoint := startHarness(target, Provision{Issuer: issuer.Name(), JWKS: issuer.JWKS(), JWKSURL: jwks.URL}) + kid := issuer.JWKS().Keys[0].KeyID + valid := jwt.MapClaims{"iss": issuer.Name(), "aud": target.ID, "sub": "e2e", "scope": "jvm:read", "exp": time.Now().Add(time.Minute).Unix()} + with := func(key string, value any) jwt.MapClaims { + claims := jwt.MapClaims{} + for name, v := range valid { + claims[name] = v + } + claims[key] = value + return claims + } + Expect(catalogStatus(endpoint, "")).To(Equal(http.StatusUnauthorized), "a missing token") + Expect(catalogStatus(endpoint, forged(signingKey(), kid, valid))).To(Equal(http.StatusUnauthorized), "a bad signature") + Expect(catalogStatus(endpoint, forged(key, kid, with("aud", TargetID("e2e", "local", "other"))))).To(Equal(http.StatusUnauthorized), "another target's token") + Expect(catalogStatus(endpoint, forged(key, kid, with("exp", time.Now().Add(-5*time.Minute).Unix())))).To(Equal(http.StatusUnauthorized), "an expired token") + Expect(catalogStatus(endpoint, forged(key, kid, with("scope", "jvm:trace")))).To(Equal(http.StatusForbidden), "an insufficient scope") + Expect(catalogStatus(endpoint, forged(key, kid, valid))).To(Equal(http.StatusOK), "a correctly scoped token") + + Expect(issuer.Rotate(signingKey())).To(Succeed()) + rotated, err := issuer.Mint(auth.Claims{Audience: target.ID, Subject: "e2e", Scopes: []auth.Scope{auth.ScopeRead}}, time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(catalogStatus(endpoint, rotated)).To(Equal(http.StatusOK), "the WAR refreshes its JWKS on the new kid") + }) + + It("executes through an apiserver proxy that deletes Authorization, with the token in X-Telltale-Authorization", func() { + issuer, err := auth.NewIssuer("e2e-issuer", signingKey()) + Expect(err).ToNot(HaveOccurred()) + target := harnessTarget() + endpoint := startHarness(target, Provision{Issuer: issuer.Name(), JWKS: issuer.JWKS()}) + apiserver := fakeAPIServer(strings.TrimSuffix(endpoint, DefaultContextPath)) + proxied := apiserver.base + DefaultContextPath + client, err := (&tomcat{spec: TomcatSpec{ContextPath: DefaultContextPath}}).Client(dbcontext.New(), "local") + Expect(err).ToNot(HaveOccurred()) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + agent, err := agentclient.DialHTTP(ctx, agentclient.HTTPEndpoint{ID: target.ID, URL: proxied, Client: client, Tokens: auth.Tokens{Issuer: issuer, Subject: "e2e", TTL: time.Minute}}) + Expect(err).ToNot(HaveOccurred(), "the client's transport carries the apiserver identity, the token rides X-Telltale-Authorization") + DeferCleanup(agent.Close) + value, err := agent.Call(ctx, agentclient.CallRequest{Op: "jvm.jmx.search", Params: json.RawMessage(`{"pattern":"java.lang:type=Runtime"}`)}) + Expect(err).ToNot(HaveOccurred()) + Expect(value).To(MatchJSON(`["java.lang:type=Runtime"]`)) + Expect(apiserver.forwardedTokens()).To(BeNumerically(">=", 2), "hello and the call reached the WAR through the proxy with the token in X-Telltale-Authorization") + + token, err := issuer.Mint(auth.Claims{Audience: target.ID, Subject: "e2e", Scopes: []auth.Scope{auth.ScopeRead}}, time.Minute) + Expect(err).ToNot(HaveOccurred()) + request, err := http.NewRequest(http.MethodGet, proxied+"/catalog", nil) + Expect(err).ToNot(HaveOccurred()) + request.Header.Set("Authorization", "Bearer "+token) + response, err := client.Do(request) + Expect(err).ToNot(HaveOccurred()) + Expect(response.Body.Close()).To(Succeed()) + Expect(response.StatusCode).To(Equal(http.StatusUnauthorized), "a bearer in Authorization alone is the apiserver's to judge and never reaches the WAR") + Expect(catalogStatus(endpoint, token)).To(Equal(http.StatusOK), "a direct connection may still use Authorization") + }) + + It("runs a jvm_trace capture end to end against a deployed target, with no hub", func() { + issuer, err := auth.NewIssuer("e2e-issuer", signingKey()) + Expect(err).ToNot(HaveOccurred()) + target := harnessTarget() + endpoint := startHarness(target, Provision{Issuer: issuer.Name(), JWKS: issuer.JWKS()}) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + agent, err := agentclient.DialHTTP(ctx, agentclient.HTTPEndpoint{ID: target.ID, URL: endpoint, Client: listenerClient(), Tokens: auth.Tokens{Issuer: issuer, Subject: "e2e", TTL: time.Minute}, + Resource: map[string]string{AttributeDeployer: "e2e", AttributePod: "harness"}}) + Expect(err).ToNot(HaveOccurred()) + registry := agentclient.NewRegistry() + Expect(registry.Add(agent)).To(Succeed()) + DeferCleanup(agent.Close) + + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{jvmtrace.Kind: jvmtrace.NewKind(registry)}) + Expect(registry.Match(map[string]string{AttributeDeployer: "e2e", AttributePod: "harness"})).To(ConsistOf(agent), "a capture selects the deployed target by deployer and pod") + // A selector capture is a fleet capture that waits for late joiners for + // its whole duration; naming the one target ends with its capture. + params, err := json.Marshal(map[string]any{"target": map[string]any{"seam": "agent", "agent": target.ID, "allowMutation": true}, + "targets": []map[string]string{{"class": harnessWorkload, "method": "compute"}}, "args": true, "return": true, "events": 4}) + Expect(err).ToNot(HaveOccurred()) + managed, err := env.Runtime.Start(dbcontext.New().Wrap(ctx), tracing.StartRequest{Kind: jvmtrace.Kind, Params: params}) + Expect(err).ToNot(HaveOccurred()) + rows := env.Rows(tracingtest.Ended(managed.Session())) + Expect(rows).ToNot(BeEmpty()) + Expect(rows).To(ContainElement(And(HaveKeyWithValue("class", harnessWorkload), HaveKeyWithValue("method", "compute")))) + patches, err := agent.Call(ctx, agentclient.CallRequest{Op: "jvm.classes.patches", Params: json.RawMessage(`{}`)}) + Expect(err).ToNot(HaveOccurred()) + Expect(patches).To(MatchJSON(`[]`), "the capture restored the traced class") + }) + + It("reports how many root calls a minCost filter withheld over direct HTTP", func() { + issuer, err := auth.NewIssuer("e2e-issuer", signingKey()) + Expect(err).ToNot(HaveOccurred()) + target := harnessTarget() + endpoint := startHarness(target, Provision{Issuer: issuer.Name(), JWKS: issuer.JWKS()}) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + agent, err := agentclient.DialHTTP(ctx, agentclient.HTTPEndpoint{ID: target.ID, URL: endpoint, Client: listenerClient(), Tokens: auth.Tokens{Issuer: issuer, Subject: "e2e", TTL: time.Minute}}) + Expect(err).ToNot(HaveOccurred()) + registry := agentclient.NewRegistry() + Expect(registry.Add(agent)).To(Succeed()) + DeferCleanup(agent.Close) + + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{jvmtrace.Kind: jvmtrace.NewKind(registry)}) + // No Math.sqrt call runs for an hour, so the filter withholds every root + // call the harness makes and the capture ends at its deadline. + params, err := json.Marshal(map[string]any{"target": map[string]any{"seam": "agent", "agent": target.ID, "allowMutation": true}, + "targets": []map[string]string{{"class": harnessWorkload, "method": "compute"}}, "minCost": "1h"}) + Expect(err).ToNot(HaveOccurred()) + stopAt := time.Now().Add(3 * time.Second) + managed, err := env.Runtime.Start(dbcontext.New().Wrap(ctx), tracing.StartRequest{Kind: jvmtrace.Kind, Params: params, StopAt: &stopAt}) + Expect(err).ToNot(HaveOccurred()) + report := tracingtest.Summary(tracingtest.Ended(managed.Session())).Report + Expect(report).To(HaveKeyWithValue("filtered", BeNumerically(">", 0)), "the WAR's X-Telltale-Report reached the capture report: %v", report) + }) +}) diff --git a/pkg/otlp/profilepublication/identity.go b/pkg/otlp/profilepublication/identity.go new file mode 100644 index 0000000..c63b97a --- /dev/null +++ b/pkg/otlp/profilepublication/identity.go @@ -0,0 +1,65 @@ +package profilepublication + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "strconv" + "strings" + + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/google/uuid" +) + +const IdentityKind = "otel_profile_identity" +const IdentityStream = "profile-identities" + +type Identity struct { + Key string `json:"key" filter:"exact"` + Digest string `json:"digest"` + ProfileID string `json:"profileId"` + Kind string `json:"kind"` +} + +func ResultType() recordresults.ResultType[Identity] { + return recordresults.ResultType[Identity]{Kind: IdentityKind, Title: "Received profile identities", KeyColumn: "key", Retention: recordstore.RetainRows} +} + +func IsProfile(frame sampleframe.Frame) bool { + _, id := frame.Summary["profileId"] + _, version := frame.Summary["profilesDevelopmentVersion"] + return id || version +} + +func identityKeys(kind, source, captureID, profileID string, sequence int64) []string { + var tuple []byte + for _, value := range []string{kind, source, captureID} { + tuple = strconv.AppendInt(tuple, int64(len(value)), 10) + tuple = append(tuple, ':') + tuple = append(tuple, value...) + } + tuple = strconv.AppendInt(tuple, sequence, 10) + digest := sha256.Sum256(tuple) + return []string{"profile/" + profileID, "sequence/" + hex.EncodeToString(digest[:])} +} + +func frameKeys(kind string, frame sampleframe.Frame) ([]string, string, string, error) { + id, _ := frame.Summary["profileId"].(string) + capture, _ := frame.Summary["captureId"].(string) + digest, _ := frame.Summary["digest"].(string) + parsed, err := uuid.Parse(id) + if err != nil || parsed == uuid.Nil || capture == "" || frame.Tick <= 0 { + return nil, "", "", fmt.Errorf("profile publication requires a nonzero UUID, capture and positive producer sequence") + } + decoded, err := hex.DecodeString(digest) + if err != nil || len(decoded) != sha256.Size { + return nil, "", "", fmt.Errorf("profile publication requires a SHA-256 semantic digest") + } + source, found := strings.CutSuffix(frame.Source, "/"+capture) + if !found || source == "" { + return nil, "", "", fmt.Errorf("profile frame source must include its producer and capture identity") + } + return identityKeys(kind, source, capture, id, frame.Tick), digest, id, nil +} diff --git a/pkg/otlp/profilepublication/identity_test.go b/pkg/otlp/profilepublication/identity_test.go new file mode 100644 index 0000000..a9fe70b --- /dev/null +++ b/pkg/otlp/profilepublication/identity_test.go @@ -0,0 +1,31 @@ +package profilepublication + +import ( + "strings" + "testing" + + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestIdentity(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Profile publication identity") +} + +var _ = Describe("profile publication identity", func() { + It("rejects an all-zero profile UUID before journal lookup", func() { + frame := sampleframe.Frame{Source: "java/capture", Tick: 1, Summary: map[string]any{"profileId": uuid.Nil.String(), "captureId": "capture", "digest": strings.Repeat("a", 64)}} + _, _, _, err := frameKeys("http_top", frame) + Expect(err).To(MatchError(ContainSubstring("nonzero UUID"))) + }) + + It("keeps source and capture tuple boundaries distinct when their strings contain NUL", func() { + id := "00000000-0000-4000-8000-000000000001" + first := identityKeys("http_top", "a\x00b", "c", id, 1) + second := identityKeys("http_top", "a", "b\x00c", id, 1) + Expect(first[1]).NotTo(Equal(second[1])) + }) +}) diff --git a/pkg/otlp/profilepublication/publication.go b/pkg/otlp/profilepublication/publication.go new file mode 100644 index 0000000..450398e --- /dev/null +++ b/pkg/otlp/profilepublication/publication.go @@ -0,0 +1,138 @@ +// Package profilepublication atomically publishes profile rows, frames and durable identities. +package profilepublication + +import ( + "context" + "errors" + "fmt" + "sync" + + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/google/uuid" +) + +var ErrConflict = errors.New("conflicting content for a profile ID or source sequence") + +// The record store elects one writer process; this lock serializes its profile identity checks and appends. +var publicationMu sync.Mutex + +type Backend interface { + sampleframe.Backend + Scan(context.Context, string, int64, func(int64, recordstore.Row) error) error +} + +type Publication struct { + Kind string + Stream string + FrameStream string + Schema recordstore.KindSchema + FrameSchema recordstore.KindSchema + Frame sampleframe.Frame + Rows []recordstore.Row + AdditionalSchemas []recordstore.KindSchema + AdditionalEntries []recordstore.BatchEntry +} + +func Append(ctx context.Context, backend Backend, publication Publication) (bool, error) { + if backend == nil || publication.Kind == "" || publication.Schema.Options.Key != "" { + return false, fmt.Errorf("profile publication requires an atomic backend, kind and unkeyed row stream") + } + frame := publication.Frame + if frame.Source == "" || frame.StartedAt.IsZero() || frame.CompletedAt.Before(frame.StartedAt) || frame.Observed < len(publication.Rows) || frame.Limit < 0 || (frame.Limit > 0 && len(publication.Rows) > frame.Limit) { + return false, fmt.Errorf("profile publication requires ordered timestamps and observed count within its limit") + } + keys, digest, profileID, err := frameKeys(publication.Kind, frame) + if err != nil { + return false, err + } + publicationMu.Lock() + defer publicationMu.Unlock() + duplicate, err := duplicateProfile(ctx, backend, keys, digest) + if err != nil || duplicate { + return false, err + } + meta, err := backend.Meta(ctx, publication.Stream) + if err != nil && !errors.Is(err, recordstore.ErrNotFound) { + return false, err + } + frame.ID, frame.AfterSeq, frame.ToSeq, frame.Retained = profileID, meta.HighSeq, meta.HighSeq+int64(len(publication.Rows)), len(publication.Rows) + publication.Frame = frame + return appendProfile(ctx, backend, publication, keys, digest) +} + +func duplicateProfile(ctx context.Context, backend Backend, keys []string, digest string) (bool, error) { + found := map[string]bool{} + err := backend.Scan(ctx, IdentityStream, 0, func(_ int64, row recordstore.Row) error { + key, ok := row["key"].(string) + if !ok { + return fmt.Errorf("profile identity has no key") + } + previous, ok := row["digest"].(string) + if !ok { + return fmt.Errorf("profile identity has no digest") + } + if key != keys[0] && key != keys[1] { + return nil + } + if previous != digest { + return ErrConflict + } + found[key] = true + return nil + }) + if errors.Is(err, recordstore.ErrNotFound) { + err = nil + } + if err == nil && len(found) == 1 { + return false, ErrConflict + } + return len(found) == 2, err +} + +func appendProfile(ctx context.Context, backend Backend, publication Publication, keys []string, digest string) (bool, error) { + encoded, err := recordstore.EncodeRow(publication.Frame) + if err != nil { + return false, err + } + schema, err := ResultType().KindSchema() + if err != nil { + return false, err + } + identities := make([]recordstore.Row, len(keys)) + for index, key := range keys { + identities[index], err = recordstore.EncodeRow(Identity{Key: key, Digest: digest, ProfileID: publication.Frame.ID, Kind: publication.Kind}) + if err != nil { + return false, err + } + } + entries := append([]recordstore.BatchEntry{ + {Op: recordstore.BatchAppend, Stream: publication.Stream, Kind: publication.Schema.Kind, Rows: publication.Rows}, + {Op: recordstore.BatchAppend, Stream: publication.FrameStream, Kind: publication.FrameSchema.Kind, Rows: []recordstore.Row{encoded}}, + {Op: recordstore.BatchAppend, Stream: IdentityStream, Kind: IdentityKind, Rows: identities}, + }, publication.AdditionalEntries...) + schemas := append([]recordstore.KindSchema{publication.Schema, publication.FrameSchema, schema}, publication.AdditionalSchemas...) + result, err := backend.AppendBatch(ctx, recordstore.Batch{ID: uuid.NewString(), Atomic: true, Producer: recordstore.Producer{Instance: publication.Frame.Source + "/" + publication.Kind, Seq: publication.Frame.Tick}, Schemas: schemas, Entries: entries}) + if err != nil { + return false, err + } + if err := validateResult(result, len(entries), publication.Frame); err != nil { + return false, err + } + return true, nil +} + +func validateResult(result recordstore.BatchResult, expected int, frame sampleframe.Frame) error { + if len(result.Entries) != expected { + return fmt.Errorf("profile atomic batch expected %d entries, received %d", expected, len(result.Entries)) + } + for _, entry := range result.Entries { + if entry.Error != nil { + return entry.Error + } + } + if result.Entries[0].Append == nil || result.Entries[0].Append.Window != (recordstore.Window{From: frame.AfterSeq + 1, To: frame.ToSeq}) { + return fmt.Errorf("profile row stream has another writer") + } + return nil +} diff --git a/pkg/otlp/profiles/canonical.go b/pkg/otlp/profiles/canonical.go new file mode 100644 index 0000000..68f985f --- /dev/null +++ b/pkg/otlp/profiles/canonical.go @@ -0,0 +1,140 @@ +package profiles + +import ( + "bytes" + "encoding/json" + "fmt" + "slices" + "strconv" + + "go.opentelemetry.io/collector/pdata/pprofile" +) + +func canonicalNative(data []byte) (json.RawMessage, error) { + profile, err := (&pprofile.ProtoUnmarshaler{}).UnmarshalProfiles(data) + if err != nil { + return nil, err + } + encoded, err := (&pprofile.JSONMarshaler{}).MarshalProfiles(profile) + if err != nil { + return nil, err + } + var value map[string]any + decoder := json.NewDecoder(bytes.NewReader(encoded)) + decoder.UseNumber() + if err := decoder.Decode(&value); err != nil { + return nil, err + } + dictionary, ok := value["dictionary"].(map[string]any) + if !ok { + return nil, fmt.Errorf("native profile dictionary is required") + } + delete(value, "dictionary") + resolved, err := resolveNative(value, dictionary, 0) + if err != nil { + return nil, err + } + return json.Marshal(resolved) +} + +var referenceTables = map[string]string{ + "keyStrindex": "stringTable", "stringValueStrindex": "stringTable", "typeStrindex": "stringTable", "unitStrindex": "stringTable", + "nameStrindex": "stringTable", "systemNameStrindex": "stringTable", "filenameStrindex": "stringTable", + "stackIndex": "stackTable", "linkIndex": "linkTable", "functionIndex": "functionTable", "mappingIndex": "mappingTable", + "attributeIndices": "attributeTable", "locationIndices": "locationTable", +} + +func resolveNative(value any, dictionary map[string]any, depth int) (any, error) { + if depth > 64 { + return nil, fmt.Errorf("native profile reference depth exceeded") + } + switch item := value.(type) { + case map[string]any: + resolved := make(map[string]any, len(item)) + for key, field := range item { + var err error + if table, reference := referenceTables[key]; reference { + resolved[key], err = resolveReference(field, dictionary, table, depth+1) + } else { + resolved[key], err = resolveNative(field, dictionary, depth+1) + } + if err != nil { + return nil, fmt.Errorf("native profile %s: %w", key, err) + } + if key == "attributeIndices" || key == "samples" || key == "attributes" { + if err := sortNative(resolved[key]); err != nil { + return nil, err + } + } + if key == "kvlistValue" { + if object, ok := resolved[key].(map[string]any); ok && object["values"] != nil { + if err := sortNative(object["values"]); err != nil { + return nil, err + } + } + } + } + return resolved, nil + case []any: + resolved := make([]any, len(item)) + for index, field := range item { + var err error + resolved[index], err = resolveNative(field, dictionary, depth+1) + if err != nil { + return nil, err + } + } + return resolved, nil + default: + return value, nil + } +} + +func resolveReference(value any, dictionary map[string]any, table string, depth int) (any, error) { + if references, ok := value.([]any); ok { + result := make([]any, len(references)) + for index, reference := range references { + var err error + result[index], err = resolveReference(reference, dictionary, table, depth+1) + if err != nil { + return nil, err + } + } + return result, nil + } + number, ok := value.(json.Number) + if !ok { + return nil, fmt.Errorf("reference index must be an integer") + } + index, err := strconv.Atoi(string(number)) + if err != nil { + return nil, err + } + entries, ok := dictionary[table].([]any) + if !ok || index < 0 || index >= len(entries) { + return nil, fmt.Errorf("%s index %d out of range", table, index) + } + return resolveNative(entries[index], dictionary, depth+1) +} + +func sortNative(value any) error { + entries, ok := value.([]any) + if !ok { + return fmt.Errorf("canonical native field must be a list") + } + encoded := make([]string, len(entries)) + indexed := make(map[string]any, len(entries)) + for index, entry := range entries { + data, err := json.Marshal(entry) + if err != nil { + return err + } + encoded[index] = string(data) + indexed[string(data)] = entry + } + slices.Sort(encoded) + for index, key := range encoded { + entries[index] = indexed[key] + } + return nil +} diff --git a/pkg/otlp/profiles/control.go b/pkg/otlp/profiles/control.go new file mode 100644 index 0000000..9093ad9 --- /dev/null +++ b/pkg/otlp/profiles/control.go @@ -0,0 +1,58 @@ +package profiles + +import ( + "fmt" + "time" + + "go.opentelemetry.io/collector/pdata/pprofile" +) + +// WORKAROUND(collector-empty-profiles): Carry zero-operation observations as explicit control samples because released Collectors discard zero-sample profiles. +// Correct fix: Collector must preserve metadata-only profiles through receiving, item-sized queues, merging and splitting. +// Ref: https://github.com/open-telemetry/opentelemetry-collector/blob/v0.165.0/receiver/otlpreceiver/internal/profiles/otlp.go#L36-L38; approved by user 2026-11-15. +func encodeControl(d *dictionary, sample pprofile.Sample, observedAt time.Time) error { + if err := d.attributes(map[string]any{sampleKindAttribute: snapshotControlKind}, sample.AttributeIndices()); err != nil { + return err + } + sample.Values().Append(0) + sample.TimestampsUnixNano().Append(uint64(observedAt.UnixNano())) + return nil +} + +func decodeSamples(d pprofile.ProfilesDictionary, profile pprofile.Profile, snapshot Snapshot) ([]Row, error) { + if profile.Samples().Len() == 0 { + return nil, fmt.Errorf("top snapshots require an operation or snapshot control sample") + } + rows := make([]Row, 0, profile.Samples().Len()) + for _, sample := range profile.Samples().All() { + attributes, err := readAttributes(d, sample.AttributeIndices()) + if err != nil { + return nil, err + } + if kind, control := attributes[sampleKindAttribute]; control { + if kind != snapshotControlKind || len(attributes) != 1 || profile.Samples().Len() != 1 || snapshot.Observed != 0 { + return nil, fmt.Errorf("snapshot control requires its exact marker as the sole sample with zero observed operations") + } + if err := validateControl(sample, snapshot.CompletedAt); err != nil { + return nil, err + } + return rows, nil + } + row, err := decodeRow(d, sample, attributes) + if err != nil { + return nil, err + } + rows = append(rows, row) + } + return rows, nil +} + +// WORKAROUND(collector-empty-profiles): Validate and retain the transport control sample without projecting a fabricated active operation. +// Correct fix: Collector must preserve metadata-only profiles without a control sample. +// Ref: https://github.com/open-telemetry/opentelemetry-collector/blob/v0.165.0/receiver/otlpreceiver/internal/profiles/otlp.go#L36-L38; approved by user 2026-11-15. +func validateControl(sample pprofile.Sample, observedAt time.Time) error { + if sample.Values().Len() != 1 || sample.Values().At(0) != 0 || sample.TimestampsUnixNano().Len() != 1 || sample.TimestampsUnixNano().At(0) != uint64(observedAt.UnixNano()) || sample.StackIndex() != 0 || sample.LinkIndex() != 0 { + return fmt.Errorf("snapshot control requires a zero elapsed value, the snapshot observation timestamp and zero stack/link references") + } + return nil +} diff --git a/pkg/otlp/profiles/control_test.go b/pkg/otlp/profiles/control_test.go new file mode 100644 index 0000000..fe7bc24 --- /dev/null +++ b/pkg/otlp/profiles/control_test.go @@ -0,0 +1,167 @@ +package profiles_test + +import ( + "math" + "time" + + "github.com/flanksource/telltale/pkg/otlp/profiles" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pprofile" +) + +const controlAttribute = "telltale.sample.kind" +const controlKind = "snapshot_control" + +func controlSnapshot() profiles.Snapshot { + at := time.Date(2026, 11, 15, 12, 0, 0, 0, time.UTC) + return profiles.Snapshot{Kind: "http_top", CaptureID: "capture-control", Source: "jvm-control", ProfileID: "97fa6305-a036-4d96-8789-37c1bbe378f3", Sequence: 3, + StartedAt: at, CompletedAt: at.Add(time.Millisecond), Limit: 50, Complete: true} +} + +func nativeProfile(value pprofile.Profiles) pprofile.Profile { + return value.ResourceProfiles().At(0).ScopeProfiles().At(0).Profiles().At(0) +} + +func sampleAttribute(value pprofile.Profiles, sample pprofile.Sample, key string, item any) { + GinkgoHelper() + dictionary := value.Dictionary() + dictionary.StringTable().Append(key) + attribute := dictionary.AttributeTable().AppendEmpty() + attribute.SetKeyStrindex(int32(dictionary.StringTable().Len() - 1)) + Expect(attribute.Value().FromRaw(item)).To(Succeed()) + sample.AttributeIndices().Append(int32(dictionary.AttributeTable().Len() - 1)) +} + +func profileAttribute(value pprofile.Profiles, key string) pcommon.Value { + GinkgoHelper() + for _, index := range nativeProfile(value).AttributeIndices().All() { + attribute := value.Dictionary().AttributeTable().At(int(index)) + if value.Dictionary().StringTable().At(int(attribute.KeyStrindex())) == key { + return attribute.Value() + } + } + Fail("missing profile attribute " + key) + return pcommon.Value{} +} + +func nativeControl() pprofile.Profiles { + GinkgoHelper() + data, err := profiles.Encode([]profiles.Snapshot{controlSnapshot()}) + Expect(err).NotTo(HaveOccurred()) + value, err := (&pprofile.ProtoUnmarshaler{}).UnmarshalProfiles(data) + Expect(err).NotTo(HaveOccurred()) + profileAttribute(value, "telltale.schema.version").SetInt(2) + profile := nativeProfile(value) + profile.Samples().RemoveIf(func(pprofile.Sample) bool { return true }) + sample := profile.Samples().AppendEmpty() + sampleAttribute(value, sample, controlAttribute, controlKind) + sample.Values().Append(0) + sample.TimestampsUnixNano().Append(uint64(controlSnapshot().CompletedAt.UnixNano())) + return value +} + +var _ = Describe("snapshot control convention", func() { + DescribeTable("encodes a zero-operation snapshot as one control sample and retains zero projected rows", func(complete bool) { + snapshot := controlSnapshot() + snapshot.Complete = complete + data, err := profiles.Encode([]profiles.Snapshot{snapshot}) + Expect(err).NotTo(HaveOccurred()) + value, err := (&pprofile.ProtoUnmarshaler{}).UnmarshalProfiles(data) + Expect(err).NotTo(HaveOccurred()) + Expect(profileAttribute(value, "telltale.schema.version").Int()).To(Equal(int64(2))) + Expect(nativeProfile(value).Samples().Len()).To(Equal(1)) + Expect(nativeProfile(value).Samples().At(0).Values().AsRaw()).To(Equal([]int64{0})) + decoded, err := profiles.Decode(data, profiles.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(decoded).To(HaveLen(1)) + Expect(decoded[0].Rows).To(BeEmpty()) + Expect(decoded[0].Observed).To(BeZero()) + Expect(decoded[0].Complete).To(Equal(complete)) + Expect(decoded[0].Native).NotTo(BeEmpty()) + retained, err := (&pprofile.ProtoUnmarshaler{}).UnmarshalProfiles(decoded[0].Native) + Expect(err).NotTo(HaveOccurred()) + Expect(retained.SampleCount()).To(Equal(1)) + }, Entry("complete clear", true), Entry("incomplete observation", false)) + + It("decodes an independently assembled exact control record", func() { + data, err := (&pprofile.ProtoMarshaler{}).MarshalProfiles(nativeControl()) + Expect(err).NotTo(HaveOccurred()) + decoded, err := profiles.Decode(data, profiles.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(decoded).To(HaveLen(1)) + Expect(decoded[0].Rows).To(BeEmpty()) + Expect(decoded[0].Digest).NotTo(BeEmpty()) + }) + + It("rejects a positive observed count with no retained operations instead of emitting empty telemetry", func() { + snapshot := controlSnapshot() + snapshot.Observed = 1 + _, err := profiles.Encode([]profiles.Snapshot{snapshot}) + Expect(err).To(MatchError(ContainSubstring("retained operation"))) + }) + + It("rejects operation attributes that claim the reserved control marker", func() { + snapshot := controlSnapshot() + snapshot.Observed = 1 + snapshot.Rows = []profiles.Row{{OperationID: "operation", SampledAt: snapshot.CompletedAt, Attributes: map[string]any{controlAttribute: controlKind}}} + _, err := profiles.Encode([]profiles.Snapshot{snapshot}) + Expect(err).To(MatchError(ContainSubstring("reserved"))) + }) + + DescribeTable("rejects malformed control samples before projecting an operation", func(change func(pprofile.Profiles)) { + value := nativeControl() + change(value) + data, err := (&pprofile.ProtoMarshaler{}).MarshalProfiles(value) + Expect(err).NotTo(HaveOccurred()) + _, err = profiles.Decode(data, profiles.Options{}) + Expect(err).To(HaveOccurred()) + }, Entry("unknown marker kind", func(value pprofile.Profiles) { + dictionary := value.Dictionary() + dictionary.AttributeTable().At(int(nativeProfile(value).Samples().At(0).AttributeIndices().At(0))).Value().SetStr("unknown") + }), Entry("nonstrings marker", func(value pprofile.Profiles) { + dictionary := value.Dictionary() + dictionary.AttributeTable().At(int(nativeProfile(value).Samples().At(0).AttributeIndices().At(0))).Value().SetBool(true) + }), Entry("missing marker", func(value pprofile.Profiles) { + nativeProfile(value).Samples().At(0).AttributeIndices().FromRaw(nil) + }), Entry("no samples", func(value pprofile.Profiles) { + nativeProfile(value).Samples().RemoveIf(func(pprofile.Sample) bool { return true }) + }), Entry("missing value", func(value pprofile.Profiles) { + nativeProfile(value).Samples().At(0).Values().FromRaw(nil) + }), Entry("multiple values", func(value pprofile.Profiles) { + nativeProfile(value).Samples().At(0).Values().Append(0) + }), Entry("nonzero value", func(value pprofile.Profiles) { + nativeProfile(value).Samples().At(0).Values().SetAt(0, 1) + }), Entry("missing timestamp", func(value pprofile.Profiles) { + nativeProfile(value).Samples().At(0).TimestampsUnixNano().FromRaw(nil) + }), Entry("multiple timestamps", func(value pprofile.Profiles) { + nativeProfile(value).Samples().At(0).TimestampsUnixNano().Append(uint64(controlSnapshot().CompletedAt.UnixNano())) + }), Entry("wrong observation timestamp", func(value pprofile.Profiles) { + nativeProfile(value).Samples().At(0).TimestampsUnixNano().SetAt(0, uint64(controlSnapshot().StartedAt.UnixNano())) + }), Entry("out of range timestamp", func(value pprofile.Profiles) { + nativeProfile(value).Samples().At(0).TimestampsUnixNano().SetAt(0, math.MaxUint64) + }), Entry("stack reference", func(value pprofile.Profiles) { + nativeProfile(value).Samples().At(0).SetStackIndex(1) + }), Entry("trace link", func(value pprofile.Profiles) { + nativeProfile(value).Samples().At(0).SetLinkIndex(1) + }), Entry("operation ID", func(value pprofile.Profiles) { + sampleAttribute(value, nativeProfile(value).Samples().At(0), "telltale.operation.id", "invented") + }), Entry("HTTP operation metadata", func(value pprofile.Profiles) { + sampleAttribute(value, nativeProfile(value).Samples().At(0), "method", "GET") + }), Entry("SQL operation metadata", func(value pprofile.Profiles) { + sampleAttribute(value, nativeProfile(value).Samples().At(0), "sql", "SELECT id FROM orders") + }), Entry("unrecognized metadata", func(value pprofile.Profiles) { + sampleAttribute(value, nativeProfile(value).Samples().At(0), "extra", "value") + }), Entry("mixed with operation", func(value pprofile.Profiles) { + profile := nativeProfile(value) + operation := profile.Samples().AppendEmpty() + sampleAttribute(value, operation, "telltale.operation.id", "actual-operation") + operation.Values().Append(1) + operation.TimestampsUnixNano().Append(uint64(controlSnapshot().CompletedAt.UnixNano())) + }), Entry("positive observed count", func(value pprofile.Profiles) { + profileAttribute(value, "telltale.snapshot.observed").SetInt(1) + }), Entry("old schema", func(value pprofile.Profiles) { + profileAttribute(value, "telltale.schema.version").SetInt(1) + })) +}) diff --git a/pkg/otlp/profiles/decode.go b/pkg/otlp/profiles/decode.go new file mode 100644 index 0000000..3cf63e6 --- /dev/null +++ b/pkg/otlp/profiles/decode.go @@ -0,0 +1,314 @@ +package profiles + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "slices" + "time" + + "github.com/flanksource/telltale/pkg/tracing" + "github.com/google/uuid" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pprofile" +) + +func Decode(data []byte, options Options) ([]Snapshot, error) { + if options.MaxBytes < 0 || options.MaxProfiles < 0 || options.MaxSamples < 0 { + return nil, fmt.Errorf("profile limits must be nonnegative") + } + if options.MaxBytes == 0 { + options.MaxBytes = 4 << 20 + } + if options.MaxProfiles == 0 { + options.MaxProfiles = 256 + } + if options.MaxSamples == 0 { + options.MaxSamples = 10000 + } + if len(data) > options.MaxBytes { + return nil, fmt.Errorf("%w: profile byte limit exceeded", ErrLimit) + } + value, err := (&pprofile.ProtoUnmarshaler{}).UnmarshalProfiles(data) + if err != nil { + return nil, fmt.Errorf("decode OTLP profiles: %w", err) + } + if value.ProfileCount() > options.MaxProfiles || value.SampleCount() > options.MaxSamples { + return nil, fmt.Errorf("%w: profile or sample limit exceeded", ErrLimit) + } + if value.Dictionary().StringTable().Len() == 0 || value.Dictionary().StringTable().At(0) != "" { + return nil, fmt.Errorf("profile dictionary requires a zero string entry") + } + d := value.Dictionary() + if d.MappingTable().Len() == 0 || d.FunctionTable().Len() == 0 || d.LocationTable().Len() == 0 || d.LinkTable().Len() == 0 || d.AttributeTable().Len() == 0 || d.StackTable().Len() == 0 { + return nil, fmt.Errorf("profile dictionary requires reserved zero entries") + } + return decodeProfiles(value, options) +} + +func decodeProfiles(value pprofile.Profiles, options Options) ([]Snapshot, error) { + result := []Snapshot{} + expanded := 0 + for _, resource := range value.ResourceProfiles().All() { + for _, scope := range resource.ScopeProfiles().All() { + if scope.Scope().Name() != Scope { + return nil, fmt.Errorf("unsupported profile scope %q", scope.Scope().Name()) + } + for _, profile := range scope.Profiles().All() { + snapshot, err := decodeSnapshot(value.Dictionary(), profile) + if err != nil { + return nil, err + } + snapshot.Resource = resource.Resource().Attributes().AsRaw() + for key, item := range snapshot.Resource { + snapshot.Resource[key] = tracing.MaskAttribute(key, item) + } + if err = validateIdentity(snapshot, options); err != nil { + return nil, err + } + snapshot.Native, err = nativeProfile(value.Dictionary(), resource, scope, profile) + if err != nil { + return nil, err + } + expanded += len(snapshot.Native) + if expanded > options.MaxBytes*8 { + return nil, fmt.Errorf("%w: expanded profile byte limit exceeded", ErrLimit) + } + if err = seal(&snapshot); err != nil { + return nil, err + } + result = append(result, snapshot) + } + } + } + return result, nil +} + +func validateIdentity(s Snapshot, o Options) error { + if o.ExpectedKind != "" && s.Kind != o.ExpectedKind { + return fmt.Errorf("profile kind %q does not match authenticated capture", s.Kind) + } + if o.ExpectedCaptureID != "" && s.CaptureID != o.ExpectedCaptureID { + return fmt.Errorf("profile capture %q does not match authenticated session", s.CaptureID) + } + if o.ExpectedSource != "" && s.Source != o.ExpectedSource { + return fmt.Errorf("profile source %q does not match authenticated agent", s.Source) + } + return validate(s) +} + +func decodeSnapshot(d pprofile.ProfilesDictionary, p pprofile.Profile) (Snapshot, error) { + if p.OriginalPayload().Len() > 0 { + return Snapshot{}, fmt.Errorf("active-operation profiles must not contain opaque payloads") + } + attributes, err := readAttributes(d, p.AttributeIndices()) + if err != nil { + return Snapshot{}, err + } + if attributes["telltale.schema.version"] != conventionVersion { + return Snapshot{}, fmt.Errorf("unsupported Telltale profile convention version") + } + s := Snapshot{Kind: text(attributes, "telltale.kind"), CaptureID: text(attributes, "telltale.capture.id"), Source: text(attributes, "telltale.source.instance"), + ProfileID: uuid.UUID(p.ProfileID()).String(), Checkpoint: text(attributes, "telltale.checkpoint.id"), StartedAt: p.Time().AsTime(), Rows: []Row{}} + if p.Time() == 0 || uint64(p.Time()) > math.MaxInt64 || p.DurationNano() > math.MaxInt64-uint64(p.Time()) { + return s, fmt.Errorf("invalid profile time bounds") + } + s.CompletedAt = s.StartedAt.Add(time.Duration(p.DurationNano())) + if poll, found := attributes["telltale.snapshot.poll_interval_ms"]; found { + var valid bool + s.PollIntervalMS, valid = poll.(int64) + if !valid || s.PollIntervalMS <= 0 { + return s, fmt.Errorf("profile poll interval must be positive") + } + } + var ok bool + if s.Sequence, ok = attributes["telltale.snapshot.sequence"].(int64); !ok { + return s, fmt.Errorf("profile sequence must be an integer") + } + observed, ok := attributes["telltale.snapshot.observed"].(int64) + if !ok || observed < 0 || observed > math.MaxInt { + return s, fmt.Errorf("profile observed count must be nonnegative") + } + s.Observed = int(observed) + limit, ok := attributes["telltale.snapshot.limit"].(int64) + if !ok || limit < 0 || limit > math.MaxInt { + return s, fmt.Errorf("profile limit must be nonnegative") + } + s.Limit = int(limit) + dropped, ok := attributes["telltale.snapshot.dropped"].(int64) + if !ok || dropped < 0 { + return s, fmt.Errorf("profile dropped count must be nonnegative") + } + s.Dropped = uint64(dropped) + if s.Complete, ok = attributes["telltale.snapshot.complete"].(bool); !ok { + return s, fmt.Errorf("profile completeness is required") + } + metric, err := readString(d, p.SampleType().TypeStrindex()) + if err != nil { + return s, err + } + unit, err := readString(d, p.SampleType().UnitStrindex()) + if err != nil { + return s, err + } + if metric != sampleTypes[s.Kind] || unit != "ns" { + return s, fmt.Errorf("profile sample type must describe elapsed nanoseconds") + } + s.Rows, err = decodeSamples(d, p, s) + return s, err +} + +func decodeRow(d pprofile.ProfilesDictionary, sample pprofile.Sample, attributes map[string]any) (Row, error) { + if sample.Values().Len() != 1 || sample.TimestampsUnixNano().Len() != 1 { + return Row{}, fmt.Errorf("top samples require exactly one value and timestamp") + } + if sample.TimestampsUnixNano().At(0) > math.MaxInt64 { + return Row{}, fmt.Errorf("sample timestamp exceeds signed nanoseconds") + } + row := Row{OperationID: text(attributes, "telltale.operation.id"), ElapsedNS: sample.Values().At(0), SampledAt: time.Unix(0, int64(sample.TimestampsUnixNano().At(0))).UTC(), Attributes: attributes} + delete(row.Attributes, "telltale.operation.id") + if index := sample.LinkIndex(); index != 0 { + if index < 0 || int(index) >= d.LinkTable().Len() { + return row, fmt.Errorf("profile link index is out of range") + } + link := d.LinkTable().At(int(index)) + row.TraceID = link.TraceID().String() + row.SpanID = link.SpanID().String() + if link.TraceID().IsEmpty() || link.SpanID().IsEmpty() { + return row, fmt.Errorf("profile link requires trace and span IDs") + } + } + stack, err := readStack(d, sample.StackIndex()) + row.Stack = stack + return row, err +} + +func readString(d pprofile.ProfilesDictionary, index int32) (string, error) { + if index < 0 || int(index) >= d.StringTable().Len() { + return "", fmt.Errorf("profile string index is out of range") + } + return d.StringTable().At(int(index)), nil +} + +func readAttributes(d pprofile.ProfilesDictionary, indices pcommon.Int32Slice) (map[string]any, error) { + attributes := map[string]any{} + for _, index := range indices.All() { + if index <= 0 || int(index) >= d.AttributeTable().Len() { + return nil, fmt.Errorf("profile attribute index is out of range") + } + attribute := d.AttributeTable().At(int(index)) + key, err := readString(d, attribute.KeyStrindex()) + if err != nil { + return nil, err + } + if key == "" { + return nil, fmt.Errorf("profile attribute key is empty") + } + if _, found := attributes[key]; found { + return nil, fmt.Errorf("duplicate profile attribute %q", key) + } + attributes[key] = tracing.MaskAttribute(key, attribute.Value().AsRaw()) + } + return attributes, nil +} + +func readStack(d pprofile.ProfilesDictionary, index int32) ([]string, error) { + if index == 0 { + return nil, nil + } + if index < 0 || int(index) >= d.StackTable().Len() { + return nil, fmt.Errorf("profile stack index is out of range") + } + frames := []string{} + for _, locationIndex := range d.StackTable().At(int(index)).LocationIndices().All() { + if locationIndex <= 0 || int(locationIndex) >= d.LocationTable().Len() { + return nil, fmt.Errorf("profile location index is out of range") + } + for _, line := range d.LocationTable().At(int(locationIndex)).Lines().All() { + functionIndex := line.FunctionIndex() + if functionIndex <= 0 || int(functionIndex) >= d.FunctionTable().Len() { + return nil, fmt.Errorf("profile function index is out of range") + } + name, err := readString(d, d.FunctionTable().At(int(functionIndex)).NameStrindex()) + if err != nil { + return nil, err + } + frames = append(frames, name) + } + } + return frames, nil +} + +func seal(s *Snapshot) error { + slices.SortFunc(s.Rows, func(a, b Row) int { + if a.OperationID < b.OperationID { + return -1 + } + if a.OperationID > b.OperationID { + return 1 + } + return 0 + }) + canonical := *s + canonical.Native = nil + canonical.Digest = "" + native, err := canonicalNative(s.Native) + if err != nil { + return fmt.Errorf("canonical native profile: %w", err) + } + data, err := json.Marshal(struct { + Snapshot Snapshot + Native json.RawMessage + }{canonical, native}) + if err != nil { + return fmt.Errorf("canonical profile: %w", err) + } + digest := sha256.Sum256(data) + s.Digest = hex.EncodeToString(digest[:]) + return nil +} + +func nativeProfile(dictionary pprofile.ProfilesDictionary, resource pprofile.ResourceProfiles, scope pprofile.ScopeProfiles, profile pprofile.Profile) ([]byte, error) { + value := pprofile.NewProfiles() + dictionary.CopyTo(value.Dictionary()) + copyResource := value.ResourceProfiles().AppendEmpty() + resource.Resource().CopyTo(copyResource.Resource()) + copyResource.SetSchemaUrl(resource.SchemaUrl()) + for key, item := range copyResource.Resource().Attributes().All() { + if err := item.FromRaw(tracing.MaskAttribute(key, item.AsRaw())); err != nil { + return nil, err + } + } + copyScope := copyResource.ScopeProfiles().AppendEmpty() + scope.Scope().CopyTo(copyScope.Scope()) + for key, item := range copyScope.Scope().Attributes().All() { + if err := item.FromRaw(tracing.MaskAttribute(key, item.AsRaw())); err != nil { + return nil, err + } + } + copyScope.SetSchemaUrl(scope.SchemaUrl()) + profile.CopyTo(copyScope.Profiles().AppendEmpty()) + for _, attribute := range value.Dictionary().AttributeTable().All() { + key, err := readString(value.Dictionary(), attribute.KeyStrindex()) + if err != nil { + return nil, err + } + if err = attribute.Value().FromRaw(tracing.MaskAttribute(key, attribute.Value().AsRaw())); err != nil { + return nil, err + } + } + // Rebuild references after masking so unused dictionary strings cannot retain secrets. + redacted := pprofile.NewProfiles() + newDictionary(redacted.Dictionary()) + if err := value.MergeTo(redacted); err != nil { + return nil, fmt.Errorf("rebuild redacted profile dictionary: %w", err) + } + return (&pprofile.ProtoMarshaler{}).MarshalProfiles(redacted) +} + +func text(attributes map[string]any, key string) string { + value, _ := attributes[key].(string) + return value +} diff --git a/pkg/otlp/profiles/encode.go b/pkg/otlp/profiles/encode.go new file mode 100644 index 0000000..eaae4d1 --- /dev/null +++ b/pkg/otlp/profiles/encode.go @@ -0,0 +1,185 @@ +package profiles + +import ( + "encoding/hex" + "fmt" + "math" + "slices" + + "github.com/flanksource/telltale/pkg/tracing" + "github.com/google/uuid" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pprofile" +) + +type dictionary struct { + value pprofile.ProfilesDictionary + strings map[string]int32 +} + +func newDictionary(value pprofile.ProfilesDictionary) *dictionary { + value.StringTable().Append("") + value.MappingTable().AppendEmpty() + value.LocationTable().AppendEmpty() + value.FunctionTable().AppendEmpty() + value.LinkTable().AppendEmpty() + value.AttributeTable().AppendEmpty() + value.StackTable().AppendEmpty() + return &dictionary{value: value, strings: map[string]int32{"": 0}} +} + +func (d *dictionary) text(value string) int32 { + if index, found := d.strings[value]; found { + return index + } + index := int32(d.value.StringTable().Len()) + d.value.StringTable().Append(value) + d.strings[value] = index + return index +} + +func (d *dictionary) attributes(values map[string]any, indices pcommon.Int32Slice) error { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + slices.Sort(keys) + for _, key := range keys { + attribute := d.value.AttributeTable().AppendEmpty() + attribute.SetKeyStrindex(d.text(key)) + if err := attribute.Value().FromRaw(tracing.MaskAttribute(key, values[key])); err != nil { + return fmt.Errorf("profile attribute %s: %w", key, err) + } + indices.Append(int32(d.value.AttributeTable().Len() - 1)) + } + return nil +} + +func metadata(s Snapshot) map[string]any { + attributes := map[string]any{"telltale.schema.version": conventionVersion, "telltale.kind": s.Kind, "telltale.capture.id": s.CaptureID, + "telltale.source.instance": s.Source, "telltale.snapshot.sequence": s.Sequence, "telltale.snapshot.observed": int64(s.Observed), + "telltale.snapshot.limit": int64(s.Limit), "telltale.snapshot.dropped": int64(s.Dropped), "telltale.snapshot.complete": s.Complete, + "telltale.checkpoint.id": s.Checkpoint} + if s.PollIntervalMS > 0 { + attributes["telltale.snapshot.poll_interval_ms"] = s.PollIntervalMS + } + return attributes +} + +func validate(s Snapshot) error { + if _, ok := sampleTypes[s.Kind]; !ok { + return fmt.Errorf("unsupported top profile kind %q", s.Kind) + } + if s.CaptureID == "" || s.Source == "" || s.Sequence <= 0 { + return fmt.Errorf("profile capture, source and positive sequence are required") + } + if id, err := uuid.Parse(s.ProfileID); err != nil || id == uuid.Nil { + return fmt.Errorf("profile ID must be a nonzero UUID") + } + if s.StartedAt.IsZero() || s.CompletedAt.Before(s.StartedAt) || s.StartedAt.UnixNano() <= 0 { + return fmt.Errorf("profile requires ordered producer timestamps") + } + if s.Observed < len(s.Rows) || s.Limit < 0 || (s.Limit > 0 && len(s.Rows) > s.Limit) || s.Dropped > math.MaxInt64 { + return fmt.Errorf("profile observed/retained counts exceed snapshot limits") + } + if s.Observed > 0 && len(s.Rows) == 0 { + return fmt.Errorf("a positive profile observed count requires a retained operation") + } + seen := map[string]bool{} + for _, row := range s.Rows { + if _, exists := row.Attributes[sampleKindAttribute]; exists { + return fmt.Errorf("sample kind is reserved for snapshot control") + } + if row.OperationID == "" || seen[row.OperationID] || row.ElapsedNS < 0 || row.SampledAt.Before(s.StartedAt) || row.SampledAt.After(s.CompletedAt) { + return fmt.Errorf("profile requires distinct operation IDs, nonnegative elapsed time and bounded sample timestamps") + } + seen[row.OperationID] = true + } + return nil +} + +func Encode(snapshots []Snapshot) ([]byte, error) { + value := pprofile.NewProfiles() + dictionary := newDictionary(value.Dictionary()) + for _, snapshot := range snapshots { + if err := validate(snapshot); err != nil { + return nil, err + } + if err := encodeSnapshot(value, dictionary, snapshot); err != nil { + return nil, err + } + } + return (&pprofile.ProtoMarshaler{}).MarshalProfiles(value) +} + +func encodeSnapshot(value pprofile.Profiles, d *dictionary, s Snapshot) error { + resource := value.ResourceProfiles().AppendEmpty() + masked := map[string]any{} + for key, item := range s.Resource { + masked[key] = tracing.MaskAttribute(key, item) + } + if err := resource.Resource().Attributes().FromRaw(masked); err != nil { + return err + } + scope := resource.ScopeProfiles().AppendEmpty() + scope.Scope().SetName(Scope) + scope.Scope().SetVersion("1") + profile := scope.Profiles().AppendEmpty() + id, _ := uuid.Parse(s.ProfileID) + profile.SetProfileID(pprofile.ProfileID(id)) + profile.SetTime(pcommon.NewTimestampFromTime(s.StartedAt)) + profile.SetDurationNano(uint64(s.CompletedAt.Sub(s.StartedAt))) + profile.SampleType().SetTypeStrindex(d.text(sampleTypes[s.Kind])) + profile.SampleType().SetUnitStrindex(d.text("ns")) + if err := d.attributes(metadata(s), profile.AttributeIndices()); err != nil { + return err + } + if len(s.Rows) == 0 { + return encodeControl(d, profile.Samples().AppendEmpty(), s.CompletedAt) + } + for _, row := range s.Rows { + if err := encodeRow(d, profile.Samples().AppendEmpty(), row); err != nil { + return err + } + } + return nil +} + +func encodeRow(d *dictionary, sample pprofile.Sample, row Row) error { + attributes := map[string]any{} + for key, value := range row.Attributes { + attributes[key] = value + } + attributes["telltale.operation.id"] = row.OperationID + if err := d.attributes(attributes, sample.AttributeIndices()); err != nil { + return err + } + sample.Values().Append(row.ElapsedNS) + sample.TimestampsUnixNano().Append(uint64(row.SampledAt.UnixNano())) + if row.TraceID != "" || row.SpanID != "" { + trace, err := hex.DecodeString(row.TraceID) + if err != nil || len(trace) != 16 { + return fmt.Errorf("profile requires a 16-byte trace ID") + } + span, err := hex.DecodeString(row.SpanID) + if err != nil || len(span) != 8 { + return fmt.Errorf("profile requires an 8-byte span ID") + } + link := d.value.LinkTable().AppendEmpty() + link.SetTraceID(pcommon.TraceID(trace)) + link.SetSpanID(pcommon.SpanID(span)) + sample.SetLinkIndex(int32(d.value.LinkTable().Len() - 1)) + } + if len(row.Stack) > 0 { + stack := d.value.StackTable().AppendEmpty() + sample.SetStackIndex(int32(d.value.StackTable().Len() - 1)) + for _, name := range row.Stack { + function := d.value.FunctionTable().AppendEmpty() + function.SetNameStrindex(d.text(name)) + location := d.value.LocationTable().AppendEmpty() + location.Lines().AppendEmpty().SetFunctionIndex(int32(d.value.FunctionTable().Len() - 1)) + stack.LocationIndices().Append(int32(d.value.LocationTable().Len() - 1)) + } + } + return nil +} diff --git a/pkg/otlp/profiles/java_test.go b/pkg/otlp/profiles/java_test.go new file mode 100644 index 0000000..b205a4c --- /dev/null +++ b/pkg/otlp/profiles/java_test.go @@ -0,0 +1,34 @@ +package profiles_test + +import ( + "os" + "time" + + "github.com/flanksource/telltale/pkg/otlp/profiles" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Java Profiles golden wire contract", func() { + DescribeTable("decodes official Java ProfilesData without JSON integer loss", func(name string, sequence int64, elapsed int64) { + data, err := os.ReadFile("testdata/java-http-top-" + name + ".pb") + Expect(err).NotTo(HaveOccurred()) + decoded, err := profiles.Decode(data, profiles.Options{ExpectedKind: "http_top", ExpectedSource: "jvm-a", ExpectedCaptureID: "capture-a"}) + Expect(err).NotTo(HaveOccurred()) + Expect(decoded).To(HaveLen(1)) + snapshot := decoded[0] + Expect(snapshot.Sequence).To(Equal(sequence)) + Expect(snapshot.Complete).To(BeTrue()) + Expect(snapshot.PollIntervalMS).To(Equal(int64(5000))) + if name == "empty" { + Expect(snapshot.Rows).To(BeEmpty()) + Expect(snapshot.Observed).To(BeZero()) + return + } + Expect(snapshot.Resource).To(HaveKeyWithValue("service.name", "checkout")) + Expect(snapshot.Rows).To(ConsistOf(profiles.Row{OperationID: "request-a", ElapsedNS: elapsed, + SampledAt: time.Date(2026, 11, 14, 12, 0, 0, 0, time.UTC), + Attributes: map[string]any{"method": "GET", "url": "https://example.test/orders", "route": "/orders/{id}", "direction": "incoming", "startedAt": "2026-11-14T11:59:59Z", "threadId": int64(9007199254740993)}, + TraceID: "0102030405060708090a0b0c0d0e0f10", SpanID: "0102030405060708", Stack: []string{"application.handle", "application.run"}})) + }, Entry("active", "active", int64(1), int64(250000000)), Entry("updated", "updated", int64(2), int64(500000000)), Entry("empty", "empty", int64(3), int64(0))) +}) diff --git a/pkg/otlp/profiles/profiles.go b/pkg/otlp/profiles/profiles.go new file mode 100644 index 0000000..7b7ad01 --- /dev/null +++ b/pkg/otlp/profiles/profiles.go @@ -0,0 +1,60 @@ +// Package profiles encodes Telltale active-operation snapshots as OTLP Profiles. +package profiles + +import ( + "encoding/base64" + "errors" + "time" + + "github.com/flanksource/commons-db/recordstore/sampleframe" +) + +const ContentType = "application/x-protobuf;message=opentelemetry.proto.profiles.v1development.ProfilesData" +const DevelopmentVersion = "1" +const Scope = "com.flanksource.telltale.top" +const conventionVersion int64 = 2 +const sampleKindAttribute = "telltale.sample.kind" +const snapshotControlKind = "snapshot_control" + +var sampleTypes = map[string]string{"http_top": "telltale.http.active.elapsed", "sql_top": "telltale.sql.active.elapsed", "jms_top": "telltale.jms.active.elapsed"} + +var ErrLimit = errors.New("OTLP Profiles limit exceeded") + +type Options struct { + MaxBytes, MaxProfiles, MaxSamples int + ExpectedKind, ExpectedCaptureID, ExpectedSource string +} + +type Row struct { + OperationID string `json:"operationId"` + ElapsedNS int64 `json:"elapsedNs"` + SampledAt time.Time `json:"sampledAt"` + Attributes map[string]any `json:"attributes"` + TraceID string `json:"traceId,omitempty"` + SpanID string `json:"spanId,omitempty"` + Stack []string `json:"stack,omitempty"` +} + +type Snapshot struct { + Kind, CaptureID, Source, ProfileID, Checkpoint string + Sequence int64 + PollIntervalMS int64 + StartedAt, CompletedAt time.Time + Observed, Limit int + Dropped uint64 + Complete bool + Rows []Row + Resource map[string]any + Native []byte + Digest string +} + +func (s Snapshot) Frame() sampleframe.Frame { + frame := sampleframe.Frame{Source: s.Source + "/" + s.CaptureID, Tick: s.Sequence, StartedAt: s.StartedAt, CompletedAt: s.CompletedAt, + Observed: s.Observed, Limit: s.Limit, Summary: map[string]any{"profileId": s.ProfileID, "captureId": s.CaptureID, "checkpoint": s.Checkpoint, + "receivedAt": time.Now().UTC().Format(time.RFC3339Nano), "dropped": s.Dropped, "complete": s.Complete, "otlp": base64.StdEncoding.EncodeToString(s.Native), "digest": s.Digest, "profilesDevelopmentVersion": DevelopmentVersion}} + if s.PollIntervalMS > 0 { + frame.Summary["pollIntervalMs"] = s.PollIntervalMS + } + return frame +} diff --git a/pkg/otlp/profiles/profiles_test.go b/pkg/otlp/profiles/profiles_test.go new file mode 100644 index 0000000..0453b4d --- /dev/null +++ b/pkg/otlp/profiles/profiles_test.go @@ -0,0 +1,177 @@ +package profiles_test + +import ( + "bytes" + "testing" + "time" + + "github.com/flanksource/telltale/pkg/otlp/profiles" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.opentelemetry.io/collector/pdata/pprofile" +) + +func TestProfiles(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "OTLP top profiles") } + +var _ = Describe("active operation profiles", func() { + fixture := func() profiles.Snapshot { + at := time.Date(2026, 11, 14, 12, 0, 0, 0, time.UTC) + return profiles.Snapshot{Kind: "http_top", CaptureID: "capture-a", Source: "jvm-a", ProfileID: "10f5dd68-2656-4c07-abd7-bc884147e020", Sequence: 1, + StartedAt: at, CompletedAt: at.Add(time.Millisecond), Observed: 1, Limit: 50, Complete: true, + Resource: map[string]any{"service.name": "checkout"}, Rows: []profiles.Row{{OperationID: "request-a", ElapsedNS: 250000000, + SampledAt: at, Attributes: map[string]any{"direction": "incoming", "route": "/orders/{id}", "threadId": int64(9007199254740993)}, + TraceID: "0102030405060708090a0b0c0d0e0f10", SpanID: "0102030405060708", Stack: []string{"application.handle", "application.run"}}}} + } + It("carries active, updated and empty snapshots without losing producer identity or integer precision", func() { + first := fixture() + second := fixture() + second.Sequence = 2 + second.ProfileID = "20f5dd68-2656-4c07-abd7-bc884147e020" + second.Rows[0].ElapsedNS = 500000000 + empty := fixture() + empty.Sequence = 3 + empty.ProfileID = "30f5dd68-2656-4c07-abd7-bc884147e020" + empty.Rows = nil + empty.Observed = 0 + data, err := profiles.Encode([]profiles.Snapshot{first, second, empty}) + Expect(err).NotTo(HaveOccurred()) + native, err := (&pprofile.ProtoUnmarshaler{}).UnmarshalProfiles(data) + Expect(err).NotTo(HaveOccurred()) + Expect(native.ProfileCount()).To(Equal(3)) + decoded, err := profiles.Decode(data, profiles.Options{ExpectedKind: "http_top", ExpectedCaptureID: "capture-a"}) + Expect(err).NotTo(HaveOccurred()) + Expect(decoded).To(HaveLen(3)) + Expect(decoded[0].Rows).To(Equal(first.Rows)) + Expect(decoded[0].StartedAt).To(Equal(first.StartedAt)) + Expect(decoded[0].Source).To(Equal(first.Source)) + Expect(decoded[1].Rows[0].ElapsedNS).To(Equal(int64(500000000))) + Expect(decoded[2].Rows).To(BeEmpty()) + Expect(decoded[2].Sequence).To(Equal(int64(3))) + Expect(decoded[0].Native).NotTo(BeEmpty()) + Expect(decoded[0].Digest).NotTo(BeEmpty()) + }) + It("rejects foreign capture identities and oversized input", func() { + data, err := profiles.Encode([]profiles.Snapshot{fixture()}) + Expect(err).NotTo(HaveOccurred()) + _, err = profiles.Decode(data, profiles.Options{ExpectedCaptureID: "other"}) + Expect(err).To(MatchError(ContainSubstring("capture"))) + _, err = profiles.Decode(data, profiles.Options{MaxBytes: 1}) + Expect(err).To(MatchError(ContainSubstring("byte limit"))) + }) + It("deduplicates semantic content independently of sample and dictionary ordering", func() { + first := fixture() + first.Resource["service.instance.id"] = "instance-a" + other := first.Rows[0] + other.OperationID, other.ElapsedNS, other.Stack = "request-b", 750000000, []string{"worker.run"} + first.Rows = append(first.Rows, other) + first.Observed = 2 + data, err := profiles.Encode([]profiles.Snapshot{first}) + Expect(err).NotTo(HaveOccurred()) + original, err := profiles.Decode(data, profiles.Options{}) + Expect(err).NotTo(HaveOccurred()) + first.Rows[0], first.Rows[1] = first.Rows[1], first.Rows[0] + reordered, err := profiles.Encode([]profiles.Snapshot{first}) + Expect(err).NotTo(HaveOccurred()) + Expect(reordered).NotTo(Equal(data)) + decoded, err := profiles.Decode(reordered, profiles.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(decoded[0].Digest).To(Equal(original[0].Digest)) + Expect(decoded[0].Rows).To(Equal(original[0].Rows)) + value, err := (&pprofile.ProtoUnmarshaler{}).UnmarshalProfiles(data) + Expect(err).NotTo(HaveOccurred()) + attributes := value.ResourceProfiles().At(0).Resource().Attributes() + attributes.Clear() + attributes.PutStr("service.instance.id", "instance-a") + attributes.PutStr("service.name", "checkout") + reordered, err = (&pprofile.ProtoMarshaler{}).MarshalProfiles(value) + Expect(err).NotTo(HaveOccurred()) + decoded, err = profiles.Decode(reordered, profiles.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(decoded[0].Digest).To(Equal(original[0].Digest)) + }) + It("preserves native profile detail while redacting both native and projected attributes", func() { + data, err := profiles.Encode([]profiles.Snapshot{fixture()}) + Expect(err).NotTo(HaveOccurred()) + value, err := (&pprofile.ProtoUnmarshaler{}).UnmarshalProfiles(data) + Expect(err).NotTo(HaveOccurred()) + resource := value.ResourceProfiles().At(0) + resource.SetSchemaUrl("https://example.test/resource-schema") + scope := resource.ScopeProfiles().At(0) + scope.Scope().Attributes().PutStr("password", "private-scope-value") + profile := scope.Profiles().At(0) + profile.SetPeriod(77) + profile.SetDroppedAttributesCount(3) + dictionary := value.Dictionary() + dictionary.StringTable().Append("authorization") + attribute := dictionary.AttributeTable().AppendEmpty() + attribute.SetKeyStrindex(int32(dictionary.StringTable().Len() - 1)) + attribute.Value().SetStr("private-attribute-value") + profile.Samples().At(0).AttributeIndices().Append(int32(dictionary.AttributeTable().Len() - 1)) + data, err = (&pprofile.ProtoMarshaler{}).MarshalProfiles(value) + Expect(err).NotTo(HaveOccurred()) + Expect(bytes.Contains(data, []byte("private-attribute-value"))).To(BeTrue()) + decoded, err := profiles.Decode(data, profiles.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(decoded[0].Rows[0].Attributes).To(HaveKeyWithValue("authorization", "****")) + Expect(string(decoded[0].Native)).NotTo(ContainSubstring("private-")) + native, err := (&pprofile.ProtoUnmarshaler{}).UnmarshalProfiles(decoded[0].Native) + Expect(err).NotTo(HaveOccurred()) + Expect(native.ResourceProfiles().At(0).SchemaUrl()).To(Equal(resource.SchemaUrl())) + preserved := native.ResourceProfiles().At(0).ScopeProfiles().At(0).Profiles().At(0) + Expect(preserved.Period()).To(Equal(int64(77))) + Expect(preserved.DroppedAttributesCount()).To(Equal(uint32(3))) + }) + It("rejects invalid dictionary references and opaque payloads before projection", func() { + data, err := profiles.Encode([]profiles.Snapshot{fixture()}) + Expect(err).NotTo(HaveOccurred()) + value, err := (&pprofile.ProtoUnmarshaler{}).UnmarshalProfiles(data) + Expect(err).NotTo(HaveOccurred()) + profile := value.ResourceProfiles().At(0).ScopeProfiles().At(0).Profiles().At(0) + profile.Samples().At(0).SetLinkIndex(999) + data, err = (&pprofile.ProtoMarshaler{}).MarshalProfiles(value) + Expect(err).NotTo(HaveOccurred()) + _, err = profiles.Decode(data, profiles.Options{}) + Expect(err).To(MatchError(ContainSubstring("link index"))) + profile.Samples().At(0).SetLinkIndex(0) + profile.OriginalPayload().FromRaw([]byte("opaque-secret")) + data, err = (&pprofile.ProtoMarshaler{}).MarshalProfiles(value) + Expect(err).NotTo(HaveOccurred()) + _, err = profiles.Decode(data, profiles.Options{}) + Expect(err).To(MatchError(ContainSubstring("opaque"))) + }) + It("requires reserved zero entries even for empty snapshots", func() { + snapshot := fixture() + snapshot.Rows, snapshot.Observed = nil, 0 + data, err := profiles.Encode([]profiles.Snapshot{snapshot}) + Expect(err).NotTo(HaveOccurred()) + value, err := (&pprofile.ProtoUnmarshaler{}).UnmarshalProfiles(data) + Expect(err).NotTo(HaveOccurred()) + value.Dictionary().StackTable().RemoveIf(func(pprofile.Stack) bool { return true }) + data, err = (&pprofile.ProtoMarshaler{}).MarshalProfiles(value) + Expect(err).NotTo(HaveOccurred()) + _, err = profiles.Decode(data, profiles.Options{}) + Expect(err).To(MatchError(ContainSubstring("zero entries"))) + }) + DescribeTable("includes native metadata in semantic conflicts", func(change func(pprofile.Profiles)) { + data, err := profiles.Encode([]profiles.Snapshot{fixture()}) + Expect(err).NotTo(HaveOccurred()) + original, err := profiles.Decode(data, profiles.Options{}) + Expect(err).NotTo(HaveOccurred()) + value, err := (&pprofile.ProtoUnmarshaler{}).UnmarshalProfiles(data) + Expect(err).NotTo(HaveOccurred()) + change(value) + data, err = (&pprofile.ProtoMarshaler{}).MarshalProfiles(value) + Expect(err).NotTo(HaveOccurred()) + changed, err := profiles.Decode(data, profiles.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(changed[0].Digest).NotTo(Equal(original[0].Digest)) + }, Entry("period", func(value pprofile.Profiles) { + value.ResourceProfiles().At(0).ScopeProfiles().At(0).Profiles().At(0).SetPeriod(7) + }), Entry("resource schema", func(value pprofile.Profiles) { + value.ResourceProfiles().At(0).SetSchemaUrl("https://example.test/new-schema") + }), Entry("scope attributes", func(value pprofile.Profiles) { + value.ResourceProfiles().At(0).ScopeProfiles().At(0).Scope().Attributes().PutStr("extra", "changed") + }), Entry("native stack address", func(value pprofile.Profiles) { + value.Dictionary().LocationTable().At(1).SetAddress(1234) + })) +}) diff --git a/pkg/otlp/profiles/testdata/java-http-top-active.pb b/pkg/otlp/profiles/testdata/java-http-top-active.pb new file mode 100644 index 0000000..aff9ed6 Binary files /dev/null and b/pkg/otlp/profiles/testdata/java-http-top-active.pb differ diff --git a/pkg/otlp/profiles/testdata/java-http-top-empty.pb b/pkg/otlp/profiles/testdata/java-http-top-empty.pb new file mode 100644 index 0000000..277d8ab Binary files /dev/null and b/pkg/otlp/profiles/testdata/java-http-top-empty.pb differ diff --git a/pkg/otlp/profiles/testdata/java-http-top-updated.pb b/pkg/otlp/profiles/testdata/java-http-top-updated.pb new file mode 100644 index 0000000..c2aba88 Binary files /dev/null and b/pkg/otlp/profiles/testdata/java-http-top-updated.pb differ diff --git a/pkg/otlp/spans/codec.go b/pkg/otlp/spans/codec.go new file mode 100644 index 0000000..371bbb5 --- /dev/null +++ b/pkg/otlp/spans/codec.go @@ -0,0 +1,42 @@ +package spans + +import ( + "encoding/base64" + "errors" + "fmt" + + sdk "github.com/flanksource/telltale/sdk" + "go.opentelemetry.io/collector/pdata/ptrace/ptraceotlp" +) + +var ErrLimit = errors.New("OTLP request limit exceeded") + +type Options struct { + MaxSpans int + MaxRequestBytes int +} + +type Batch struct { + Spans []sdk.Span + Rejected int64 + Reason string +} + +func DecodeNative(encoded string) (sdk.Span, error) { + if encoded == "" { + return sdk.Span{}, errors.New("native OTLP span is required") + } + data, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return sdk.Span{}, fmt.Errorf("decode native OTLP span: %w", err) + } + request := ptraceotlp.NewExportRequest() + if err := request.UnmarshalProto(data); err != nil { + return sdk.Span{}, fmt.Errorf("decode native OTLP span: %w", err) + } + resources := request.Traces().ResourceSpans() + if resources.Len() != 1 || resources.At(0).ScopeSpans().Len() != 1 || resources.At(0).ScopeSpans().At(0).Spans().Len() != 1 { + return sdk.Span{}, errors.New("native OTLP span must contain exactly one resource, scope and span") + } + return convertSpan(spanContext{resource: resources.At(0), scope: resources.At(0).ScopeSpans().At(0), span: resources.At(0).ScopeSpans().At(0).Spans().At(0)}) +} diff --git a/internal/otelreceiver/digest.go b/pkg/otlp/spans/digest.go similarity index 97% rename from internal/otelreceiver/digest.go rename to pkg/otlp/spans/digest.go index 06e4ecc..4133c73 100644 --- a/internal/otelreceiver/digest.go +++ b/pkg/otlp/spans/digest.go @@ -1,4 +1,4 @@ -package otelreceiver +package spans import ( "bytes" diff --git a/pkg/otlp/spans/normalize.go b/pkg/otlp/spans/normalize.go new file mode 100644 index 0000000..79a7719 --- /dev/null +++ b/pkg/otlp/spans/normalize.go @@ -0,0 +1,126 @@ +package spans + +import ( + "encoding/base64" + "fmt" + "math" + "time" + + "github.com/flanksource/telltale/pkg/tracing" + sdk "github.com/flanksource/telltale/sdk" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/ptrace" + "go.opentelemetry.io/collector/pdata/ptrace/ptraceotlp" +) + +type spanContext struct { + resource ptrace.ResourceSpans + scope ptrace.ScopeSpans + span ptrace.Span +} + +func Normalize(request ptraceotlp.ExportRequest, options Options) (Batch, error) { + result := Batch{Spans: []sdk.Span{}} + if request.Traces().SpanCount() > options.MaxSpans { + return result, fmt.Errorf("%w: at most %d spans", ErrLimit, options.MaxSpans) + } + size := 0 + resources := request.Traces().ResourceSpans() + for i := 0; i < resources.Len(); i++ { + resource := resources.At(i) + for j := 0; j < resource.ScopeSpans().Len(); j++ { + scope := resource.ScopeSpans().At(j) + for k := 0; k < scope.Spans().Len(); k++ { + span, err := convertSpan(spanContext{resource: resource, scope: scope, span: scope.Spans().At(k)}) + if err != nil { + result.Rejected++ + result.Reason = err.Error() + continue + } + size += len(span.Native) + if size > options.MaxRequestBytes*8 { + return Batch{}, fmt.Errorf("%w: expanded span contexts exceed %d bytes; reduce the batch size", ErrLimit, options.MaxRequestBytes*8) + } + result.Spans = append(result.Spans, span) + } + } + } + return result, nil +} + +func convertSpan(input spanContext) (sdk.Span, error) { + span := input.span + if span.TraceID().IsEmpty() || span.SpanID().IsEmpty() || span.StartTimestamp() == 0 || span.EndTimestamp() < span.StartTimestamp() || uint64(span.EndTimestamp()) > math.MaxInt64 || span.SpanID() == span.ParentSpanID() { + return sdk.Span{}, fmt.Errorf("invalid span: nonzero trace/span IDs, valid timestamps and a distinct parent are required") + } + native := ptraceotlp.NewExportRequest() + resource := native.Traces().ResourceSpans().AppendEmpty() + input.resource.Resource().CopyTo(resource.Resource()) + resource.SetSchemaUrl(input.resource.SchemaUrl()) + scope := resource.ScopeSpans().AppendEmpty() + input.scope.Scope().CopyTo(scope.Scope()) + scope.SetSchemaUrl(input.scope.SchemaUrl()) + span.CopyTo(scope.Spans().AppendEmpty()) + span = scope.Spans().At(0) + if err := redact(spanContext{resource: resource, scope: scope, span: span}); err != nil { + return sdk.Span{}, err + } + encoded, err := native.MarshalProto() + if err != nil { + return sdk.Span{}, fmt.Errorf("encode OTLP span: %w", err) + } + jsonData, err := native.MarshalJSON() + if err != nil { + return sdk.Span{}, err + } + digest, err := nativeDigest(jsonData) + if err != nil { + return sdk.Span{}, err + } + parent := "" + if !span.ParentSpanID().IsEmpty() { + parent = span.ParentSpanID().String() + } + record := sdk.Span{ + TraceID: span.TraceID().String(), SpanID: span.SpanID().String(), ParentID: parent, + Operation: span.Name(), Kind: span.Kind().String(), + StartedAt: span.StartTimestamp().AsTime(), + DurationMs: float64(span.EndTimestamp()-span.StartTimestamp()) / float64(time.Millisecond), + Status: span.Status().Code().String(), + Attributes: span.Attributes().AsRaw(), Resource: resource.Resource().Attributes().AsRaw(), + Native: base64.StdEncoding.EncodeToString(encoded), Digest: digest, + Source: sdk.SpanSource{Type: sdk.SpanSourceOTLP}, + } + if err := record.Normalize(); err != nil { + return sdk.Span{}, fmt.Errorf("invalid span: %w", err) + } + return record, nil +} + +func redact(input spanContext) error { + span := input.span + maps := []pcommon.Map{input.resource.Resource().Attributes(), input.scope.Scope().Attributes(), span.Attributes()} + for i := 0; i < span.Events().Len(); i++ { + event := span.Events().At(i) + event.SetName(sdk.Mask(event.Name())) + maps = append(maps, event.Attributes()) + } + for i := 0; i < span.Links().Len(); i++ { + maps = append(maps, span.Links().At(i).Attributes()) + } + for _, attributes := range maps { + values := attributes.AsRaw() + if len(values) != attributes.Len() { + return fmt.Errorf("invalid span: duplicate attribute keys") + } + for key, value := range values { + values[key] = tracing.MaskAttribute(key, value) + } + if err := attributes.FromRaw(values); err != nil { + return fmt.Errorf("redact OTLP attributes: %w", err) + } + } + span.SetName(sdk.Mask(span.Name())) + span.Status().SetMessage(sdk.Mask(span.Status().Message())) + return nil +} diff --git a/pkg/proxy/endpoint/auth.go b/pkg/proxy/endpoint/auth.go new file mode 100644 index 0000000..049c7b3 --- /dev/null +++ b/pkg/proxy/endpoint/auth.go @@ -0,0 +1,80 @@ +package endpoint + +import ( + "context" + "encoding/base64" + "fmt" + "slices" + "strings" + + "github.com/flanksource/telltale/pkg/credentials/apitoken" +) + +type Auth struct { + Mode string `json:"mode,omitempty"` +} + +func (a Auth) Validate() error { + if a.Mode != "" && a.Mode != "passthrough" && a.Mode != "token" { + return fmt.Errorf("auth.mode must be passthrough or token") + } + return nil +} + +type Policy struct { + Name string `json:"name"` + Connection string `json:"connection,omitempty"` + Auth Auth `json:"auth"` +} + +func (p Policy) Validate() error { + if err := p.Auth.Validate(); err != nil { + return err + } + if p.Name == "" { + return fmt.Errorf("endpoint name is required") + } + if p.Auth.Mode == "token" && (!strings.HasPrefix(p.Connection, "connection://") || strings.TrimPrefix(p.Connection, "connection://") == "") { + return fmt.Errorf("token endpoint requires a connection:// backend") + } + return nil +} + +func (p Policy) Authorize(ctx context.Context, raw string, verifier *apitoken.Verifier) (string, error) { + if err := p.Validate(); err != nil { + return "", err + } + if p.Auth.Mode != "token" { + return "", nil + } + if verifier == nil { + return "", fmt.Errorf("token endpoint has no verifier") + } + record, err := verifier.VerifyScope(ctx, raw, apitoken.ScopeProxy) + if err != nil { + return "", err + } + if !slices.Contains(record.Endpoints, p.Name) { + return "", apitoken.ErrScope + } + return record.ID, nil +} + +func HTTPToken(header string) (string, error) { + if value, ok := apitoken.BearerFromHeader(header); ok { + return value, nil + } + scheme, value, ok := strings.Cut(strings.TrimSpace(header), " ") + if !ok || !strings.EqualFold(scheme, "Basic") { + return "", apitoken.ErrMalformed + } + decoded, err := base64.StdEncoding.DecodeString(strings.TrimSpace(value)) + if err != nil { + return "", apitoken.ErrMalformed + } + user, password, ok := strings.Cut(string(decoded), ":") + if !ok || user != "telltale" || password == "" { + return "", apitoken.ErrMalformed + } + return password, nil +} diff --git a/pkg/proxy/endpoint/auth_test.go b/pkg/proxy/endpoint/auth_test.go new file mode 100644 index 0000000..09ca98a --- /dev/null +++ b/pkg/proxy/endpoint/auth_test.go @@ -0,0 +1,75 @@ +package endpoint + +import ( + "context" + "encoding/base64" + "testing" + "time" + + "github.com/flanksource/telltale/pkg/credentials/apitoken" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestEndpoint(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Endpoint authentication") } + +var _ = Describe("endpoint authentication", func() { + It("isolates endpoints and checks revocation despite cached verification", func() { + minted, err := apitoken.Mint() + Expect(err).NotTo(HaveOccurred()) + expires := time.Now().Add(time.Hour) + record := apitoken.Record{ID: minted.ID, SecretHash: minted.Hash, Scope: apitoken.ScopeProxy, Endpoints: []string{"deploy"}, ExpiresAt: &expires} + verifier := apitoken.NewVerifier(func(_ context.Context, id string) (apitoken.Record, error) { + if id != record.ID { + return apitoken.Record{}, apitoken.ErrUnknown + } + return record, nil + }) + endpoint := Policy{Name: "deploy", Connection: "connection://deployment", Auth: Auth{Mode: "token"}} + id, err := endpoint.Authorize(context.Background(), minted.Secret.Value(), verifier) + Expect(err).NotTo(HaveOccurred()) + Expect(id).To(Equal(minted.ID)) + endpoint.Name = "other" + _, err = endpoint.Authorize(context.Background(), minted.Secret.Value(), verifier) + Expect(err).To(MatchError(apitoken.ErrScope)) + endpoint.Name = "deploy" + now := time.Now() + record.RevokedAt = &now + _, err = endpoint.Authorize(context.Background(), minted.Secret.Value(), verifier) + Expect(err).To(MatchError(apitoken.ErrRevoked)) + }) + It("never downgrades token authentication and rejects expiry and wrong scopes", func() { + minted, err := apitoken.Mint() + Expect(err).NotTo(HaveOccurred()) + expires := time.Now().Add(-time.Second) + record := apitoken.Record{ID: minted.ID, SecretHash: minted.Hash, Scope: apitoken.ScopeProxy, Endpoints: []string{"deploy"}, ExpiresAt: &expires} + verifier := apitoken.NewVerifier(func(context.Context, string) (apitoken.Record, error) { return record, nil }) + p := Policy{Name: "deploy", Connection: "connection://deployment", Auth: Auth{Mode: "token"}} + _, err = p.Authorize(context.Background(), "", verifier) + Expect(err).To(HaveOccurred()) + _, err = p.Authorize(context.Background(), minted.Secret.Value(), verifier) + Expect(err).To(MatchError(apitoken.ErrExpired)) + record.ExpiresAt = nil + record.Scope = apitoken.ScopeAPI + _, err = p.Authorize(context.Background(), minted.Secret.Value(), verifier) + Expect(err).To(MatchError(ContainSubstring(apitoken.ErrScope.Error()))) + }) + It("allows independent passthrough endpoints without a token verifier", func() { + p := Policy{Name: "public"} + _, err := p.Authorize(context.Background(), "", nil) + Expect(err).NotTo(HaveOccurred()) + p.Auth.Mode = "typo" + Expect(p.Validate()).NotTo(Succeed()) + p.Auth.Mode = "token" + Expect(p.Validate()).NotTo(Succeed()) + }) + It("accepts native proxy Basic and Bearer credentials", func() { + for _, header := range []string{"Bearer consumer-token", "Basic " + base64.StdEncoding.EncodeToString([]byte("telltale:consumer-token"))} { + got, err := HTTPToken(header) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(Equal("consumer-token")) + } + _, err := HTTPToken("Basic invalid") + Expect(err).To(MatchError(apitoken.ErrMalformed)) + }) +}) diff --git a/pkg/sessions/authorize.go b/pkg/sessions/authorize.go new file mode 100644 index 0000000..934ac97 --- /dev/null +++ b/pkg/sessions/authorize.go @@ -0,0 +1,160 @@ +package sessions + +import ( + "context" + "errors" + "fmt" + "net/http" + "sync" + + "github.com/flanksource/commons-db/query" +) + +// Action is what a caller does with a session: read it, or control it. +type Action string + +const ( + // ActionRead lists a session, or serves its info, events or result. + ActionRead Action = "read" + // ActionControl starts, stops, extends or restarts a session. + ActionControl Action = "control" + // ActionCheckpoint has a running trace capture take a reading now and + // commit what it emitted, without changing when or whether it ends; a + // host serving captures read-only may still allow it. + ActionCheckpoint Action = "checkpoint" +) + +// AuthorizeFunc decides whether r may perform action on a session of profile. +// A non-nil error refuses. On a route naming a session id, a caller who may not +// read the session is answered 404 exactly as for a session that does not +// exist; a caller who may read it but not control or checkpoint it gets 403 with the error. +// Where the caller named the profile itself, a refusal is 403 with the error. +type AuthorizeFunc func(r *http.Request, profile string, action Action) error + +// ErrReadOnly refuses a control a ReadOnly host does not serve. +var ErrReadOnly = errors.New("sessions are served read-only here") + +// ReadOnly lets every caller read and checkpoint sessions but control none: +// a host serving a trace store whose captures it starts and stops itself. +func ReadOnly(_ *http.Request, profile string, action Action) error { + if action == ActionControl { + return fmt.Errorf("%w: %s of %s", ErrReadOnly, action, profile) + } + return nil +} + +// authorized asks Options.Authorize whether r may perform action on a session +// of profile, answering a refusal 403 itself. It serves routes where the caller +// named the profile, so a refusal tells it nothing it did not already know. +func (h *sessionHandler) authorized(w http.ResponseWriter, r *http.Request, profile string, action Action) bool { + if err := h.refusal(r, profile, action); err != nil { + http.Error(w, err.Error(), http.StatusForbidden) + return false + } + return true +} + +// readAllow is SessionFilter.Allow for r: Authorize for reading, asked once per +// profile for the whole request. Nil when every profile is allowed. +func (h *sessionHandler) readAllow(r *http.Request) func(profile string) bool { + if h.authorize == nil { + return nil + } + var mu sync.Mutex + decided := map[string]bool{} + return func(profile string) bool { + mu.Lock() + defer mu.Unlock() + allowed, ok := decided[profile] + if !ok { + allowed = h.authorize(r, profile, ActionRead) == nil + decided[profile] = allowed + } + return allowed + } +} + +// resolvedSession is a session found live in the registry or only in the +// store; Live is nil for the latter. +type resolvedSession struct { + Live *query.Session + Info query.SessionInfo +} + +// authorizedSession resolves session id — live from the registry, or from the +// session store — and authorizes r for action on its profile before anything +// of it is used, with the info's overlays derived. It writes the response +// itself when the session is unknown, refused or cannot be read. +// +// A session r may not read is answered 404 with the body a missing session +// gets, so an id alone never tells a caller that a session it cannot see +// exists. Only a caller who may read it is told 403 for a control it may not +// perform. +func (h *sessionHandler) authorizedSession(w http.ResponseWriter, r *http.Request, id string, action Action) (resolvedSession, bool) { + ctx := withSessionReadMemo(r.Context()) + resolved, err := h.resolve(ctx, id) + var notFound errSessionNotFound + switch { + case errors.As(err, ¬Found): + http.Error(w, err.Error(), http.StatusNotFound) + return resolved, false + case err != nil: + http.Error(w, err.Error(), http.StatusInternalServerError) + return resolved, false + } + if refusal := h.refusal(r, resolved.Info.Profile, action); refusal != nil { + status, body := http.StatusNotFound, errSessionNotFound(id).Error() + if action != ActionRead && h.authorize(r, resolved.Info.Profile, ActionRead) == nil { + status, body = http.StatusForbidden, refusal.Error() + } + http.Error(w, body, status) + return resolved, false + } + infos, err := h.overlay(ctx, []query.SessionRecord{resolved.Info.SessionRecord}, h.readAllow(r)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return resolved, false + } + resolved.Info = infos[0] + return resolved, true +} + +// refusal is Authorize's refusal of action on profile for r, or nil. +func (h *sessionHandler) refusal(r *http.Request, profile string, action Action) error { + if h.authorize == nil { + return nil + } + return h.authorize(r, profile, action) +} + +type errSessionNotFound string + +func (e errSessionNotFound) Error() string { return fmt.Sprintf("session %q not found", string(e)) } + +// resolve finds session id live in the registry, or else in the session store. +// A live session outside the scope ctx names is not found: it is never looked +// up in the store either, whose copy of it would show it across scopes. +func (h *sessionHandler) resolve(ctx context.Context, id string) (resolvedSession, error) { + visible, err := h.registry.Visible(ctx) + if err != nil { + return resolvedSession{}, err + } + if session, ok := h.registry.Get(id); ok { + info := session.Snapshot() + if !visible(info.SessionRecord) { + return resolvedSession{}, errSessionNotFound(id) + } + return resolvedSession{Live: session, Info: info}, nil + } + if h.sessions == nil { + return resolvedSession{}, errSessionNotFound(id) + } + rec, found, err := h.sessions.Get(ctx, id) + switch { + case err != nil: + return resolvedSession{}, err + case !found: + return resolvedSession{}, errSessionNotFound(id) + } + return resolvedSession{Info: query.SessionInfo{SessionRecord: rec}}, nil +} diff --git a/pkg/sessions/authorize_test.go b/pkg/sessions/authorize_test.go new file mode 100644 index 0000000..adc70c2 --- /dev/null +++ b/pkg/sessions/authorize_test.go @@ -0,0 +1,153 @@ +package sessions + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "time" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + profilepkg "github.com/flanksource/commons-db/cmd/query/profiles" + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" +) + +var _ = ginkgo.Describe("stream sessions authorized per profile", func() { + const ( + providerType = "sess-authz-stream" + allowed = "allowed-trace" + denied = "denied-trace" + ) + + var ( + handler *sessionHandler + registry *query.SessionRegistry + store *Store + ) + + authorize := func(_ *http.Request, profile string, _ Action) error { + if profile != allowed { + return fmt.Errorf("profile %q is not readable by this caller", profile) + } + return nil + } + + request := func(method, path string) *httptest.ResponseRecorder { + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(method, path, nil)) + return response + } + + ginkgo.BeforeEach(func() { + query.RegisterProvider(&sessionStreamMock{typ: providerType, rows: []query.Row{{"n": 1.0}}, block: true}) + profiles, err := profilepkg.NewFileStore(ginkgo.GinkgoT().TempDir()) + Expect(err).ToNot(HaveOccurred()) + for _, name := range []string{allowed, denied} { + Expect(profiles.Save(context.Background(), traceTestProfile(name, providerType))).To(Succeed()) + } + store = newGormStore(sessionStoreDB(), time.Hour) + registry = query.NewSessionRegistry(query.RegistryOptions{Store: store, Events: store}) + store.BindResolver(registry.Get) + ginkgo.DeferCleanup(func() { Expect(registry.StopAll(context.Background())).To(Succeed()) }) + handler = newSessionHandler(sessionHandlerOptions{ + Prefix: "/api/v1", Ctx: dbcontext.New(), Store: profiles, Registry: registry, Sessions: store, + EventLog: store, Authorize: authorize, Next: &nextMarker{}, + }) + }) + + // liveSession starts a session on profile the way another caller would, + // past this caller's authorization. + liveSession := func(profile string) string { + session, err := query.ExecuteStream(dbcontext.New(), registry, traceTestProfile(profile, providerType)) + Expect(err).ToNot(HaveOccurred()) + Eventually(func() int64 { return session.Snapshot().EventCount }).Should(BeNumerically(">=", 1)) + return session.ID() + } + + // persistedSession records a finished stream session on profile that only + // the durable store still knows. + persistedSession := func(id, profile string) string { + started := time.Now() + Expect(store.Begin(context.Background(), query.SessionRecord{ + SessionStart: query.SessionStart{ID: id, Profile: profile, Kind: query.KindTrace, Role: query.SessionRoleCapture, StartedAt: started}, + SessionStatus: query.SessionStatus{State: query.SessionCompleted, EventCount: 1, StoppedAt: &started, + UpdatedAt: started, HeartbeatAt: started}, + })).To(Succeed()) + Expect(store.Append(context.Background(), query.Event{SessionID: id, Sequence: 1, Time: started, Row: query.Row{"n": 1.0}})).To(Succeed()) + Expect(store.Flush()).To(Succeed()) + return id + } + + expectAccess := func(id string, status int) { + base := "/api/v1/sessions/" + id + for _, path := range []string{base, base + "/events?format=ndjson", base + "/result"} { + response := request(http.MethodGet, path) + Expect(response.Code).To(Equal(status), "%s: %s", path, response.Body.String()) + if status == http.StatusNotFound { + Expect(response.Body.String()).To(Equal(errSessionNotFound(id).Error()+"\n"), "a refusal reads as a missing session: %s", path) + } + } + } + + eventsAvailable := func(id string) bool { + response := request(http.MethodGet, "/api/v1/sessions/"+id) + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + return decodeInfo(response).EventsAvailable + } + + listed := func() []string { + response := request(http.MethodGet, "/api/v1/sessions") + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + var body sessionListResponse + Expect(json.Unmarshal(response.Body.Bytes(), &body)).To(Succeed()) + return infoIDs(body.Items) + } + + ginkgo.It("refuses a live session on a profile the caller may not read, and serves one it may", func() { + deniedID, allowedID := liveSession(denied), liveSession(allowed) + + expectAccess(deniedID, http.StatusNotFound) + Expect(request(http.MethodPost, "/api/v1/sessions/"+deniedID+"/stop").Code).To(Equal(http.StatusNotFound)) + session, ok := registry.Get(deniedID) + Expect(ok).To(BeTrue()) + Expect(session.Snapshot().State.Terminal()).To(BeFalse(), "a refused stop must not stop the session") + + Expect(listed()).To(And(ContainElement(allowedID), Not(ContainElement(deniedID)))) + expectAccess(allowedID, http.StatusOK) + }) + + ginkgo.It("refuses a persisted session on a profile the caller may not read, and serves one it may", func() { + deniedID := persistedSession("persisted-denied", denied) + allowedID := persistedSession("persisted-allowed", allowed) + + expectAccess(deniedID, http.StatusNotFound) + Expect(listed()).To(And(ContainElement(allowedID), Not(ContainElement(deniedID)))) + expectAccess(allowedID, http.StatusOK) + }) + + ginkgo.It("reports a persisted stream session's events available only when the event log holds some", func() { + withEvents := persistedSession("persisted-events", allowed) + started := time.Now() + Expect(store.Begin(context.Background(), query.SessionRecord{ + SessionStart: query.SessionStart{ID: "persisted-silent", Profile: allowed, Kind: query.KindTrace, Role: query.SessionRoleCapture, StartedAt: started}, + SessionStatus: query.SessionStatus{State: query.SessionCompleted, StoppedAt: &started, UpdatedAt: started, HeartbeatAt: started}, + })).To(Succeed()) + + Expect(eventsAvailable(withEvents)).To(BeTrue()) + Expect(eventsAvailable("persisted-silent")).To(BeFalse(), "an event log being configured is not the session having events") + }) + + ginkgo.It("refuses to start a session on a profile the caller may not control", func() { + response := request(http.MethodPost, "/api/v1/profile/"+denied+"/sessions") + Expect(response.Code).To(Equal(http.StatusForbidden), response.Body.String()) + Expect(response.Body.String()).To(ContainSubstring("not readable by this caller")) + Expect(registry.List()).To(BeEmpty()) + + response = request(http.MethodPost, "/api/v1/profile/"+allowed+"/sessions") + Expect(response.Code).To(Equal(http.StatusCreated), response.Body.String()) + }) +}) diff --git a/pkg/sessions/connection_trace.go b/pkg/sessions/connection_trace.go new file mode 100644 index 0000000..1f38184 --- /dev/null +++ b/pkg/sessions/connection_trace.go @@ -0,0 +1,123 @@ +package sessions + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/models" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/types" + "github.com/flanksource/telltale/pkg/tracing/sqlserver/xeprovider" + "github.com/google/uuid" +) + +// connectionTraceProfile is the virtual profile a connection's trace sessions +// are authorized and recorded under. +func connectionTraceProfile(id uuid.UUID) string { + return "connection-" + id.String() + "-sql-xevent" +} + +// connectionTraceInput is the body a connection trace is started with. +type connectionTraceInput struct { + Database string `json:"database"` + Users []string `json:"users"` + Apps []string `json:"apps"` + Hosts []string `json:"hosts"` + Events []string `json:"events"` + MinDuration string `json:"minDuration"` + Duration string `json:"duration"` +} + +// providerOptions are the sqlserver-xevent options the input asks for. The +// form's database is one database pattern; left blank, the capture scopes to +// the connection's own database. +func (input connectionTraceInput) providerOptions(sessionName string) map[string]any { + options := map[string]any{ + "sessionName": sessionName, + "users": input.Users, "apps": input.Apps, "hosts": input.Hosts, + "events": input.Events, "minDuration": input.MinDuration, + } + if database := strings.TrimSpace(input.Database); database != "" { + options["databases"] = []string{database} + } + return options +} + +// startConnectionTrace creates an ephemeral profile in the shared session +// registry. Only POST starts capture; connection discovery remains read-only. +func (h *sessionHandler) startConnectionTrace(w http.ResponseWriter, r *http.Request, id string) { + var input connectionTraceInput + decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&input); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + duration := 5 * time.Minute + if input.Duration != "" { + var err error + duration, err = time.ParseDuration(input.Duration) + if err != nil || duration <= 0 { + http.Error(w, "duration must be positive (for example 5m)", http.StatusBadRequest) + return + } + } + reference, conn, ok := h.traceableConnection(w, r, id) + if !ok { + return + } + if conn.Type != models.ConnectionTypeSQLServer { + http.Error(w, "XEvents tracing requires a SQL Server connection", http.StatusBadRequest) + return + } + profile := query.Profile{ + Name: connectionTraceProfile(conn.ID), Virtual: true, + // The server-side session carries the same name this connection's trace + // profile does, so a DBA reading sys.dm_xe_sessions can tell which + // connection asked for it. + Provider: query.ProviderConfig{Type: xeprovider.SQLXEventProviderType, Connection: reference, + Options: input.providerOptions(connectionTraceProfile(conn.ID))}, + Trace: &query.TraceSpec{MaxDuration: types.Duration{Duration: duration}}, + } + session, err := query.ExecuteStream(h.sessionContext(r), h.registry, profile) + if err != nil { + status := http.StatusBadRequest + if errors.Is(err, query.ErrMaxSessions) { + status = http.StatusConflict + } + http.Error(w, err.Error(), status) + return + } + h.writeLive(w, r, http.StatusCreated, session) +} + +// traceableConnection resolves the connection id names, with the reference it +// was looked up by, once r may start a trace of it. A connection a caller may +// not trace is answered exactly as one that does not exist: a uuid names the +// virtual profile before anything is looked up, so it is authorized first; a +// name is authorized once it resolves to one. +func (h *sessionHandler) traceableConnection(w http.ResponseWriter, r *http.Request, id string) (string, *models.Connection, bool) { + reference, notFound := id, fmt.Sprintf("connection %q not found", id) + if parsed, err := uuid.Parse(id); err != nil { + reference = "connection://" + id + notFound = fmt.Sprintf("connection %q not found", reference) + } else if h.refusal(r, connectionTraceProfile(parsed), ActionControl) != nil { + http.Error(w, notFound, http.StatusNotFound) + return "", nil, false + } + conn, err := dbcontext.HydrateConnectionByURL(h.ctx.Wrap(r.Context()), reference) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return "", nil, false + } + if h.refusal(r, connectionTraceProfile(conn.ID), ActionControl) != nil { + http.Error(w, notFound, http.StatusNotFound) + return "", nil, false + } + return reference, conn, true +} diff --git a/pkg/sessions/connection_trace_test.go b/pkg/sessions/connection_trace_test.go new file mode 100644 index 0000000..c76895a --- /dev/null +++ b/pkg/sessions/connection_trace_test.go @@ -0,0 +1,23 @@ +package sessions + +import ( + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = ginkgo.Describe("connection trace provider options", func() { + ginkgo.It("scopes to the database the form named, as a database pattern", func() { + options := connectionTraceInput{Database: " warehouse ", Users: []string{"svc"}}.providerOptions("trace") + + Expect(options).To(HaveKeyWithValue("databases", []string{"warehouse"})) + Expect(options).To(HaveKeyWithValue("sessionName", "trace")) + Expect(options).To(HaveKeyWithValue("users", []string{"svc"})) + Expect(options).NotTo(HaveKey("database")) + }) + + ginkgo.It("leaves the scope to the connection's own database when the form names none", func() { + options := connectionTraceInput{Database: " "}.providerOptions("trace") + + Expect(options).NotTo(HaveKey("databases")) + }) +}) diff --git a/pkg/sessions/internal/paramfile/paramfile.go b/pkg/sessions/internal/paramfile/paramfile.go new file mode 100644 index 0000000..01c3946 --- /dev/null +++ b/pkg/sessions/internal/paramfile/paramfile.go @@ -0,0 +1,199 @@ +// Package paramfile expands a --param value that points at a file of values. +// +// It deliberately lives under cmd/query/internal so the compiler keeps it out of +// reach of anything the server links: reading a path named by an HTTP request +// would be an arbitrary local-file read. Every caller here runs in the operator's +// own process, from their own shell. +package paramfile + +import ( + "encoding/csv" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" +) + +// Load expands one --param value. A value that does not start with "@" comes +// back unchanged as a single element; "@path[#selector]" reads path, where the +// selector names a CSV header column or a JSON object key. +// +// A "!" prefix survives, so a file may carry exclusions and reads exactly like +// the comma-joined form typed on the command line. +func Load(value string) ([]string, error) { + if !strings.HasPrefix(value, "@") { + return []string{value}, nil + } + path, selector, _ := strings.Cut(strings.TrimPrefix(value, "@"), "#") + if strings.TrimSpace(path) == "" { + return nil, fmt.Errorf("@ must be followed by a file path") + } + content, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read param file: %w", err) + } + + var values []string + switch ext := strings.ToLower(filepath.Ext(path)); ext { + case ".csv": + values, err = parseCSV(content, selector) + case ".json": + values, err = parseJSON(content, selector) + case ".txt": + values, err = parseLines(content, selector) + default: + return nil, fmt.Errorf("unsupported param file %q: expected .csv, .json or .txt", path) + } + if err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + + values = clean(values) + if len(values) == 0 { + return nil, fmt.Errorf("%s contained no values", path) + } + return values, nil +} + +// parseCSV reads one column. The first record is always treated as a header, so +// a named selector can resolve; a headerless file should use .txt instead. +func parseCSV(content []byte, selector string) ([]string, error) { + reader := csv.NewReader(strings.NewReader(string(content))) + reader.FieldsPerRecord = -1 + records, err := reader.ReadAll() + if err != nil { + return nil, fmt.Errorf("parse CSV: %w", err) + } + if len(records) == 0 { + return nil, nil + } + + header := records[0] + index := 0 + if selector != "" { + index = -1 + for i, name := range header { + if strings.EqualFold(strings.TrimSpace(name), selector) { + index = i + break + } + } + if index < 0 { + return nil, fmt.Errorf("column %q not found (have: %s)", selector, strings.Join(header, ", ")) + } + } + + values := make([]string, 0, len(records)-1) + for _, record := range records[1:] { + if index < len(record) { + values = append(values, record[index]) + } + } + return values, nil +} + +// parseJSON reads a flat string array, or an array of objects when a selector +// names the key to read. +func parseJSON(content []byte, selector string) ([]string, error) { + var decoded any + if err := json.Unmarshal(content, &decoded); err != nil { + return nil, fmt.Errorf("parse JSON: %w", err) + } + items, ok := decoded.([]any) + if !ok { + return nil, fmt.Errorf("expected a JSON array, found %s", jsonKind(decoded)) + } + + values := make([]string, 0, len(items)) + for i, item := range items { + switch typed := item.(type) { + case string: + if selector != "" { + return nil, fmt.Errorf("selector %q needs an array of objects, but item %d is a string", selector, i) + } + values = append(values, typed) + case map[string]any: + if selector == "" { + return nil, fmt.Errorf("item %d is an object; name the key to read with #key", i) + } + raw, present := typed[selector] + if !present { + return nil, fmt.Errorf("item %d has no key %q", i, selector) + } + text, ok := raw.(string) + if !ok { + return nil, fmt.Errorf("item %d key %q is %s, not a string", i, selector, jsonKind(raw)) + } + values = append(values, text) + default: + return nil, fmt.Errorf("item %d is %s, not a string or an object", i, jsonKind(item)) + } + } + return values, nil +} + +func parseLines(content []byte, selector string) ([]string, error) { + if selector != "" { + return nil, fmt.Errorf("a .txt file holds one value per line and takes no #selector") + } + return strings.Split(strings.ReplaceAll(string(content), "\r\n", "\n"), "\n"), nil +} + +// clean trims, drops blanks, and dedupes while preserving first-seen order, so a +// file and a hand-typed list produce the same selection. +func clean(values []string) []string { + seen := make(map[string]bool, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" || seen[value] { + continue + } + seen[value] = true + out = append(out, value) + } + return out +} + +func jsonKind(v any) string { + switch v.(type) { + case nil: + return "null" + case bool: + return "a boolean" + case float64: + return "a number" + case string: + return "a string" + case []any: + return "an array" + case map[string]any: + return "an object" + default: + return fmt.Sprintf("%T", v) + } +} + +// Parse turns repeatable key=value flags into the param map query.Execute takes, +// expanding any @file reference. A plain value stays a string so a scalar param +// behaves exactly as it always has; an expanded file becomes a []string. +func Parse(pairs []string) (map[string]any, error) { + params := make(map[string]any, len(pairs)) + for _, pair := range pairs { + key, value, ok := strings.Cut(pair, "=") + if key = strings.TrimSpace(key); !ok || key == "" { + return nil, fmt.Errorf("invalid --param %q: expected key=value", pair) + } + if !strings.HasPrefix(value, "@") { + params[key] = value + continue + } + values, err := Load(value) + if err != nil { + return nil, fmt.Errorf("param %q: %w", key, err) + } + params[key] = values + } + return params, nil +} diff --git a/pkg/sessions/internal/paramfile/paramfile_test.go b/pkg/sessions/internal/paramfile/paramfile_test.go new file mode 100644 index 0000000..ca7cca9 --- /dev/null +++ b/pkg/sessions/internal/paramfile/paramfile_test.go @@ -0,0 +1,212 @@ +package paramfile_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/flanksource/telltale/pkg/sessions/internal/paramfile" +) + +// write drops content into a temp file with the given name and returns its path. +func write(t *testing.T, name, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + return path +} + +func equal(t *testing.T, got, want []string) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("got %q, want %q", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("got %q, want %q", got, want) + } + } +} + +func TestLoadPassesThroughAPlainValue(t *testing.T) { + got, err := paramfile.Load("us-east,!eu") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + equal(t, got, []string{"us-east,!eu"}) +} + +func TestLoadCSVReadsTheFirstColumnByDefault(t *testing.T) { + path := write(t, "accounts.csv", "account_id,region\nA-1,us-east\nA-2,eu\n") + + got, err := paramfile.Load("@" + path) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + equal(t, got, []string{"A-1", "A-2"}) +} + +func TestLoadCSVSelectsANamedColumnCaseInsensitively(t *testing.T) { + path := write(t, "accounts.csv", "account_id,region\nA-1,us-east\nA-2,eu\n") + + got, err := paramfile.Load("@" + path + "#Region") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + equal(t, got, []string{"us-east", "eu"}) +} + +func TestLoadCSVKeepsQuotedFieldsContainingCommas(t *testing.T) { + path := write(t, "accounts.csv", "name\n\"Acme, Inc\"\nBeta\n") + + got, err := paramfile.Load("@" + path) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + equal(t, got, []string{"Acme, Inc", "Beta"}) +} + +func TestLoadCSVNamesTheAvailableColumnsWhenTheSelectorIsWrong(t *testing.T) { + path := write(t, "accounts.csv", "account_id,region\nA-1,us-east\n") + + _, err := paramfile.Load("@" + path + "#missing") + if err == nil { + t.Fatal("expected an error naming the available columns") + } + for _, want := range []string{"missing", "account_id", "region"} { + if !contains(err.Error(), want) { + t.Fatalf("error %q does not mention %q", err, want) + } + } +} + +func TestLoadJSONReadsAFlatStringArray(t *testing.T) { + path := write(t, "ids.json", `["A-1","A-2"]`) + + got, err := paramfile.Load("@" + path) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + equal(t, got, []string{"A-1", "A-2"}) +} + +func TestLoadJSONReadsAKeyFromAnObjectArray(t *testing.T) { + path := write(t, "ids.json", `[{"id":"A-1","region":"eu"},{"id":"A-2","region":"us"}]`) + + got, err := paramfile.Load("@" + path + "#id") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + equal(t, got, []string{"A-1", "A-2"}) +} + +func TestLoadJSONRequiresASelectorForObjectItems(t *testing.T) { + path := write(t, "ids.json", `[{"id":"A-1"}]`) + + _, err := paramfile.Load("@" + path) + if err == nil || !contains(err.Error(), "#key") { + t.Fatalf("expected an error pointing at #key, got %v", err) + } +} + +func TestLoadJSONRejectsATopLevelObject(t *testing.T) { + path := write(t, "ids.json", `{"id":"A-1"}`) + + _, err := paramfile.Load("@" + path) + if err == nil || !contains(err.Error(), "an object") { + t.Fatalf("expected an error naming what was found, got %v", err) + } +} + +func TestLoadTxtReadsOneValuePerLine(t *testing.T) { + path := write(t, "ids.txt", "A-1\r\nA-2\n\nA-3\n") + + got, err := paramfile.Load("@" + path) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + equal(t, got, []string{"A-1", "A-2", "A-3"}) +} + +func TestLoadTrimsDedupesAndPreservesFirstSeenOrder(t *testing.T) { + path := write(t, "ids.txt", " B \nA\nB\n A \nC\n") + + got, err := paramfile.Load("@" + path) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + equal(t, got, []string{"B", "A", "C"}) +} + +func TestLoadPreservesAnExclusionPrefix(t *testing.T) { + path := write(t, "ids.txt", "A-1\n!A-2\n") + + got, err := paramfile.Load("@" + path) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + equal(t, got, []string{"A-1", "!A-2"}) +} + +func TestLoadRejectsAnUnsupportedExtension(t *testing.T) { + path := write(t, "ids.yaml", "- A-1\n") + + _, err := paramfile.Load("@" + path) + if err == nil || !contains(err.Error(), ".csv, .json or .txt") { + t.Fatalf("expected an error listing the supported extensions, got %v", err) + } +} + +func TestLoadRejectsAFileWithNoValues(t *testing.T) { + path := write(t, "ids.txt", "\n \n") + + _, err := paramfile.Load("@" + path) + if err == nil || !contains(err.Error(), "no values") { + t.Fatalf("expected an empty-file error, got %v", err) + } +} + +func TestLoadReportsAMissingFile(t *testing.T) { + _, err := paramfile.Load("@" + filepath.Join(t.TempDir(), "absent.csv")) + if err == nil || !contains(err.Error(), "read param file") { + t.Fatalf("expected a read error, got %v", err) + } +} + +func TestParseKeepsScalarsAsStringsAndExpandsFiles(t *testing.T) { + path := write(t, "ids.txt", "A-1\nA-2\n") + + got, err := paramfile.Parse([]string{"region=eu", "ids=@" + path}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got["region"] != "eu" { + t.Fatalf("scalar param changed shape: %#v", got["region"]) + } + values, ok := got["ids"].([]string) + if !ok { + t.Fatalf("expanded param is %T, want []string", got["ids"]) + } + equal(t, values, []string{"A-1", "A-2"}) +} + +func TestParseRejectsAPairWithoutAnEquals(t *testing.T) { + if _, err := paramfile.Parse([]string{"region"}); err == nil { + t.Fatal("expected an error for a malformed pair") + } +} + +func TestParseKeepsAValueContainingEquals(t *testing.T) { + got, err := paramfile.Parse([]string{"filter=a=b"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got["filter"] != "a=b" { + t.Fatalf("got %#v, want %q", got["filter"], "a=b") + } +} + +func contains(haystack, needle string) bool { return strings.Contains(haystack, needle) } diff --git a/pkg/sessions/internal/sse/sse.go b/pkg/sessions/internal/sse/sse.go new file mode 100644 index 0000000..2bef21a --- /dev/null +++ b/pkg/sessions/internal/sse/sse.go @@ -0,0 +1,120 @@ +// Package sse holds the server-sent-events wire format shared by every stream +// this server serves — session events and the devtools console alike. +// +// It is deliberately only the wire: framing, the resume header, and the +// flush-and-keepalive contract. What to stream, and when a stream is over, is +// the caller's business. +package sse + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" +) + +// DefaultKeepalive is how often an otherwise silent stream reassures the +// proxies between here and the browser that the connection is alive. A trace +// that ran for seconds never needed one; a tail can sit idle for minutes, and +// the common idle timeout in front of it is a minute. +const DefaultKeepalive = 20 * time.Second + +// KeepaliveFrame is a comment frame: the one thing that can be written into an +// event stream without being an event. +// +// Write it from the same goroutine — ideally the same select — that writes +// event frames. A ping emitted from a timer goroutine can land between a +// frame's id and its data, which splits the frame and desynchronises the +// client for the life of the connection. +const KeepaliveFrame = ": ping\n\n" + +// Frame is one wire frame. ID carries the event's sequence, which is what a +// reconnecting browser echoes back in Last-Event-ID — a frame without it is a +// frame the client cannot resume from, so only a terminal frame (which nobody +// resumes into) leaves it unset. +type Frame struct { + Event string + ID int64 + Data any +} + +// Begin writes the event-stream headers and returns the flusher the stream +// must call after every frame. +// +// It lifts the server's write deadline for this response. A stream is open for +// as long as it has something to follow, and a server WriteTimeout sized for +// ordinary requests would otherwise cut every stream at that age. A writer that +// cannot set deadlines has none to lift; any other failure is returned, and the +// caller should end the response. +// +// It panics rather than degrading when the writer cannot flush: an unflushed +// event stream looks like a hung request, which is far harder to diagnose than +// a stack trace naming the middleware that swallowed the Flusher. +func Begin(w http.ResponseWriter) (http.Flusher, error) { + flusher, ok := w.(http.Flusher) + if !ok { + panic("server-sent events require a flushable ResponseWriter") + } + if err := http.NewResponseController(w).SetWriteDeadline(time.Time{}); err != nil && !errors.Is(err, http.ErrNotSupported) { + return nil, fmt.Errorf("lift the event stream's write deadline: %w", err) + } + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Access-Control-Allow-Origin", "*") + return flusher, nil +} + +// WriteFrame writes one frame; an error means the client disconnected and the +// stream should stop. +func WriteFrame(w io.Writer, frame Frame) error { + data, err := json.Marshal(frame.Data) + if err != nil { + data = []byte(fmt.Sprintf(`{"error":%q}`, err.Error())) + } + var out strings.Builder + if frame.ID > 0 { + fmt.Fprintf(&out, "id: %d\n", frame.ID) + } + fmt.Fprintf(&out, "event: %s\ndata: %s\n\n", frame.Event, data) + _, err = io.WriteString(w, out.String()) + return err +} + +// LastEventSequence reads the resume header a browser sends after a dropped +// connection. Anything that is not a sequence is a client the server cannot +// answer correctly — replaying the whole ring instead would silently duplicate +// every event it already has. +func LastEventSequence(r *http.Request) (int64, error) { + raw := strings.TrimSpace(r.Header.Get("Last-Event-ID")) + if raw == "" { + return 0, nil + } + sequence, err := strconv.ParseInt(raw, 10, 64) + if err != nil || sequence < 0 { + return 0, fmt.Errorf("invalid Last-Event-ID %q: expected an event sequence", raw) + } + return sequence, nil +} + +// BeginNDJSON sets the headers of a downloadable NDJSON file and returns the +// encoder its lines are written with, one Encode per line, as they are read. +func BeginNDJSON(w http.ResponseWriter, filename string) *json.Encoder { + w.Header().Set("Content-Type", "application/x-ndjson") + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", filename)) + return json.NewEncoder(w) +} + +// WriteNDJSON serves the same events as a downloadable file, for a caller that +// wants the stream's contents without following it. +func WriteNDJSON[T any](w http.ResponseWriter, filename string, items []T) { + encoder := BeginNDJSON(w, filename) + for _, item := range items { + if err := encoder.Encode(item); err != nil { + return + } + } +} diff --git a/pkg/sessions/internal/sse/sse_test.go b/pkg/sessions/internal/sse/sse_test.go new file mode 100644 index 0000000..d9b67bc --- /dev/null +++ b/pkg/sessions/internal/sse/sse_test.go @@ -0,0 +1,60 @@ +package sse_test + +import ( + "bufio" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/telltale/pkg/sessions/internal/sse" +) + +func TestSSE(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "SSE Suite") +} + +var _ = Describe("Begin", func() { + It("lifts the server's write timeout, so a stream outlives it", func() { + const writeTimeout = 100 * time.Millisecond + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + defer GinkgoRecover() + flusher, err := sse.Begin(w) + Expect(err).ToNot(HaveOccurred()) + for index := int64(1); index <= 3; index++ { + time.Sleep(writeTimeout) + Expect(sse.WriteFrame(w, sse.Frame{Event: "event", ID: index, Data: index})).To(Succeed()) + flusher.Flush() + } + })) + server.Config.WriteTimeout = writeTimeout + server.Start() + DeferCleanup(server.Close) + + response, err := http.Get(server.URL) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(response.Body.Close) + var ids []string + scanner := bufio.NewScanner(response.Body) + for scanner.Scan() { + if id, ok := strings.CutPrefix(scanner.Text(), "id: "); ok { + ids = append(ids, id) + } + } + Expect(scanner.Err()).ToNot(HaveOccurred()) + Expect(ids).To(Equal([]string{"1", "2", "3"})) + }) + + It("streams through a writer that cannot set deadlines", func() { + recorder := httptest.NewRecorder() + flusher, err := sse.Begin(recorder) + Expect(err).ToNot(HaveOccurred()) + flusher.Flush() + Expect(recorder.Header().Get("Content-Type")).To(Equal("text/event-stream")) + }) +}) diff --git a/pkg/sessions/prepare_test.go b/pkg/sessions/prepare_test.go new file mode 100644 index 0000000..89f8732 --- /dev/null +++ b/pkg/sessions/prepare_test.go @@ -0,0 +1,92 @@ +package sessions + +import ( + "bytes" + "context" + "errors" + "fmt" + "net/http" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + profilepkg "github.com/flanksource/commons-db/cmd/query/profiles" + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/query/profilestore" +) + +var _ = ginkgo.Describe("a session over a profile whose data is prepared per read", func() { + var ( + store profilestore.Store + hookErr error + hooked chan string + ) + + hook := func(_ context.Context, reads []profilestore.ReadRequest) (func(), error) { + hooked <- fmt.Sprintf("%s region=%v", reads[0].Profile.Name, reads[0].Params["region"]) + return func() { hooked <- "release" }, hookErr + } + + ginkgo.BeforeEach(func() { + hookErr, hooked = nil, make(chan string, 10) + query.RegisterProvider(&execMock{rows: []query.Row{{"id": 1}}}) + fileStore, err := profilepkg.NewFileStore(ginkgo.GinkgoT().TempDir()) + Expect(err).ToNot(HaveOccurred()) + Expect(fileStore.Save(context.Background(), execProfile("plain-top"))).To(Succeed()) + store = fileStore + }) + + start := func() (int, string) { + registry := query.NewSessionRegistry(query.RegistryOptions{BeforeRead: func( + ctx context.Context, p query.Profile, params map[string]any, + ) (func(), error) { + return profilestore.PrepareReads(ctx, hook, []profilestore.ReadRequest{{Profile: p, Params: params}}) + }}) + ginkgo.DeferCleanup(registry.StopAll) + handler := newSessionHandler(sessionHandlerOptions{ + Prefix: "/api/v1", Ctx: dbcontext.New(), Store: store, Registry: registry, Next: &nextMarker{}, + }) + response := doReq(handler, http.MethodPost, "/api/v1/profile/plain-top/sessions?interval=1s®ion=EU") + return response.Code, response.Body.String() + } + + ginkgo.It("prepares the data with the request's params before the session starts", func() { + code, body := start() + Expect(code).To(Equal(http.StatusCreated), body) + Eventually(hooked).Should(Receive(Equal("plain-top region=EU"))) + Eventually(hooked).Should(Receive(Equal("release"))) + }) + + ginkgo.DescribeTable("answers a start it could not prepare by the hook's cause", + func(cause error, status int) { + hookErr = cause + code, body := start() + Expect(code).To(Equal(status), body) + Expect(body).To(ContainSubstring("run-9")) + Expect(hooked).To(Receive(Equal("plain-top region=EU"))) + Expect(hooked).To(Receive(Equal("release"))) + }, + ginkgo.Entry("data that does not exist", fmt.Errorf("stream run-9: %w", profilestore.ErrProfileDataNotFound), http.StatusNotFound), + ginkgo.Entry("a request the hook refused", fmt.Errorf("stream run-9: %w", profilestore.ErrProfileRequestInvalid), http.StatusBadRequest), + ginkgo.Entry("a store that is down", errors.New("stream run-9: connection refused"), http.StatusInternalServerError), + ) + + ginkgo.It("prepares the data of a session started from the CLI", func() { + hookErr = errors.New("index unavailable") + var stdout, stderr bytes.Buffer + runner, err := NewRunner(RunnerOptions{ + Profiles: func() (profilestore.Store, error) { return store, nil }, + Context: func() dbcontext.Context { return dbcontext.New() }, + Stdout: &stdout, Stderr: &stderr, BeforeExecute: hook, + }) + Expect(err).ToNot(HaveOccurred()) + profile := execProfile("plain-top") + profile.Top = &query.TopSpec{} + + _, err = runner.startCLISession(store, profile, map[string]any{"region": "US"}) + Expect(errors.Is(err, query.ErrPrepareRead)).To(BeTrue(), "%v", err) + Expect(hooked).To(Receive(Equal("plain-top region=US"))) + Expect(hooked).To(Receive(Equal("release"))) + }) +}) diff --git a/pkg/sessions/profile_tables_test.go b/pkg/sessions/profile_tables_test.go new file mode 100644 index 0000000..12725b9 --- /dev/null +++ b/pkg/sessions/profile_tables_test.go @@ -0,0 +1,121 @@ +package sessions + +import ( + "bytes" + "context" + "net/http" + "strings" + "sync" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + profilepkg "github.com/flanksource/commons-db/cmd/query/profiles" + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/query/profilestore" +) + +// sessionRefProvider answers `ref:` by reading profiles. the way +// metric SQL does (query.ProfileTables), and any other query with one row +// naming it. +type sessionRefProvider struct{} + +func (sessionRefProvider) Type() string { return "session-profile-ref-mock" } + +func (sessionRefProvider) Execute(ctx dbcontext.Context, req query.ProviderRequest) ([]query.Row, error) { + name, ok := strings.CutPrefix(req.Query, "ref:") + if !ok { + return []query.Row{{"from": req.Query, "namespace": req.Params["namespace"]}}, nil + } + table, err := query.ProfileTables(ctx, req.Params)(name) + return table.Rows, err +} + +var _ = ginkgo.Describe("profiles. in a top session", func() { + const namespace = "acme" + var ( + store profilestore.Store + mu sync.Mutex + reads []string + ) + refProfile := func(name, q string) query.Profile { + return query.Profile{ + Name: name, Query: q, Provider: query.ProviderConfig{Type: "session-profile-ref-mock"}, + Params: []query.ParamDef{{Name: "namespace"}}, + } + } + hook := func(_ context.Context, batch []profilestore.ReadRequest) (func(), error) { + mu.Lock() + defer mu.Unlock() + for _, read := range batch { + reads = append(reads, read.Profile.Name) + } + return func() {}, nil + } + prepared := func() []string { + mu.Lock() + defer mu.Unlock() + return append([]string(nil), reads...) + } + wantRows := []query.Row{{"from": "leaf", "namespace": namespace}} + + ginkgo.BeforeEach(func() { + reads = nil + query.RegisterProvider(sessionRefProvider{}) + fileStore, err := profilepkg.NewFileStore(ginkgo.GinkgoT().TempDir()) + Expect(err).ToNot(HaveOccurred()) + for _, p := range []query.Profile{refProfile("parent", "ref:child"), refProfile("child", "leaf")} { + Expect(fileStore.Save(context.Background(), p)).To(Succeed()) + } + store = fileStore + }) + + latestRows := func(session *query.Session) func() []query.Row { + return func() []query.Row { + if latest := session.Latest(); latest != nil { + return latest.Rows + } + return nil + } + } + + ginkgo.It("reads the child of a session started over HTTP, preparing it with the registry's BeforeRead", func() { + registry := query.NewSessionRegistry(query.RegistryOptions{BeforeRead: func( + ctx context.Context, p query.Profile, params map[string]any, + ) (func(), error) { + return profilestore.PrepareReads(ctx, hook, []profilestore.ReadRequest{{Profile: p, Params: params}}) + }}) + ginkgo.DeferCleanup(registry.StopAll) + handler := newSessionHandler(sessionHandlerOptions{ + Prefix: "/api/v1", Ctx: dbcontext.New(), Store: store, Registry: registry, Next: &nextMarker{}, + }) + + response := doReq(handler, http.MethodPost, "/api/v1/profile/parent/sessions?interval=1s&namespace="+namespace) + Expect(response.Code).To(Equal(http.StatusCreated), response.Body.String()) + session, ok := registry.Get(decodeInfo(response).ID) + Expect(ok).To(BeTrue()) + + Eventually(latestRows(session)).Should(Equal(wantRows)) + Expect(prepared()[:2]).To(Equal([]string{"parent", "child"}), "a later sample may prepare again") + }) + + ginkgo.It("reads the child of a session started from the CLI, preparing it with BeforeExecute", func() { + var stdout, stderr bytes.Buffer + runner, err := NewRunner(RunnerOptions{ + Profiles: func() (profilestore.Store, error) { return store, nil }, + Context: func() dbcontext.Context { return dbcontext.New() }, + Stdout: &stdout, Stderr: &stderr, BeforeExecute: hook, + }) + Expect(err).ToNot(HaveOccurred()) + parent := refProfile("parent", "ref:child") + parent.Top = &query.TopSpec{} + + session, err := runner.startCLISession(store, parent, map[string]any{"namespace": namespace}) + Expect(err).ToNot(HaveOccurred()) + ginkgo.DeferCleanup(func() { session.Stop("spec done") }) + + Eventually(latestRows(session)).Should(Equal(wantRows)) + Expect(prepared()[:2]).To(Equal([]string{"parent", "child"}), "a later sample may prepare again") + }) +}) diff --git a/pkg/sessions/service.go b/pkg/sessions/service.go new file mode 100644 index 0000000..d424da8 --- /dev/null +++ b/pkg/sessions/service.go @@ -0,0 +1,232 @@ +package sessions + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/flanksource/commons-db/cmd/query/profiles" + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/query/profilestore" + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/telltale/pkg/tracing" +) + +type ProfileStoreProvider func() (profilestore.Store, error) +type ContextProvider func() dbcontext.Context + +// SessionEventLog holds the events a capture stream session emitted through +// the registry's EventSink, for replay after the session left the registry. +type SessionEventLog interface { + Events(ctx context.Context, id string) ([]query.Event, error) + + // HasEvents reports whether the log holds any event of session id, which + // is what a session's eventsAvailable says without reading them all. + HasEvents(ctx context.Context, id string) (bool, error) +} + +// EventRecords reads the record streams a session's status.events names: +// recordstore.Notifier is one. +type EventRecords interface { + Meta(ctx context.Context, stream string) (recordstore.Meta, error) + Scan(ctx context.Context, stream string, afterSeq int64, fn func(seq int64, row recordstore.Row) error) error + Tail(ctx context.Context, stream string, afterSeq int64, fn func(seq int64, row recordstore.Row) error) error +} + +type Options struct { + Profiles ProfileStoreProvider + Context ContextProvider + + // Registry holds the live sessions. When it has a Scope, a request lists, + // reads and controls only the live sessions of the scope its context names + // — another scope's is answered 404, as a missing one is — and Store must + // partition its records by the same scope. + Registry *query.SessionRegistry + + // Store answers for sessions that are not live in the registry: the list, + // info, lineage, and the records restart and replay start from. Nil lists + // only the registry's sessions. + Store query.SessionStore + + // EventLog replays the events of a stream session that recorded no + // status.events, after it left the registry. + EventLog SessionEventLog + + // Records replays a session's status.events after its owner process is + // gone, and follows it while this process still writes it. + Records EventRecords + + // Authorize decides whether r may read or control a session of the named + // profile. It is asked before a session starts, stops, extends or restarts + // (ActionControl) and before a session's info, events or result is served + // (ActionRead): routes that name no profile, so a gate on profile paths + // alone cannot cover them. A session the caller may not read is answered + // 404, as a missing one is, so its id never confirms it exists; any other + // refusal is answered 403 with its error. The list, its totals, its + // lookups and restartedAs leave out refused profiles. Nil allows + // every caller everything, which is right only when every profile served + // has the same permission. A connection trace session is named by its + // virtual profile, connection--sql-xevent. + Authorize AuthorizeFunc + + // Principal names the caller of r, for a stop reason and the principal of + // a restarted session. Nil stops "via API" and restarts as the previous + // principal. + Principal func(r *http.Request) string + + // Traces starts trace plugin captures, each authorized as its profile + // traces/. Nil serves no trace routes. + Traces *tracing.Runtime + + // TraceStartError answers a trace start the runtime refused, when the host + // recognises the refusal and can describe it better than its error text + // (a missing database grant, say), and reports whether it answered. Nil, + // or false, answers the error as text. + TraceStartError func(w http.ResponseWriter, r *http.Request, err error) bool +} + +type Service struct { + options Options +} + +func New(options Options) (*Service, error) { + if options.Profiles == nil { + return nil, fmt.Errorf("profile store provider is required") + } + if options.Context == nil { + return nil, fmt.Errorf("context provider is required") + } + if options.Registry == nil { + return nil, fmt.Errorf("session registry is required") + } + return &Service{options: options}, nil +} + +func (s *Service) Handler(prefix string, next http.Handler) (http.Handler, error) { + store, err := s.options.Profiles() + if err != nil { + return nil, err + } + o := s.options + return newSessionHandler(sessionHandlerOptions{ + Prefix: prefix, Ctx: o.Context(), Store: store, Registry: o.Registry, Sessions: o.Store, + EventLog: o.EventLog, Records: o.Records, Authorize: o.Authorize, Principal: o.Principal, Traces: o.Traces, + TraceStartError: o.TraceStartError, Next: next, + }), nil +} + +// sessionHandler serves the session API: +// +// POST {prefix}/profile/{name}/sessions start (?interval samples; ?follow is refused: GET {prefix}/profile/{name}?follow=true tails) +// POST {prefix}/connection/{id}/trace/sessions start a connection trace +// GET {prefix}/traces/kinds the trace plugin kinds served +// POST {prefix}/traces/{kind}/sessions start a trace plugin capture +// GET {prefix}/traces/{kind}/export download a window (?stream, ?afterSeq, ?toSeq, ?format) +// GET {prefix}/traces/{kind}/seq the seq a stream committed through (?stream) +// GET {prefix}/sessions list (filters, window, sort, paging, ?__lookup=filters) +// GET {prefix}/sessions/{id} info +// POST {prefix}/sessions/{id}/stop stop a live session +// POST {prefix}/sessions/{id}/extend move a live session's deadline (?duration) +// POST {prefix}/sessions/{id}/restart start a new session from an ended one (?duration) +// POST {prefix}/sessions/{id}/checkpoint have a running trace capture take a reading and commit (JSON body: the kind's checkpoint params) +// GET {prefix}/sessions/{id}/events SSE, resumable via Last-Event-ID (?format=ndjson exports) +// GET {prefix}/sessions/{id}/result materialized rows, or the recorded result +type sessionHandler struct { + prefix string + ctx dbcontext.Context + store profilestore.Store + registry *query.SessionRegistry + sessions query.SessionStore + eventLog SessionEventLog + records EventRecords + authorize AuthorizeFunc + principal func(r *http.Request) string + traces *tracing.Runtime + // traceStartError answers a refused trace start the host describes. + traceStartError func(w http.ResponseWriter, r *http.Request, err error) bool + next http.Handler +} + +type sessionHandlerOptions struct { + Prefix string + Ctx dbcontext.Context + Store profilestore.Store + Registry *query.SessionRegistry + Sessions query.SessionStore + EventLog SessionEventLog + Records EventRecords + Authorize AuthorizeFunc + Principal func(r *http.Request) string + Traces *tracing.Runtime + // TraceStartError answers a refused trace start the host describes. + TraceStartError func(w http.ResponseWriter, r *http.Request, err error) bool + Next http.Handler +} + +func newSessionHandler(opts sessionHandlerOptions) *sessionHandler { + return &sessionHandler{ + prefix: strings.TrimRight(opts.Prefix, "/"), ctx: opts.Ctx, store: opts.Store, registry: opts.Registry, + sessions: opts.Sessions, eventLog: opts.EventLog, records: opts.Records, authorize: opts.Authorize, + principal: opts.Principal, traces: opts.Traces, traceStartError: opts.TraceStartError, next: opts.Next, + } +} + +func (h *sessionHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + rel := strings.Trim(strings.TrimPrefix(strings.TrimSuffix(r.URL.Path, "/"), h.prefix), "/") + parts := strings.Split(rel, "/") + get, post := r.Method == http.MethodGet, r.Method == http.MethodPost + switch { + case post && len(parts) == 4 && parts[0] == "connection" && parts[2] == "trace" && parts[3] == "sessions": + h.startConnectionTrace(w, r, parts[1]) + case post && len(parts) == 3 && parts[0] == "profile" && parts[2] == "sessions": + h.start(w, r, parts[1]) + case h.traces != nil && get && len(parts) == 2 && parts[0] == "traces" && parts[1] == "kinds": + h.traceKinds(w, r) + case h.traces != nil && post && len(parts) == 3 && parts[0] == "traces" && parts[2] == "sessions": + h.startTrace(w, r, parts[1]) + case h.traces != nil && get && len(parts) == 3 && parts[0] == "traces" && parts[2] == "export": + h.exportTrace(w, r, parts[1]) + case h.traces != nil && get && len(parts) == 3 && parts[0] == "traces" && parts[2] == "seq": + h.traceSeq(w, r, parts[1]) + case parts[0] != "sessions": + h.next.ServeHTTP(w, r) + case get && len(parts) == 1: + h.list(w, r) + case get && len(parts) == 2: + h.info(w, r, parts[1]) + case post && len(parts) == 3 && parts[2] == "stop": + h.stop(w, r, parts[1]) + case post && len(parts) == 3 && parts[2] == "extend": + h.extend(w, r, parts[1]) + case post && len(parts) == 3 && parts[2] == "restart": + h.restart(w, r, parts[1]) + case h.traces != nil && post && len(parts) == 3 && parts[2] == "checkpoint": + h.checkpoint(w, r, parts[1]) + case get && len(parts) == 3 && parts[2] == "events": + h.events(w, r, parts[1]) + case get && len(parts) == 3 && parts[2] == "result": + h.result(w, r, parts[1]) + default: + h.next.ServeHTTP(w, r) + } +} + +// sessionReservedParam extends reservedParam with the session transport keys. +func sessionReservedParam(key string) bool { + return profiles.IsReservedParam(key) || key == "interval" || key == "duration" +} + +func writeSessionJSON(w http.ResponseWriter, status int, v any) { + data, err := json.Marshal(v) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Access-Control-Allow-Origin", "*") + w.WriteHeader(status) + _, _ = w.Write(append(data, '\n')) +} diff --git a/pkg/sessions/service_actions.go b/pkg/sessions/service_actions.go new file mode 100644 index 0000000..338cc5d --- /dev/null +++ b/pkg/sessions/service_actions.go @@ -0,0 +1,159 @@ +package sessions + +import ( + "errors" + "fmt" + "io" + "net/http" + "time" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/telltale/pkg/tracing" +) + +func (h *sessionHandler) info(w http.ResponseWriter, r *http.Request, id string) { + if resolved, ok := h.authorizedSession(w, r, id, ActionRead); ok { + writeSessionJSON(w, http.StatusOK, resolved.Info) + } +} + +// liveFor resolves a session an action applies to: authorized +// for it, live in this registry and not ended. It writes the refusal +// itself. +func (h *sessionHandler) liveFor(w http.ResponseWriter, r *http.Request, id, verb string, action Action) (*query.Session, bool) { + resolved, ok := h.authorizedSession(w, r, id, action) + switch { + case !ok: + return nil, false + case resolved.Live == nil: + http.Error(w, fmt.Sprintf("%s session %s: %v (owner %s, pid %d)", verb, id, query.ErrSessionNotLive, + resolved.Info.Owner.Host, resolved.Info.Owner.PID), http.StatusConflict) + return nil, false + case resolved.Info.State.Terminal(): + http.Error(w, fmt.Sprintf("%s session %s: it is %s", verb, id, resolved.Info.State), http.StatusConflict) + return nil, false + } + return resolved.Live, true +} + +func (h *sessionHandler) stop(w http.ResponseWriter, r *http.Request, id string) { + session, ok := h.liveFor(w, r, id, "stop", ActionControl) + if !ok { + return + } + reason := "stopped via API" + if h.principal != nil { + reason = "stopped by " + h.principal(r) + } + session.Stop(reason) + h.writeLive(w, r, http.StatusOK, session) +} + +// maxCheckpointBody bounds a checkpoint's params, which can carry a stack +// dumped elsewhere. +const maxCheckpointBody = 8 << 20 + +// checkpoint has the running trace capture of session id take a reading, +// with the params its JSON body gives the kind, and commit what it emitted, +// then answers the session. A session that is no running capture of this +// server's trace runtime is a conflict; params its kind refuses or does not +// read are a bad request. +func (h *sessionHandler) checkpoint(w http.ResponseWriter, r *http.Request, id string) { + session, ok := h.liveFor(w, r, id, "checkpoint", ActionCheckpoint) + if !ok { + return + } + params, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxCheckpointBody)) + if err != nil { + http.Error(w, fmt.Sprintf("checkpoint session %s: read params: %v", id, err), http.StatusBadRequest) + return + } + if err := h.traces.CheckpointSession(tracing.WithCheckpointParams(r.Context(), params), id); err != nil { + status := http.StatusInternalServerError + switch { + case errors.Is(err, tracing.ErrNoCapture): + status = http.StatusConflict + case errors.Is(err, tracing.ErrCheckpointParams): + status = http.StatusBadRequest + } + http.Error(w, err.Error(), status) + return + } + h.writeLive(w, r, http.StatusOK, session) +} + +// extend adds ?duration to the session's deadline; the registry clamps it to +// its maximum duration from the session's start. +func (h *sessionHandler) extend(w http.ResponseWriter, r *http.Request, id string) { + duration, err := durationParam(r, true) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + session, ok := h.liveFor(w, r, id, "extend", ActionControl) + if !ok { + return + } + info := session.Snapshot() + stopAt := time.Now() + if info.StopAt != nil { + stopAt = *info.StopAt + } + if _, err := h.registry.Extend(id, stopAt.Add(duration)); err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + h.writeLive(w, r, http.StatusOK, session) +} + +// restart starts a new session from an ended, restartable one and answers 201 +// with the new session. +func (h *sessionHandler) restart(w http.ResponseWriter, r *http.Request, id string) { + duration, err := durationParam(r, false) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + resolved, ok := h.authorizedSession(w, r, id, ActionControl) + switch { + case !ok: + return + case !resolved.Info.State.Terminal(): + http.Error(w, fmt.Sprintf("restart session %s: it is %s; only an ended session restarts", id, resolved.Info.State), http.StatusConflict) + return + case !resolved.Info.Restartable: + http.Error(w, fmt.Sprintf("restart session %s (%s): %v", id, resolved.Info.Profile, query.ErrNotRestartable), http.StatusConflict) + return + } + overrides := query.RestartOverrides{Duration: duration} + if h.principal != nil { + overrides.Principal = h.principal(r) + } + session, err := h.registry.Restart(h.sessionContext(r), id, overrides) + switch { + case errors.Is(err, query.ErrMaxSessions), errors.Is(err, query.ErrNotRestartable): + http.Error(w, err.Error(), http.StatusConflict) + return + case err != nil: + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + h.writeLive(w, r, http.StatusCreated, session) +} + +// durationParam reads ?duration as a positive Go duration; zero when absent +// and not required. +func durationParam(r *http.Request, required bool) (time.Duration, error) { + raw := r.URL.Query().Get("duration") + if raw == "" { + if required { + return 0, errors.New("duration is required, e.g. ?duration=15m") + } + return 0, nil + } + duration, err := time.ParseDuration(raw) + if err != nil || duration <= 0 { + return 0, fmt.Errorf("duration %q must be a positive Go duration, e.g. 15m", raw) + } + return duration, nil +} diff --git a/pkg/sessions/service_actions_test.go b/pkg/sessions/service_actions_test.go new file mode 100644 index 0000000..5a9c612 --- /dev/null +++ b/pkg/sessions/service_actions_test.go @@ -0,0 +1,139 @@ +package sessions + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "time" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/commons-db/query" +) + +var _ = ginkgo.Describe("session actions", func() { + var h *apiHarness + + ginkgo.BeforeEach(func() { h = newAPIHarness() }) + + ginkgo.It("serves one session with its overlays and lineage, and answers a caller who may not read it 404", func() { + response := h.do(http.MethodGet, "/api/v1/sessions/c") + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + Expect(decodeInfo(response)).To(And(HaveField("State", query.SessionStopped), + HaveField("Restartable", true), HaveField("RestartedAs", []string{"f"}))) + + Expect(h.do(http.MethodGet, "/api/v1/sessions/nope").Code).To(Equal(http.StatusNotFound)) + h.unread[jvmProfile] = true + refused := h.do(http.MethodGet, "/api/v1/sessions/c") + Expect(refused.Code).To(Equal(http.StatusNotFound)) + Expect(refused.Body.String()).To(Equal("session \"c\" not found\n")) + Expect(h.do(http.MethodPost, "/api/v1/sessions/c/restart").Code).To(Equal(http.StatusNotFound), + "a control on a session the caller cannot read does not confirm it exists either") + }) + + ginkgo.It("names in restartedAs only the restarted sessions the caller may read", func() { + hidden := contractRecord(h.epoch, "g", sqlProfile, query.SessionRunning, 6, 0, nil) + hidden.RestartOf = "c" + Expect(h.store.Begin(context.Background(), hidden)).To(Succeed()) + Expect(decodeInfo(h.do(http.MethodGet, "/api/v1/sessions/c")).RestartedAs).To(Equal([]string{"f", "g"})) + + h.unread[sqlProfile] = true + Expect(decodeInfo(h.do(http.MethodGet, "/api/v1/sessions/c")).RestartedAs).To(Equal([]string{"f"})) + byID := map[string]query.SessionInfo{} + for _, info := range h.list("limit=500").Items { + byID[info.ID] = info + } + Expect(byID["c"].RestartedAs).To(Equal([]string{"f"})) + }) + + ginkgo.It("answers a connection trace the caller may not start 404, authorizing before it looks the connection up", func() { + const connection = "5b0c4c1e-6b1f-4c1a-9f0e-2d3c4b5a6978" + profile := "connection-" + connection + "-sql-xevent" + h.uncontrol[profile] = true + request := httptest.NewRequest(http.MethodPost, "/api/v1/connection/"+connection+"/trace/sessions", strings.NewReader(`{}`)) + response := httptest.NewRecorder() + h.handler.ServeHTTP(response, request) + + Expect(response.Code).To(Equal(http.StatusNotFound)) + Expect(response.Body.String()).To(Equal(`connection "` + connection + `" not found` + "\n")) + Expect(h.callsFor(profile, ActionControl)).To(Equal(1)) + }) + + ginkgo.It("stops a live session under control authorization only", func() { + live := h.track(nil) + h.uncontrol[jvmProfile] = true + refused := h.do(http.MethodPost, "/api/v1/sessions/"+live.ID()+"/stop") + Expect(refused.Code).To(Equal(http.StatusForbidden), refused.Body.String()) + Expect(live.Snapshot().State).To(Equal(query.SessionRunning), "a refused stop leaves the session running") + + h.uncontrol[jvmProfile] = false + response := h.do(http.MethodPost, "/api/v1/sessions/"+live.ID()+"/stop") + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + Expect(decodeInfo(response).State).To(BeElementOf(query.SessionStopping, query.SessionStopped)) + Eventually(live.Done()).Should(BeClosed()) + Expect(live.Snapshot()).To(And(HaveField("State", query.SessionStopped), HaveField("StopReason", "stopped by tester"))) + + Expect(h.do(http.MethodPost, "/api/v1/sessions/"+live.ID()+"/stop").Code).To(Equal(http.StatusConflict)) + Expect(h.do(http.MethodPost, "/api/v1/sessions/a/stop").Code).To(Equal(http.StatusConflict), "a is not live here") + Expect(h.do(http.MethodPost, "/api/v1/sessions/nope/stop").Code).To(Equal(http.StatusNotFound)) + }) + + ginkgo.It("no longer stops a session by DELETE", func() { + live := h.track(nil) + h.do(http.MethodDelete, "/api/v1/sessions/"+live.ID()) + Expect(h.handler.next.(*nextMarker).hit).To(BeTrue()) + Expect(live.Snapshot().State).To(Equal(query.SessionRunning)) + }) + + ginkgo.It("extends a live session by adding the duration to its deadline", func() { + live := h.track(nil) + before := *live.Snapshot().StopAt + + Expect(h.do(http.MethodPost, "/api/v1/sessions/"+live.ID()+"/extend").Code).To(Equal(http.StatusBadRequest)) + Expect(h.do(http.MethodPost, "/api/v1/sessions/"+live.ID()+"/extend?duration=soon").Code).To(Equal(http.StatusBadRequest)) + response := h.do(http.MethodPost, "/api/v1/sessions/"+live.ID()+"/extend?duration=3m") + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + Expect(*decodeInfo(response).StopAt).To(BeTemporally("~", before.Add(3*time.Minute), time.Second)) + + Expect(h.do(http.MethodPost, "/api/v1/sessions/c/extend?duration=3m").Code).To(Equal(http.StatusConflict)) + }) + + ginkgo.It("restarts for the duration first requested when no duration is given", func() { + requested := contractFixture(h.epoch)[0] + requested.ID, requested.State = "requested", query.SessionStopped + requested.Params = map[string]any{"class": "com.example.Listener", "durationMs": 120000.0} + extended := requested.StartedAt.Add(14 * time.Minute) + requested.StopAt = &extended + Expect(h.store.Begin(context.Background(), requested)).To(Succeed()) + + response := h.do(http.MethodPost, "/api/v1/sessions/requested/restart") + Expect(response.Code).To(Equal(http.StatusCreated), response.Body.String()) + Expect(*decodeInfo(response).StopAt).To(BeTemporally("~", time.Now().Add(2*time.Minute), 5*time.Second)) + }) + + ginkgo.It("clamps an extension to the registry's maximum duration from the start", func() { + live := h.track(nil) + response := h.do(http.MethodPost, "/api/v1/sessions/"+live.ID()+"/extend?duration=10h") + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + info := decodeInfo(response) + Expect(*info.StopAt).To(BeTemporally("~", info.StartedAt.Add(query.DefaultMaxDuration), time.Second)) + }) + + ginkgo.It("restarts an ended, restartable session as a new one", func() { + response := h.do(http.MethodPost, "/api/v1/sessions/c/restart?duration=2m") + Expect(response.Code).To(Equal(http.StatusCreated), response.Body.String()) + next := decodeInfo(response) + Expect(next).To(And(HaveField("RestartOf", "c"), HaveField("Principal", "tester"), + HaveField("Controllable", true), HaveField("Labels", contractFixture(h.epoch)[0].Labels))) + Expect(next.ID).ToNot(Equal("c")) + Expect(*next.StopAt).To(BeTemporally("~", time.Now().Add(2*time.Minute), 5*time.Second)) + + Expect(h.do(http.MethodPost, "/api/v1/sessions/a/restart").Code).To(Equal(http.StatusConflict), "a is still running") + Expect(h.do(http.MethodPost, "/api/v1/sessions/b/restart").Code).To(Equal(http.StatusConflict), "no restarter for sql_xevent") + Expect(h.do(http.MethodPost, "/api/v1/sessions/c/restart?duration=-1m").Code).To(Equal(http.StatusBadRequest)) + h.uncontrol[jvmProfile] = true + Expect(h.do(http.MethodPost, "/api/v1/sessions/c/restart").Code).To(Equal(http.StatusForbidden)) + }) +}) diff --git a/pkg/sessions/service_events.go b/pkg/sessions/service_events.go new file mode 100644 index 0000000..412c86f --- /dev/null +++ b/pkg/sessions/service_events.go @@ -0,0 +1,316 @@ +package sessions + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "time" + + "github.com/flanksource/commons/logger" + "github.com/flanksource/telltale/pkg/sessions/internal/sse" + + "github.com/flanksource/commons-db/cmd/query/profiles" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore" +) + +// events serves a session's events: from the record stream its status names +// (followed while this process still writes it), from a live stream session's +// ring, or from the event log a finished stream session was persisted to. +func (h *sessionHandler) events(w http.ResponseWriter, r *http.Request, id string) { + after, err := sse.LastEventSequence(r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + resolved, ok := h.authorizedSession(w, r, id, ActionRead) + if !ok { + return + } + info, ndjson := resolved.Info, r.URL.Query().Get("format") == "ndjson" + switch { + case info.Events != nil: + h.replayRecords(w, r, resolved, after, ndjson) + case resolved.Live != nil && info.Kind != query.KindCapture && ndjson: + sse.WriteNDJSON(w, sessionEventsFilename(id), resolved.Live.Events()) + case resolved.Live != nil && info.Kind != query.KindCapture: + sseStream{session: resolved.Live, after: after, keepalive: sse.DefaultKeepalive}.run(w, r) + case h.eventLog != nil && info.Kind != query.KindCapture: + events, err := h.eventLog.Events(r.Context(), id) + switch { + case err != nil: + http.Error(w, err.Error(), http.StatusInternalServerError) + case ndjson: + sse.WriteNDJSON(w, sessionEventsFilename(id), events) + default: + sseHistory{events: events, info: info, after: after}.write(w) + } + default: + http.Error(w, fmt.Sprintf("session %s recorded no events", id), http.StatusNotFound) + } +} + +func sessionEventsFilename(id string) string { return "session-" + id + ".ndjson" } + +// goneResponse is the 410 body for a session whose record stream no longer +// exists: where it was, so an operator knows what was lost. +type goneResponse struct { + Error string `json:"error"` + Store query.EventsStoreLocation `json:"store"` +} + +// replayRecords serves the session's recorded window, following the stream +// past it while this process writes the still-active session. A stream that no +// longer exists, or exists as another generation than the ref's — its id was +// reused after the recorded one expired — is gone: 410 with where it was. +func (h *sessionHandler) replayRecords(w http.ResponseWriter, r *http.Request, resolved resolvedSession, after int64, ndjson bool) { + info := resolved.Info + ref := info.Events + if h.records == nil { + http.Error(w, fmt.Sprintf("session %s: its events are in record stream %q, but this server has no record store", info.ID, ref.Stream), http.StatusInternalServerError) + return + } + meta, err := h.records.Meta(r.Context(), ref.Stream) + switch { + case errors.Is(err, recordstore.ErrNotFound): + writeSessionJSON(w, http.StatusGone, goneResponse{ + Error: fmt.Sprintf("session %s: record stream %q is gone: %v", info.ID, ref.Stream, recordstore.ErrNotFound), + Store: ref.Store, + }) + return + case err != nil: + http.Error(w, fmt.Sprintf("session %s: read record stream %q: %v", info.ID, ref.Stream, err), http.StatusInternalServerError) + return + case meta.Generation != ref.Generation: + writeSessionJSON(w, http.StatusGone, goneResponse{ + Error: fmt.Sprintf("session %s: record stream %q generation %q is gone; the stream now holds generation %q", + info.ID, ref.Stream, ref.Generation, meta.Generation), + Store: ref.Store, + }) + return + } + window := recordWindow{ref: *ref, after: max(after, ref.From-1)} + if ndjson { + h.exportRecords(w, r, info.ID, window) + return + } + h.streamRecords(w, r, resolved, window) +} + +// streamRecords writes the window as SSE, then a done frame carrying the +// session as it stands, or an error frame when the stream failed mid-way. A +// session this process still writes is followed until its stream is sealed or +// the session ends, whichever comes first. +func (h *sessionHandler) streamRecords(w http.ResponseWriter, r *http.Request, resolved resolvedSession, window recordWindow) { + info := resolved.Info + flusher, err := sse.Begin(w) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + stream := recordStream{records: h.records, id: info.ID, window: window, w: w, flusher: flusher} + if info.LocalWriter && !info.State.Terminal() { + err = stream.follow(r.Context(), sse.DefaultKeepalive, sessionEnded(resolved.Live)) + } else { + err = stream.scan(r.Context()) + } + var final resolvedSession + if err == nil { + final, err = h.resolve(r.Context(), info.ID) + } + if err != nil { + _ = sse.WriteFrame(w, sse.Frame{Event: "error", Data: map[string]string{"error": err.Error()}}) + flusher.Flush() + return + } + info.SessionRecord = final.Info.SessionRecord + _ = sse.WriteFrame(w, sse.Frame{Event: "done", Data: info}) + flusher.Flush() +} + +// exportRecords streams the window as NDJSON, one line per row as the scan +// reads it. A scan failing before its first row is a 500; one failing after +// the file began aborts the response, so the client sees a broken transfer +// rather than a short file that looks complete. +func (h *sessionHandler) exportRecords(w http.ResponseWriter, r *http.Request, id string, window recordWindow) { + var encoder *json.Encoder + err := h.records.Scan(r.Context(), window.ref.Stream, window.after, func(seq int64, row recordstore.Row) error { + if window.past(seq) { + return errWindowEnd + } + if encoder == nil { + encoder = sse.BeginNDJSON(w, sessionEventsFilename(id)) + } + return encoder.Encode(query.Event{SessionID: id, Sequence: seq, Row: row}) + }) + switch { + case (err == nil || errors.Is(err, errWindowEnd)) && encoder == nil: + sse.BeginNDJSON(w, sessionEventsFilename(id)) + case err == nil || errors.Is(err, errWindowEnd): + case encoder == nil: + http.Error(w, fmt.Sprintf("session %s: read record stream %q: %v", id, window.ref.Stream, err), http.StatusInternalServerError) + default: + logger.Errorf("session %s: export of record stream %q failed mid-file: %v", id, window.ref.Stream, err) + panic(http.ErrAbortHandler) + } +} + +// sessionEnded is closed once live ends. A record this process writes through +// another registry has no session here to watch: its follow ends when the +// stream is sealed or the client leaves, and the nil channel never fires. +func sessionEnded(live *query.Session) <-chan struct{} { + if live == nil { + return nil + } + return live.Done() +} + +// errWindowEnd stops a scan at the end of the recorded window. +var errWindowEnd = errors.New("end of the recorded window") + +// recordWindow is the part of a record stream a session's ref names, after +// what the client already holds. +type recordWindow struct { + ref query.EventsRef + after int64 +} + +// past reports whether seq lies beyond the window's recorded end. +func (w recordWindow) past(seq int64) bool { return w.ref.To > 0 && seq > w.ref.To } + +// recordStream writes a record stream's rows as SSE event frames. +type recordStream struct { + records EventRecords + id string + window recordWindow + w http.ResponseWriter + flusher http.Flusher +} + +func (s recordStream) write(seq int64, row recordstore.Row) error { + if err := sse.WriteFrame(s.w, sse.Frame{Event: "event", ID: seq, Data: query.Event{SessionID: s.id, Sequence: seq, Row: row}}); err != nil { + return err + } + s.flusher.Flush() + return nil +} + +func (s recordStream) scan(ctx context.Context) error { + err := s.records.Scan(ctx, s.window.ref.Stream, s.window.after, func(seq int64, row recordstore.Row) error { + if s.window.past(seq) { + return errWindowEnd + } + return s.write(seq, row) + }) + if errors.Is(err, errWindowEnd) { + return nil + } + return err +} + +type tailedRow struct { + seq int64 + row recordstore.Row +} + +// follow tails the stream until it is sealed, the client leaves, or ended +// closes — a session that stopped, timed out or aborted without sealing its +// stream — after which it drains the rows up to the stream's high seq. Every +// byte is written from this goroutine, so a keepalive never splits a frame. +func (s recordStream) follow(ctx context.Context, keepalive time.Duration, ended <-chan struct{}) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + rows, done := make(chan tailedRow), make(chan error, 1) + go func() { + done <- s.records.Tail(ctx, s.window.ref.Stream, s.window.after, func(seq int64, row recordstore.Row) error { + select { + case rows <- tailedRow{seq: seq, row: row}: + return nil + case <-ctx.Done(): + return ctx.Err() + } + }) + }() + ticker := time.NewTicker(keepalive) + defer ticker.Stop() + last, drainTo := s.window.after, int64(-1) + for drainTo < 0 || last < drainTo { + select { + case err := <-done: + return err + case <-ended: + meta, err := s.records.Meta(ctx, s.window.ref.Stream) + if err != nil { + return fmt.Errorf("session %s ended: read record stream %q: %w", s.id, s.window.ref.Stream, err) + } + drainTo, ended = meta.HighSeq, nil + case <-ticker.C: + if _, err := s.w.Write([]byte(sse.KeepaliveFrame)); err != nil { + return err + } + s.flusher.Flush() + case tailed := <-rows: + if err := s.write(tailed.seq, tailed.row); err != nil { + return err + } + last = tailed.seq + } + } + return nil +} + +// result serves the result a capture recorded in its status, or a stream +// session's materialized rows. +func (h *sessionHandler) result(w http.ResponseWriter, r *http.Request, id string) { + resolved, ok := h.authorizedSession(w, r, id, ActionRead) + if !ok { + return + } + info := resolved.Info + switch { + case len(info.Result) > 0: + writeSessionJSON(w, http.StatusOK, info.Result) + case info.Kind == query.KindCapture: + http.Error(w, fmt.Sprintf("session %s recorded no result", id), http.StatusNotFound) + case resolved.Live != nil: + result, err := resolved.Live.Result(h.ctx) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + writeRows(w, result) + case h.eventLog != nil: + h.materializeLogged(w, r, info) + default: + http.Error(w, fmt.Sprintf("session %s recorded no result", id), http.StatusNotFound) + } +} + +func (h *sessionHandler) materializeLogged(w http.ResponseWriter, r *http.Request, info query.SessionInfo) { + resolved, err := profiles.Resolve(r.Context(), h.store, info.Profile) + if err != nil { + http.Error(w, fmt.Sprintf("session %q: %v", info.ID, err), http.StatusNotFound) + return + } + events, err := h.eventLog.Events(r.Context(), info.ID) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + result, err := query.MaterializeEvents(h.ctx, resolved.Profile, events) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + writeRows(w, result) +} + +func writeRows(w http.ResponseWriter, result *query.Result) { + rows := result.Rows + if rows == nil { + rows = []query.Row{} + } + writeSessionJSON(w, http.StatusOK, rows) +} diff --git a/pkg/sessions/service_events_test.go b/pkg/sessions/service_events_test.go new file mode 100644 index 0000000..1d05802 --- /dev/null +++ b/pkg/sessions/service_events_test.go @@ -0,0 +1,186 @@ +package sessions + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "strings" + "time" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore/recordstoretest" +) + +var _ = ginkgo.Describe("events and results of a session no longer live", func() { + var h *apiHarness + ctx := context.Background() + + // recordEvents appends rows to stream and points session id's status at + // them, as a capture's final status would. + recordEvents := func(id, stream string, rows int, state query.SessionState) { + batch := make([]query.Row, rows) + for i := range batch { + batch[i] = query.Row{"name": "row", "count": float64(i + 1)} + } + appended, err := h.records.Append(ctx, stream, recordstoretest.Kind, batch) + Expect(err).ToNot(HaveOccurred()) + meta, err := h.records.Meta(ctx, stream) + Expect(err).ToNot(HaveOccurred()) + rec, _, err := h.store.Get(ctx, id) + Expect(err).ToNot(HaveOccurred()) + rec.State = state + rec.Events = &query.EventsRef{ + Stream: stream, Kind: recordstoretest.Kind, Generation: meta.Generation, From: appended.Window.From, To: appended.Window.To, + Store: query.EventsStoreLocation{Backend: "sqlite", Host: "pod-a", File: "/data/records.sqlite"}, + } + rec.Result = json.RawMessage(`{"calls":3}`) + Expect(h.store.Update(ctx, id, rec.SessionStatus)).To(Succeed()) + } + + ginkgo.BeforeEach(func() { h = newAPIHarness() }) + + ginkgo.It("replays a terminal session's recorded window as SSE, resuming after Last-Event-ID", func() { + recordEvents("c", "stream-c", 3, query.SessionStopped) + + body := h.do(http.MethodGet, "/api/v1/sessions/c/events").Body.String() + Expect(sseFrames(body, "event")).To(Equal(3)) + Expect(sseFrames(body, "done")).To(Equal(1)) + Expect(body).To(ContainSubstring(`"row":{"count":2,"name":"row"}`)) + + resumed := h.do(http.MethodGet, "/api/v1/sessions/c/events", "Last-Event-ID", "2").Body.String() + Expect(sseFrames(resumed, "event")).To(Equal(1)) + Expect(resumed).To(ContainSubstring("id: 3\n")) + }) + + ginkgo.It("exports the recorded window as NDJSON", func() { + recordEvents("c", "stream-c", 2, query.SessionStopped) + response := h.do(http.MethodGet, "/api/v1/sessions/c/events?format=ndjson") + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + lines := strings.Split(strings.TrimSpace(response.Body.String()), "\n") + Expect(lines).To(HaveLen(2)) + var event query.Event + Expect(json.Unmarshal([]byte(lines[1]), &event)).To(Succeed()) + Expect(event).To(And(HaveField("SessionID", "c"), HaveField("Sequence", int64(2)))) + }) + + ginkgo.It("tails a record this process writes until its stream is sealed", func() { + rec := contractFixture(h.epoch)[4] + rec.ID, rec.Owner = "local", h.registry.Owner() + Expect(h.store.Begin(ctx, rec)).To(Succeed()) + recordEvents("local", "stream-local", 1, query.SessionRunning) + + done := make(chan string) + go func() { + defer ginkgo.GinkgoRecover() + done <- h.do(http.MethodGet, "/api/v1/sessions/local/events").Body.String() + }() + Consistently(done, 100*time.Millisecond).ShouldNot(Receive()) + _, err := h.records.Append(ctx, "stream-local", recordstoretest.Kind, []query.Row{{"name": "late"}}) + Expect(err).ToNot(HaveOccurred()) + Expect(h.records.Seal(ctx, "stream-local")).To(Succeed()) + + var body string + Eventually(done, 5*time.Second).Should(Receive(&body)) + Expect(sseFrames(body, "event")).To(Equal(2)) + Expect(body).To(ContainSubstring(`"name":"late"`)) + }) + + ginkgo.It("answers 410 with the store location when the recorded stream is gone", func() { + recordEvents("c", "stream-c", 1, query.SessionStopped) + rec, _, err := h.store.Get(ctx, "c") + Expect(err).ToNot(HaveOccurred()) + rec.Events.Stream = "stream-expired" + Expect(h.store.Update(ctx, "c", rec.SessionStatus)).To(Succeed()) + + response := h.do(http.MethodGet, "/api/v1/sessions/c/events") + Expect(response.Code).To(Equal(http.StatusGone)) + Expect(response.Body.String()).To(MatchJSON(`{ + "error": "session c: record stream \"stream-expired\" is gone: record stream not found", + "store": {"backend": "sqlite", "host": "pod-a", "file": "/data/records.sqlite"}}`)) + }) + + ginkgo.It("answers 410, and reports no events, when the stream id now holds another generation", func() { + recordEvents("c", "stream-c", 2, query.SessionStopped) + rec, _, err := h.store.Get(ctx, "c") + Expect(err).ToNot(HaveOccurred()) + recorded := rec.Events.Generation + rec.Events.Generation = "generation-that-expired" + Expect(h.store.Update(ctx, "c", rec.SessionStatus)).To(Succeed()) + + for _, path := range []string{"/api/v1/sessions/c/events", "/api/v1/sessions/c/events?format=ndjson"} { + response := h.do(http.MethodGet, path) + Expect(response.Code).To(Equal(http.StatusGone), path) + Expect(response.Body.String()).To(MatchJSON(`{ + "error": "session c: record stream \"stream-c\" generation \"generation-that-expired\" is gone; the stream now holds generation \"` + recorded + `\"", + "store": {"backend": "sqlite", "host": "pod-a", "file": "/data/records.sqlite"}}`)) + } + Expect(decodeInfo(h.do(http.MethodGet, "/api/v1/sessions/c")).EventsAvailable).To(BeFalse()) + }) + + ginkgo.It("streams a large NDJSON export row by row, never past the recorded window", func() { + const recorded = 250 + recordEvents("c", "stream-c", recorded, query.SessionStopped) + _, err := h.records.Append(ctx, "stream-c", recordstoretest.Kind, []query.Row{{"name": "after the window"}}) + Expect(err).ToNot(HaveOccurred()) + + response := h.do(http.MethodGet, "/api/v1/sessions/c/events?format=ndjson", "Last-Event-ID", "10") + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + Expect(response.Header().Get("Content-Type")).To(Equal("application/x-ndjson")) + lines := strings.Split(strings.TrimSpace(response.Body.String()), "\n") + Expect(lines).To(HaveLen(recorded - 10)) + Expect(response.Body.String()).ToNot(ContainSubstring("after the window")) + }) + + ginkgo.It("ends a follow when the live session ends without sealing its stream, after draining it", func() { + live := h.track(nil) + _, err := h.records.Append(ctx, "stream-live", recordstoretest.Kind, []query.Row{{"name": "first"}}) + Expect(err).ToNot(HaveOccurred()) + meta, err := h.records.Meta(ctx, "stream-live") + Expect(err).ToNot(HaveOccurred()) + ref := &query.EventsRef{Stream: "stream-live", Kind: recordstoretest.Kind, Generation: meta.Generation, From: 1} + Expect(live.Running(query.RunningUpdate{Events: ref})).To(Succeed()) + + done := make(chan string) + go func() { + defer ginkgo.GinkgoRecover() + done <- h.do(http.MethodGet, "/api/v1/sessions/"+live.ID()+"/events").Body.String() + }() + Consistently(done, 100*time.Millisecond).ShouldNot(Receive()) + _, err = h.records.Append(ctx, "stream-live", recordstoretest.Kind, []query.Row{{"name": "last"}}) + Expect(err).ToNot(HaveOccurred()) + live.Abort(errors.New("event log unwritable")) // no seal + + var body string + Eventually(done, 5*time.Second).Should(Receive(&body)) + Expect(sseFrames(body, "event")).To(Equal(2)) + Expect(body).To(ContainSubstring(`"name":"last"`)) + Expect(sseFrames(body, "done")).To(Equal(1)) + Expect(body).To(ContainSubstring(`"state":"failed"`)) + }) + + ginkgo.It("serves the recorded result, and nothing for a session that recorded none", func() { + recordEvents("c", "stream-c", 1, query.SessionStopped) + response := h.do(http.MethodGet, "/api/v1/sessions/c/result") + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + Expect(response.Body.String()).To(MatchJSON(`{"calls":3}`)) + + Expect(h.do(http.MethodGet, "/api/v1/sessions/d/result").Code).To(Equal(http.StatusNotFound)) + Expect(h.do(http.MethodGet, "/api/v1/sessions/d/events").Code).To(Equal(http.StatusNotFound)) + }) + + ginkgo.It("answers a caller who may not read the profile exactly as for a session that does not exist", func() { + recordEvents("c", "stream-c", 1, query.SessionStopped) + h.unread[jvmProfile] = true + for _, route := range []string{"", "/events", "/result"} { + refused := h.do(http.MethodGet, "/api/v1/sessions/c"+route) + missing := h.do(http.MethodGet, "/api/v1/sessions/nope"+route) + Expect(refused.Code).To(Equal(http.StatusNotFound), route) + Expect(missing.Code).To(Equal(http.StatusNotFound), route) + Expect(strings.ReplaceAll(refused.Body.String(), `"c"`, `"nope"`)).To(Equal(missing.Body.String()), route) + } + }) +}) diff --git a/pkg/sessions/service_harness_test.go b/pkg/sessions/service_harness_test.go new file mode 100644 index 0000000..1cd67bb --- /dev/null +++ b/pkg/sessions/service_harness_test.go @@ -0,0 +1,171 @@ +package sessions + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "time" + + "github.com/flanksource/clicky/cache" + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + profilepkg "github.com/flanksource/commons-db/cmd/query/profiles" + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/commons-db/recordstore/kv" + "github.com/flanksource/commons-db/recordstore/recordstoretest" +) + +const ( + jvmProfile = "trace-capture/jvm_trace" + sqlProfile = "trace-capture/sql_xevent" +) + +// apiHarness serves the sessions API over a memory kv session store seeded +// with the contract fixture, a memory record store, and a registry that can +// restart jvm_trace captures. +type apiHarness struct { + handler *sessionHandler + registry *query.SessionRegistry + store *KVStore + records *recordstore.Notifier + epoch time.Time + + mu sync.Mutex + unread map[string]bool // profiles Authorize refuses to read + uncontrol map[string]bool // profiles Authorize refuses to control + calls map[string]int // Authorize calls per profile/action +} + +func newAPIHarness() *apiHarness { + h := &apiHarness{epoch: contractEpoch(), unread: map[string]bool{}, uncontrol: map[string]bool{}, calls: map[string]int{}} + var err error + h.store, err = NewMemoryKVStore(KVStoreOptions{}) + Expect(err).ToNot(HaveOccurred()) + backend, err := kv.New(kv.Options{ + Store: cache.NewMemory(), Prefix: "records", Schema: recordstoretest.Schema, TTL: time.Hour, MaxChunkBytes: 1 << 20, + }) + Expect(err).ToNot(HaveOccurred()) + h.records, err = recordstore.NewNotifier(backend, recordstore.NotifierOptions{RecheckInterval: 20 * time.Millisecond}) + Expect(err).ToNot(HaveOccurred()) + + h.registry = query.NewSessionRegistry(query.RegistryOptions{ + Store: h.store, + Restarters: map[string]query.RestartFunc{ + "trace-capture/jvm": func(ctx context.Context, _ query.SessionRecord, opts query.TrackOptions) (*query.Session, error) { + return h.registry.Track(ctx, opts) + }, + }, + }) + ginkgo.DeferCleanup(func() { Expect(h.registry.StopAll(context.Background())).To(Succeed()) }) + profiles, err := profilepkg.NewFileStore(ginkgo.GinkgoT().TempDir()) + Expect(err).ToNot(HaveOccurred()) + + h.handler = newSessionHandler(sessionHandlerOptions{ + Prefix: "/api/v1", Ctx: dbcontext.New(), Store: profiles, Registry: h.registry, Sessions: h.store, + Records: h.records, Authorize: h.authorize, Next: &nextMarker{}, + Principal: func(*http.Request) string { return "tester" }, + }) + for _, rec := range contractFixture(h.epoch) { + Expect(h.store.Begin(context.Background(), rec)).To(Succeed()) + } + return h +} + +func (h *apiHarness) authorize(_ *http.Request, profile string, action Action) error { + h.mu.Lock() + defer h.mu.Unlock() + h.calls[profile+"/"+string(action)]++ + if h.unread[profile] || (action == ActionControl && h.uncontrol[profile]) { + return fmt.Errorf("profile %q is not %s-able by this caller", profile, action) + } + return nil +} + +func (h *apiHarness) callsFor(profile string, action Action) int { + h.mu.Lock() + defer h.mu.Unlock() + return h.calls[profile+"/"+string(action)] +} + +func (h *apiHarness) do(method, path string, header ...string) *httptest.ResponseRecorder { + request := httptest.NewRequest(method, path, nil) + for i := 0; i+1 < len(header); i += 2 { + request.Header.Set(header[i], header[i+1]) + } + response := httptest.NewRecorder() + h.handler.ServeHTTP(response, request) + return response +} + +// track starts a live jvm_trace capture in the harness registry. +func (h *apiHarness) track(labels map[string]string) *query.Session { + stopAt := time.Now().Add(10 * time.Minute) + session, err := h.registry.Track(context.Background(), query.TrackOptions{ + Profile: jvmProfile, Kind: query.KindCapture, Labels: labels, Principal: "admin", + Params: map[string]any{"class": "com.example.Listener"}, StopAt: &stopAt, + }) + Expect(err).ToNot(HaveOccurred()) + session.OnStop(func(string) { session.Finish(query.FinishUpdate{}) }) + Expect(session.Running(query.RunningUpdate{Handle: "probe-1"})).To(Succeed()) + return session +} + +// filterRecordingStore records the filter of every List it serves. +type filterRecordingStore struct { + query.SessionStore + mu sync.Mutex + filters []query.SessionFilter +} + +func (s *filterRecordingStore) List(ctx context.Context, filter query.SessionFilter) (query.SessionPage, error) { + s.mu.Lock() + s.filters = append(s.filters, filter) + s.mu.Unlock() + return s.SessionStore.List(ctx, filter) +} + +func (s *filterRecordingStore) last() query.SessionFilter { + s.mu.Lock() + defer s.mu.Unlock() + Expect(s.filters).ToNot(BeEmpty()) + return s.filters[len(s.filters)-1] +} + +type sessionListBody struct { + Items []query.SessionInfo `json:"items"` + Total int `json:"total"` + Shared bool `json:"shared"` +} + +func (h *apiHarness) list(query string) sessionListBody { + response := h.do(http.MethodGet, "/api/v1/sessions?"+query) + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + var body sessionListBody + Expect(json.Unmarshal(response.Body.Bytes(), &body)).To(Succeed()) + return body +} + +func infoIDs(infos []query.SessionInfo) []string { + ids := make([]string, len(infos)) + for i, info := range infos { + ids[i] = info.ID + } + return ids +} + +func decodeInfo(response *httptest.ResponseRecorder) query.SessionInfo { + var info query.SessionInfo + Expect(json.Unmarshal(response.Body.Bytes(), &info)).To(Succeed(), response.Body.String()) + return info +} + +func sseFrames(body string, event string) int { + return strings.Count(body, "event: "+event+"\n") +} diff --git a/pkg/sessions/service_list.go b/pkg/sessions/service_list.go new file mode 100644 index 0000000..39e6676 --- /dev/null +++ b/pkg/sessions/service_list.go @@ -0,0 +1,258 @@ +package sessions + +import ( + "context" + "fmt" + "net/http" + "net/url" + "slices" + "strconv" + "strings" + "time" + + "github.com/flanksource/clicky/entity" + "github.com/flanksource/commons-db/cmd/query/profiles" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/query/datetime" +) + +// Paging bounds of the session list. +const ( + defaultSessionPageLimit = 50 + maxSessionPageLimit = 500 +) + +// sessionFilterParams are the list's filter parameters, besides label.. +var sessionFilterParams = []string{"profile", "kind", "role", "state", "principal", "restartOf"} + +// sessionListParams are every other parameter the list accepts. +var sessionListParams = []string{"from", "to", "sort", "order", "limit", "offset", "format", + "__lookup", "__lookup_filter", "__lookup_q"} + +func (h *sessionHandler) list(w http.ResponseWriter, r *http.Request) { + filter, err := parseSessionFilter(r.URL.Query(), time.Now()) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + format, err := sessionListFormat(r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + filter.Allow = h.readAllow(r) + if r.URL.Query().Get("__lookup") == "filters" { + h.lookup(w, r, filter) + return + } + ctx := withSessionReadMemo(r.Context()) + page, err := h.listRecords(ctx, filter) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + infos, err := h.overlay(ctx, page.Items, filter.Allow) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + header := w.Header() + header.Set("X-Total-Count", strconv.Itoa(page.Total)) + header.Set("X-Page-Limit", strconv.Itoa(filter.Limit)) + header.Set("X-Page-Offset", strconv.Itoa(filter.Offset)) + header.Set("X-Sessions-Shared", strconv.FormatBool(page.Shared)) + entity.ExposeHeaders(header, "X-Total-Count", "X-Page-Limit", "X-Page-Offset", "X-Sessions-Shared") + if format == "clicky-json" { + writeSessionTable(w, infos) + return + } + writeSessionJSON(w, http.StatusOK, sessionListResponse{Items: infos, Total: page.Total, Shared: page.Shared}) +} + +// sessionListFormat negotiates the list's representation as profile lists do +// (profiles.RequestedFormat): clicky-json is the table document, json the +// items. A ?format naming anything else is refused; an Accept ranking a format +// the list has no rendering for — a browser's text/html — is served json, as +// HTTP lets a server answer an Accept it cannot meet. +func sessionListFormat(r *http.Request) (string, error) { + format := profiles.RequestedFormat(r) + switch { + case format == "json" || format == "clicky-json": + return format, nil + case r.URL.Query().Get("format") != "": + return "", fmt.Errorf("session list format %q is not json or clicky-json", r.URL.Query().Get("format")) + } + return "json", nil +} + +type sessionListResponse struct { + Items []query.SessionInfo `json:"items"` + Total int `json:"total"` + Shared bool `json:"shared"` +} + +// listRecords is one page of the sessions the registry holds and the store +// persisted, a live session's own record replacing its stored copy. When the +// registry holds no session the filter could select, the store's own page is +// the answer, and the store pages it. +func (h *sessionHandler) listRecords(ctx context.Context, filter query.SessionFilter) (query.SessionPage, error) { + live, err := h.liveRecords(ctx) + if err != nil { + return query.SessionPage{}, err + } + if h.sessions != nil && !registryMaySelect(live, filter) { + page, err := h.sessions.List(ctx, filter) + if err != nil { + return query.SessionPage{}, fmt.Errorf("list sessions: %w", err) + } + return page, nil + } + candidates, shared, err := h.candidates(ctx, filter) + if err != nil { + return query.SessionPage{}, err + } + page, err := query.ApplySessionFilter(candidates, filter) + page.Shared = shared + return page, err +} + +// registryMaySelect reports whether a live session could be on filter's page. +// Only the start fields are matched: a live session's state can differ from its +// stored copy's, which it replaces. +func registryMaySelect(live []query.SessionRecord, filter query.SessionFilter) bool { + filter.State, filter.Limit, filter.Offset = nil, 0, 0 + page, err := query.ApplySessionFilter(live, filter) + if err != nil { + // parseSessionFilter validated the filter; ApplySessionFilter fails on + // nothing else. + panic(fmt.Sprintf("sessions: apply a validated filter: %v", err)) + } + return page.Total > 0 +} + +// candidates is every record filter selects, unpaged, the live ones of ctx's +// scope included. +func (h *sessionHandler) candidates(ctx context.Context, filter query.SessionFilter) ([]query.SessionRecord, bool, error) { + live, err := h.liveRecords(ctx) + if err != nil { + return nil, false, err + } + if h.sessions == nil { + return live, false, nil + } + filter.Limit, filter.Offset = 0, 0 + stored, err := h.sessions.List(ctx, filter) + if err != nil { + return nil, false, fmt.Errorf("list sessions: %w", err) + } + byID := make(map[string]int, len(stored.Items)) + records := stored.Items + for i, rec := range records { + byID[rec.ID] = i + } + for _, rec := range live { + if i, ok := byID[rec.ID]; ok { + records[i] = rec + } else { + records = append(records, rec) + } + } + return records, stored.Shared, nil +} + +// parseSessionFilter reads the list's query parameters. A filter value is a +// comma-separated list, repeated keys adding to it; role defaults to capture. +func parseSessionFilter(values url.Values, now time.Time) (query.SessionFilter, error) { + filter := query.SessionFilter{Sort: "startedAt", Desc: true, Limit: defaultSessionPageLimit, Role: []string{string(query.SessionRoleCapture)}} + targets := map[string]*[]string{ + "profile": &filter.Profile, "kind": &filter.Kind, "role": &filter.Role, "state": &filter.State, + "principal": &filter.Principal, "restartOf": &filter.RestartOf, + } + for key, raw := range values { + if patterns := splitFilterValues(raw); strings.HasPrefix(key, "label.") && key != "label." { + if len(patterns) > 0 { + if filter.Labels == nil { + filter.Labels = map[string][]string{} + } + filter.Labels[strings.TrimPrefix(key, "label.")] = patterns + } + } else if target, ok := targets[key]; ok { + if len(patterns) > 0 { + *target = patterns + } + } else if !slices.Contains(sessionListParams, key) { + return filter, fmt.Errorf("unknown session list parameter %q; filters are %s and label.", key, strings.Join(sessionFilterParams, ", ")) + } + } + if err := parseSessionPaging(values, &filter, now); err != nil { + return filter, err + } + return filter, filter.Validate() +} + +func splitFilterValues(raw []string) []string { + var patterns []string + for _, value := range raw { + for _, part := range strings.Split(value, ",") { + if part = strings.TrimSpace(part); part != "" { + patterns = append(patterns, part) + } + } + } + return patterns +} + +func parseSessionPaging(values url.Values, filter *query.SessionFilter, now time.Time) error { + if sort := values.Get("sort"); sort != "" { + filter.Sort = sort + } + switch order := values.Get("order"); order { + case "", "desc": + case "asc": + filter.Desc = false + default: + return fmt.Errorf("order %q must be asc or desc", order) + } + var err error + if filter.Limit, err = intParam(values, "limit", defaultSessionPageLimit, 1, maxSessionPageLimit); err != nil { + return err + } + if filter.Offset, err = intParam(values, "offset", 0, 0, -1); err != nil { + return err + } + if filter.From, err = timeParam(values, "from", now); err != nil { + return err + } + filter.To, err = timeParam(values, "to", now) + return err +} + +// intParam reads an integer param between minimum and maximum (a negative +// maximum is unbounded). +func intParam(values url.Values, key string, fallback, minimum, maximum int) (int, error) { + raw := values.Get(key) + if raw == "" { + return fallback, nil + } + value, err := strconv.Atoi(raw) + if err != nil || value < minimum || (maximum >= 0 && value > maximum) { + bound := fmt.Sprintf("at least %d", minimum) + if maximum >= 0 { + bound = fmt.Sprintf("between %d and %d", minimum, maximum) + } + return 0, fmt.Errorf("%s %s must be a whole number %s", key, raw, bound) + } + return value, nil +} + +func timeParam(values url.Values, key string, now time.Time) (time.Time, error) { + raw := values.Get(key) + if raw == "" { + return time.Time{}, nil + } + parsed, err := datetime.Parse(raw, now) + if err != nil { + return time.Time{}, fmt.Errorf("%s: %w", key, err) + } + return parsed.Time, nil +} diff --git a/pkg/sessions/service_list_test.go b/pkg/sessions/service_list_test.go new file mode 100644 index 0000000..fbd3621 --- /dev/null +++ b/pkg/sessions/service_list_test.go @@ -0,0 +1,214 @@ +package sessions + +import ( + "context" + "encoding/json" + "net/http" + "net/url" + "time" + + "github.com/flanksource/clicky/cache" + "github.com/flanksource/clicky/formatters" + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore/recordstoretest" +) + +var _ = ginkgo.Describe("GET /sessions", func() { + var h *apiHarness + + ginkgo.BeforeEach(func() { h = newAPIHarness() }) + + ginkgo.DescribeTable("filters from comma-separated values and repeated keys alike", + func(rawQuery string, wantIDs []string) { + Expect(infoIDs(h.list(rawQuery).Items)).To(Equal(wantIDs)) + }, + ginkgo.Entry("comma-separated states", "state=running,stopped", []string{"f", "c", "a"}), + ginkgo.Entry("repeated state keys", "state=running&state=stopped", []string{"f", "c", "a"}), + ginkgo.Entry("a glob and an exclusion", "profile=*jvm_trace&state=!failed", []string{"f", "c", "a"}), + ginkgo.Entry("a label, ascending", "label.target=cycle&order=asc", []string{"b", "c", "f"}), + ginkgo.Entry("restartOf", "restartOf=c", []string{"f"}), + ginkgo.Entry("a date-math window", "from=now-59m&to=now-56m", []string{"d", "c", "b"}), + ginkgo.Entry("view sessions only when asked for by role", "role=view", []string{}), + ) + + ginkgo.It("authorizes before totals and paging, asking once per profile", func() { + h.unread[sqlProfile] = true + body := h.list("sort=eventCount&limit=2&offset=1") + Expect(infoIDs(body.Items)).To(Equal([]string{"a", "d"})) + Expect(body.Total).To(Equal(4), "the refused sql_xevent session is not counted") + Expect(h.callsFor(jvmProfile, ActionRead)).To(Equal(1)) + Expect(h.callsFor(sqlProfile, ActionRead)).To(Equal(1)) + }) + + ginkgo.It("pages with headers and reports whether the store is shared", func() { + response := h.do(http.MethodGet, "/api/v1/sessions?sort=eventCount&order=desc&limit=2&offset=1") + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + Expect(response.Header().Get("Content-Type")).To(Equal("application/json")) + Expect(response.Header().Get("X-Total-Count")).To(Equal("5")) + Expect(response.Header().Get("X-Page-Limit")).To(Equal("2")) + Expect(response.Header().Get("X-Page-Offset")).To(Equal("1")) + Expect(response.Header().Get("X-Sessions-Shared")).To(Equal("false")) + var body map[string]json.RawMessage + Expect(json.Unmarshal(response.Body.Bytes(), &body)).To(Succeed()) + Expect(body).To(HaveLen(3)) + Expect(body["total"]).To(MatchJSON("5")) + Expect(body["shared"]).To(MatchJSON("false")) + }) + + ginkgo.It("derives the overlays each item carries", func() { + live := h.track(map[string]string{"target": "oipa"}) + ref, err := h.records.Append(context.Background(), "stream-c", recordstoretest.Kind, []query.Row{{"name": "x"}}) + Expect(err).ToNot(HaveOccurred()) + meta, err := h.records.Meta(context.Background(), "stream-c") + Expect(err).ToNot(HaveOccurred()) + stopped := contractFixture(h.epoch)[0] + stopped.Events = &query.EventsRef{Stream: "stream-c", Kind: recordstoretest.Kind, Generation: meta.Generation, From: ref.Window.From, To: ref.Window.To} + Expect(h.store.Update(context.Background(), "c", stopped.SessionStatus)).To(Succeed()) + + byID := map[string]query.SessionInfo{} + for _, info := range h.list("limit=500").Items { + byID[info.ID] = info + } + Expect(byID[live.ID()]).To(And( + HaveField("Controllable", true), HaveField("LocalWriter", true), HaveField("Unresponsive", false), + HaveField("EventsAvailable", false), HaveField("Restartable", false))) + Expect(byID["c"]).To(And( + HaveField("Controllable", false), HaveField("LocalWriter", false), HaveField("EventsAvailable", true), + HaveField("Restartable", true), HaveField("RestartedAs", []string{"f"}))) + Expect(byID["a"]).To(And(HaveField("Unresponsive", true), HaveField("Restartable", false))) + Expect(byID["b"]).To(HaveField("Restartable", false), "no restarter serves sql_xevent") + }) + + ginkgo.DescribeTable("refuses a malformed list request", + func(rawQuery, message string) { + response := h.do(http.MethodGet, "/api/v1/sessions?"+rawQuery) + Expect(response.Code).To(Equal(http.StatusBadRequest)) + Expect(response.Body.String()).To(ContainSubstring(message)) + }, + ginkgo.Entry("a sort outside the allowlist", "sort=error", `session sort "error" is not one of`), + ginkgo.Entry("an order that is neither asc nor desc", "order=up", `order "up"`), + ginkgo.Entry("a limit over 500", "limit=501", "limit 501"), + ginkgo.Entry("an unreadable from", "from=yesterday-ish", "yesterday-ish"), + ginkgo.Entry("an unknown parameter", "colour=red", `unknown session list parameter "colour"`), + ) + + ginkgo.It("serves a clicky table whose rows carry an id and whose columns bind the filters", func() { + response := h.do(http.MethodGet, "/api/v1/sessions?limit=2", "Accept", "application/json+clicky") + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + Expect(response.Header().Get("Content-Type")).To(Equal("application/json+clicky")) + Expect(response.Header().Get("X-Total-Count")).To(Equal("5")) + + var document formatters.ClickyDocument + Expect(json.Unmarshal(response.Body.Bytes(), &document)).To(Succeed()) + Expect(document.Version).To(Equal(1)) + Expect(document.Node.Kind).To(Equal("table")) + columnNames := make([]string, len(document.Node.Columns)) + filterKeys := map[string]string{} + for i, column := range document.Node.Columns { + columnNames[i] = column.Name + filterKeys[column.Name] = column.FilterKey + } + Expect(columnNames).To(Equal([]string{ + "startedAt", "session", "profile", "target", "state", "eventCount", "stoppedAt", "origin", + "principal", "lineage", "labels", "owner", + })) + Expect(filterKeys).To(Equal(map[string]string{ + "state": "state", "session": "", "profile": "profile", "target": "label.target", "origin": "label.origin", + "principal": "principal", "startedAt": "", "stoppedAt": "", "eventCount": "", "lineage": "", + "labels": "", "owner": "", + })) + Expect(document.Node.Rows).To(HaveLen(2)) + cells := document.Node.Rows[0].Cells + Expect(cells["_id"].Text).To(Equal("f")) + Expect(cells["lineage"].Text).To(Equal("restart of c")) + Expect(cells["state"]).To(And(HaveField("Kind", "badge"), HaveField("FilterValue", "running"), + HaveField("BadgeValue", "running · unresponsive"), HaveField("BadgeColor", "bg-sky-500/10"), + HaveField("BadgeText", "text-sky-700"), HaveField("BadgeShape", "pill"))) + Expect(cells["labels"].Kind).To(Equal("map")) + Expect(cells["labels"].Fields).To(HaveLen(2)) + Expect(cells["labels"].Fields[0]).To(And(HaveField("Name", "origin"), HaveField("Value.Text", "web"))) + }) + + ginkgo.It("counts lookup options within the window, after Authorize and the other filters", func() { + h.unread[sqlProfile] = true + response := h.do(http.MethodGet, "/api/v1/sessions?__lookup=filters&state=running,stopped&label.origin=web") + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + var body struct { + Filters map[string]struct { + Label string `json:"label"` + Type string `json:"type"` + Multi bool `json:"multi"` + Options map[string]formatters.ClickyNode `json:"options"` + Counts map[string]int `json:"counts"` + } `json:"filters"` + } + Expect(json.Unmarshal(response.Body.Bytes(), &body)).To(Succeed()) + Expect(body.Filters["state"].Counts).To(Equal(map[string]int{"running": 1, "stopped": 1}), + "state counts ignore the state filter but keep origin=web; b is refused") + Expect(body.Filters["label.origin"].Counts).To(Equal(map[string]int{"web": 2, "testplan": 1})) + Expect(body.Filters["profile"].Counts).To(Equal(map[string]int{jvmProfile: 2})) + Expect(body.Filters["state"]).To(And(HaveField("Type", "multi-filter"), HaveField("Multi", true))) + Expect(body.Filters["restartOf"]).To(And(HaveField("Type", "value"), HaveField("Multi", false), + HaveField("Counts", map[string]int{"c": 1}))) + Expect(body.Filters["state"].Options["running"]).To(Equal(formatters.ClickyNode{Kind: "text", Text: "running", Plain: "running"})) + + narrowed := h.do(http.MethodGet, "/api/v1/sessions?__lookup=filters&__lookup_filter=label.origin&__lookup_q="+url.QueryEscape("TEST")) + Expect(narrowed.Code).To(Equal(http.StatusOK), narrowed.Body.String()) + Expect(narrowed.Body.String()).To(MatchJSON(`{"filters":{"label.origin":{"label":"origin","type":"multi-filter","multi":true, + "options":{"testplan":{"kind":"text","text":"testplan","plain":"testplan"}},"counts":{"testplan":1}}}}`)) + }) + + ginkgo.It("negotiates the list's format from ?format as profile lists do", func() { + table := h.do(http.MethodGet, "/api/v1/sessions?limit=2&format=clicky-json") + Expect(table.Code).To(Equal(http.StatusOK), table.Body.String()) + Expect(table.Header().Get("Content-Type")).To(Equal("application/json+clicky")) + + plain := h.do(http.MethodGet, "/api/v1/sessions?limit=2&format=json", "Accept", "application/json+clicky") + Expect(plain.Header().Get("Content-Type")).To(Equal("application/json"), "?format outranks Accept") + + refused := h.do(http.MethodGet, "/api/v1/sessions?format=csv") + Expect(refused.Code).To(Equal(http.StatusBadRequest)) + Expect(refused.Body.String()).To(ContainSubstring(`session list format "csv"`)) + }) + + ginkgo.It("lets the store page when no live session could be on the page, and pages itself otherwise", func() { + recording := &filterRecordingStore{SessionStore: h.store} + h.handler.sessions = recording + + body := h.list("sort=eventCount&limit=2&offset=1") + Expect(infoIDs(body.Items)).To(Equal([]string{"b", "a"})) + Expect(body.Total).To(Equal(5)) + Expect(recording.last()).To(And(HaveField("Limit", 2), HaveField("Offset", 1))) + + h.track(map[string]string{"target": "oipa"}) + body = h.list("sort=eventCount&limit=2&offset=1") + Expect(body.Total).To(Equal(6)) + Expect(recording.last()).To(And(HaveField("Limit", 0), HaveField("Offset", 0)), + "a live session replaces its stored copy, so the merge is paged here") + }) + + ginkgo.It("reads each stored record at most once to list a page with its lineage", func() { + counting := &countingCache{Store: cache.NewMemory(), reads: map[string]int{}} + store, err := NewKVStore(func(context.Context) (cache.Store, string, error) { return counting, "", nil }, KVStoreOptions{}) + Expect(err).ToNot(HaveOccurred()) + for _, rec := range contractFixture(h.epoch) { + Expect(store.Begin(context.Background(), rec)).To(Succeed()) + } + h.handler.sessions = store + counting.reset() + + body := h.list("limit=500") + Expect(body.Items).To(HaveLen(5)) + for _, rec := range contractFixture(h.epoch) { + Expect(counting.readsOf(rec.ID)).To(Equal(1), "session %s", rec.ID) + } + }) + + ginkgo.It("keeps the window when a session started before it", func() { + body := h.list("from=" + url.QueryEscape(time.Now().Add(-57*time.Minute-30*time.Second).Format(time.RFC3339))) + Expect(infoIDs(body.Items)).To(Equal([]string{"f", "d"})) + }) +}) diff --git a/pkg/sessions/service_lookup.go b/pkg/sessions/service_lookup.go new file mode 100644 index 0000000..e5d9f2f --- /dev/null +++ b/pkg/sessions/service_lookup.go @@ -0,0 +1,127 @@ +package sessions + +import ( + "fmt" + "net/http" + "slices" + "strings" + + "github.com/flanksource/clicky/formatters" + + "github.com/flanksource/commons-db/query" +) + +// sessionLabelFilters are the label dimensions every lookup counts; a label a +// request filters on is counted too. +var sessionLabelFilters = []string{"target", "origin", "via", "runId", "plan", "step", "environment"} + +type sessionLookupResponse struct { + Filters map[string]sessionLookupFilter `json:"filters"` +} + +type sessionLookupFilter struct { + Label string `json:"label"` + Type string `json:"type"` + Multi bool `json:"multi"` + Options map[string]formatters.ClickyNode `json:"options"` + Counts map[string]int `json:"counts"` +} + +// sessionFacet is one filter a lookup counts values of. +type sessionFacet struct { + name, label string + single bool // one typed value (restartOf) rather than a multi-select + value func(query.SessionRecord) string + without func(query.SessionFilter) query.SessionFilter +} + +func sessionFacets(filter query.SessionFilter) []sessionFacet { + field := func(name, label string, value func(query.SessionRecord) string, clear func(*query.SessionFilter)) sessionFacet { + return sessionFacet{name: name, label: label, value: value, without: func(f query.SessionFilter) query.SessionFilter { + clear(&f) + return f + }} + } + facets := []sessionFacet{ + field("profile", "Profile", func(r query.SessionRecord) string { return r.Profile }, func(f *query.SessionFilter) { f.Profile = nil }), + field("kind", "Kind", func(r query.SessionRecord) string { return string(r.Kind) }, func(f *query.SessionFilter) { f.Kind = nil }), + field("state", "State", func(r query.SessionRecord) string { return string(r.State) }, func(f *query.SessionFilter) { f.State = nil }), + field("principal", "Principal", func(r query.SessionRecord) string { return r.Principal }, func(f *query.SessionFilter) { f.Principal = nil }), + field("restartOf", "Restart of", func(r query.SessionRecord) string { return r.RestartOf }, func(f *query.SessionFilter) { f.RestartOf = nil }), + } + facets[len(facets)-1].single = true + keys := slices.Clone(sessionLabelFilters) + for key := range filter.Labels { + if !slices.Contains(keys, key) { + keys = append(keys, key) + } + } + for _, key := range keys { + facets = append(facets, sessionFacet{ + name: "label." + key, label: key, + value: func(r query.SessionRecord) string { return r.Labels[key] }, + without: func(f query.SessionFilter) query.SessionFilter { + labels := map[string][]string{} + for k, v := range f.Labels { + if k != key { + labels[k] = v + } + } + f.Labels = labels + return f + }, + }) + } + return facets +} + +// lookup counts each facet's values over the authorized sessions in the window +// that match every other active filter, so a count is what selecting the +// value would list. +func (h *sessionHandler) lookup(w http.ResponseWriter, r *http.Request, filter query.SessionFilter) { + only, search := r.URL.Query().Get("__lookup_filter"), strings.ToLower(r.URL.Query().Get("__lookup_q")) + windowed := query.SessionFilter{Role: filter.Role, From: filter.From, To: filter.To, Allow: filter.Allow} + candidates, _, err := h.candidates(r.Context(), windowed) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + response := sessionLookupResponse{Filters: map[string]sessionLookupFilter{}} + for _, facet := range sessionFacets(filter) { + if only != "" && facet.name != only { + continue + } + counted := facet.without(filter) + counted.Limit, counted.Offset = 0, 0 + matches, err := query.ApplySessionFilter(candidates, counted) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + response.Filters[facet.name] = countFacet(facet, matches.Items, search) + } + if only != "" && len(response.Filters) == 0 { + http.Error(w, fmt.Sprintf("session list has no lookup filter %q", only), http.StatusBadRequest) + return + } + writeSessionJSON(w, http.StatusOK, response) +} + +func countFacet(facet sessionFacet, records []query.SessionRecord, search string) sessionLookupFilter { + filter := sessionLookupFilter{ + Label: facet.label, Type: "multi-filter", Multi: !facet.single, + Options: map[string]formatters.ClickyNode{}, Counts: map[string]int{}, + } + if facet.single { + filter.Type = "value" + } + for _, rec := range records { + value := facet.value(rec) + if value == "" || (search != "" && !strings.Contains(strings.ToLower(value), search)) { + continue + } + filter.Options[value] = formatters.ClickyNode{Kind: "text", Text: value, Plain: value} + filter.Counts[value]++ + } + return filter +} diff --git a/pkg/sessions/service_overlay.go b/pkg/sessions/service_overlay.go new file mode 100644 index 0000000..e8da20c --- /dev/null +++ b/pkg/sessions/service_overlay.go @@ -0,0 +1,165 @@ +package sessions + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore" +) + +// overlay derives what this process knows about each record: whether it is +// live here, whether this process writes it, whether its owner has gone quiet, +// whether its events can be read and whether it can be restarted, and which +// sessions were restarted from it — only those allow lets the caller read. +func (h *sessionHandler) overlay(ctx context.Context, records []query.SessionRecord, allow func(string) bool) ([]query.SessionInfo, error) { + ids := make([]string, len(records)) + for i, rec := range records { + ids[i] = rec.ID + } + lineage, err := h.lineage(ctx, ids) + if err != nil { + return nil, err + } + if lineage, err = h.readableLineage(ctx, lineage, records, allow); err != nil { + return nil, err + } + visible, err := h.registry.Visible(ctx) + if err != nil { + return nil, err + } + now, boot, staleAfter := time.Now(), h.registry.Owner().Boot, h.registry.StaleAfter() + infos := make([]query.SessionInfo, len(records)) + for i, rec := range records { + session, live := h.registry.Get(rec.ID) + live = live && visible(session.Snapshot().SessionRecord) + terminal := rec.State.Terminal() + info := query.SessionInfo{ + SessionRecord: rec, + Controllable: live && !terminal, + LocalWriter: rec.Owner.Boot == boot, + Unresponsive: !live && !terminal && now.Sub(rec.HeartbeatAt) > staleAfter, + Restartable: terminal && h.registry.Restartable(rec), + RestartedAs: lineage[rec.ID], + } + if info.EventsAvailable, err = h.eventsAvailable(ctx, rec, live); err != nil { + return nil, err + } + infos[i] = info + } + return infos, nil +} + +// eventsAvailable reports whether this process can serve rec's events: the +// generation of its record stream its ref names still exists in the record +// store, or — for a stream session that recorded none — its ring is live here +// or the event log holds events for it. +func (h *sessionHandler) eventsAvailable(ctx context.Context, rec query.SessionRecord, live bool) (bool, error) { + switch { + case rec.Events == nil: + return h.streamEventsAvailable(ctx, rec, live) + case h.records == nil: + return false, nil + } + meta, err := h.records.Meta(ctx, rec.Events.Stream) + switch { + case errors.Is(err, recordstore.ErrNotFound): + return false, nil + case err != nil: + return false, fmt.Errorf("session %s: read record stream %q: %w", rec.ID, rec.Events.Stream, err) + } + return meta.Generation == rec.Events.Generation, nil +} + +// streamEventsAvailable is eventsAvailable for a record that names no record +// stream: a capture's are nowhere, a live stream session's are in its ring, and +// an ended one's are wherever the event log says. +func (h *sessionHandler) streamEventsAvailable(ctx context.Context, rec query.SessionRecord, live bool) (bool, error) { + switch { + case rec.Kind == query.KindCapture: + return false, nil + case live: + return true, nil + case h.eventLog == nil: + return false, nil + } + held, err := h.eventLog.HasEvents(ctx, rec.ID) + if err != nil { + return false, fmt.Errorf("session %s: read event log: %w", rec.ID, err) + } + return held, nil +} + +// lineage maps ids to the sessions restarted from them: from the store when +// there is one, which holds every capture, otherwise from the registry. +func (h *sessionHandler) lineage(ctx context.Context, ids []string) (map[string][]string, error) { + if h.sessions != nil { + lineage, err := h.sessions.Lineage(ctx, ids) + if err != nil { + return nil, fmt.Errorf("session lineage: %w", err) + } + return lineage, nil + } + live, err := h.liveRecords(ctx) + if err != nil { + return nil, err + } + return lineageOf(live, ids), nil +} + +// readableLineage drops the restarted sessions allow refuses, so restartedAs +// never names a session the caller could not list. A nil allow allows all. +func (h *sessionHandler) readableLineage(ctx context.Context, lineage map[string][]string, known []query.SessionRecord, allow func(string) bool) (map[string][]string, error) { + if allow == nil { + return lineage, nil + } + live, err := h.liveRecords(ctx) + if err != nil { + return nil, err + } + profiles := map[string]string{} + for _, rec := range append(live, known...) { + profiles[rec.ID] = rec.Profile + } + readable := make(map[string][]string, len(lineage)) + for parent, children := range lineage { + for _, child := range children { + profile, ok := profiles[child] + if !ok && h.sessions == nil { + return nil, fmt.Errorf("session lineage: restarted session %s is not in the registry that named it", child) + } + if !ok { + rec, found, err := h.sessions.Get(ctx, child) + if err != nil { + return nil, fmt.Errorf("session lineage: read restarted session %s: %w", child, err) + } + if !found { + continue // gone since the lineage was read + } + profile = rec.Profile + } + if allow(profile) { + readable[parent] = append(readable[parent], child) + } + } + } + return readable, nil +} + +// liveRecords is every session the registry holds in the scope ctx names, +// oldest first. +func (h *sessionHandler) liveRecords(ctx context.Context) ([]query.SessionRecord, error) { + visible, err := h.registry.Visible(ctx) + if err != nil { + return nil, err + } + var records []query.SessionRecord + for _, info := range h.registry.List() { + if visible(info.SessionRecord) { + records = append(records, info.SessionRecord) + } + } + return records, nil +} diff --git a/pkg/sessions/service_scope_test.go b/pkg/sessions/service_scope_test.go new file mode 100644 index 0000000..b045f84 --- /dev/null +++ b/pkg/sessions/service_scope_test.go @@ -0,0 +1,202 @@ +package sessions + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "sync" + "time" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + profilepkg "github.com/flanksource/commons-db/cmd/query/profiles" + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" +) + +const scopeEnvironmentLabel = "environment" + +type scopeEnvironmentKey struct{} + +func withEnvironment(ctx context.Context, environment string) context.Context { + return context.WithValue(ctx, scopeEnvironmentKey{}, environment) +} + +func environmentOf(ctx context.Context) (string, error) { + environment, ok := ctx.Value(scopeEnvironmentKey{}).(string) + if !ok { + return "", errors.New("request names no environment") + } + return environment, nil +} + +// environmentStores routes every record to its environment's own store, as a +// host serving several environments does: the store is scoped, so only the +// registry's live sessions can cross environments. +type environmentStores struct { + mu sync.Mutex + stores map[string]*KVStore +} + +func (s *environmentStores) storeFor(ctx context.Context) (*KVStore, error) { + environment, err := environmentOf(ctx) + if err != nil { + return nil, err + } + s.mu.Lock() + defer s.mu.Unlock() + if store, ok := s.stores[environment]; ok { + return store, nil + } + store, err := NewMemoryKVStore(KVStoreOptions{}) + if err != nil { + return nil, err + } + s.stores[environment] = store + return store, nil +} + +func (s *environmentStores) Begin(ctx context.Context, rec query.SessionRecord) error { + store, err := s.storeFor(ctx) + if err != nil { + return err + } + return store.Begin(ctx, rec) +} + +func (s *environmentStores) Update(ctx context.Context, id string, status query.SessionStatus) error { + store, err := s.storeFor(ctx) + if err != nil { + return err + } + return store.Update(ctx, id, status) +} + +func (s *environmentStores) Get(ctx context.Context, id string) (query.SessionRecord, bool, error) { + store, err := s.storeFor(ctx) + if err != nil { + return query.SessionRecord{}, false, err + } + return store.Get(ctx, id) +} + +func (s *environmentStores) List(ctx context.Context, filter query.SessionFilter) (query.SessionPage, error) { + store, err := s.storeFor(ctx) + if err != nil { + return query.SessionPage{}, err + } + return store.List(ctx, filter) +} + +func (s *environmentStores) Lineage(ctx context.Context, ids []string) (map[string][]string, error) { + store, err := s.storeFor(ctx) + if err != nil { + return nil, err + } + return store.Lineage(ctx, ids) +} + +type scopedHarness struct { + handler *sessionHandler + registry *query.SessionRegistry +} + +func newScopedHarness(sessions query.SessionStore) *scopedHarness { + h := &scopedHarness{} + h.registry = query.NewSessionRegistry(query.RegistryOptions{ + Store: sessions, + Scope: &query.SessionScope{Label: scopeEnvironmentLabel, Of: environmentOf}, + }) + ginkgo.DeferCleanup(func() { Expect(h.registry.StopAll(context.Background())).To(Succeed()) }) + profiles, err := profilepkg.NewFileStore(ginkgo.GinkgoT().TempDir()) + Expect(err).ToNot(HaveOccurred()) + h.handler = newSessionHandler(sessionHandlerOptions{ + Prefix: "/api/v1", Ctx: dbcontext.New(), Store: profiles, Registry: h.registry, Sessions: sessions, Next: &nextMarker{}, + }) + return h +} + +// trackIn starts a live jvm_trace capture in environment. +func (h *scopedHarness) trackIn(environment string) *query.Session { + stopAt := time.Now().Add(10 * time.Minute) + session, err := h.registry.Track(withEnvironment(context.Background(), environment), query.TrackOptions{ + Profile: jvmProfile, Kind: query.KindCapture, StopAt: &stopAt, + }) + Expect(err).ToNot(HaveOccurred()) + session.OnStop(func(string) { session.Finish(query.FinishUpdate{}) }) + Expect(session.Running(query.RunningUpdate{Handle: "probe-" + environment})).To(Succeed()) + return session +} + +func (h *scopedHarness) doIn(environment, method, path string) *httptest.ResponseRecorder { + request := httptest.NewRequest(method, path, nil) + request = request.WithContext(withEnvironment(request.Context(), environment)) + response := httptest.NewRecorder() + h.handler.ServeHTTP(response, request) + return response +} + +func (h *scopedHarness) listIn(environment, query string) []string { + response := h.doIn(environment, http.MethodGet, "/api/v1/sessions?"+query) + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + var body sessionListBody + Expect(json.Unmarshal(response.Body.Bytes(), &body)).To(Succeed()) + return infoIDs(body.Items) +} + +var _ = ginkgo.Describe("sessions API with an environment-scoped registry", func() { + for _, variant := range []struct { + name string + store func() query.SessionStore + }{ + {"with an environment-routed store", func() query.SessionStore { return &environmentStores{stores: map[string]*KVStore{}} }}, + {"with no store", func() query.SessionStore { return nil }}, + } { + ginkgo.Context(variant.name, func() { + var ( + h *scopedHarness + a, b *query.Session + ) + ginkgo.BeforeEach(func() { + h = newScopedHarness(variant.store()) + a, b = h.trackIn("env-a"), h.trackIn("env-b") + }) + + ginkgo.It("lists, counts and looks up only the live sessions of the request's environment", func() { + Expect(h.listIn("env-b", "")).To(Equal([]string{b.ID()})) + Expect(h.listIn("env-a", "")).To(Equal([]string{a.ID()})) + Expect(h.listIn("env-b", "profile="+jvmProfile)).To(Equal([]string{b.ID()})) + }) + + ginkgo.It("answers another environment's live session 404 on info, events and stop, and leaves it running", func() { + for _, route := range []struct{ method, path string }{ + {http.MethodGet, "/api/v1/sessions/" + a.ID()}, + {http.MethodGet, "/api/v1/sessions/" + a.ID() + "/events"}, + {http.MethodGet, "/api/v1/sessions/" + a.ID() + "/result"}, + {http.MethodPost, "/api/v1/sessions/" + a.ID() + "/stop"}, + {http.MethodPost, "/api/v1/sessions/" + a.ID() + "/extend?duration=1m"}, + } { + response := h.doIn("env-b", route.method, route.path) + Expect(response.Code).To(Equal(http.StatusNotFound), "%s %s: %s", route.method, route.path, response.Body.String()) + Expect(response.Body.String()).To(ContainSubstring(errSessionNotFound(a.ID()).Error())) + } + Expect(a.Snapshot().State).To(Equal(query.SessionRunning)) + + response := h.doIn("env-a", http.MethodPost, "/api/v1/sessions/"+a.ID()+"/stop") + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + }) + + ginkgo.It("refuses a request whose environment cannot be resolved instead of showing every session", func() { + request := httptest.NewRequest(http.MethodGet, "/api/v1/sessions", nil) + response := httptest.NewRecorder() + h.handler.ServeHTTP(response, request) + + Expect(response.Code).To(Equal(http.StatusInternalServerError)) + Expect(response.Body.String()).To(ContainSubstring("request names no environment")) + }) + }) + } +}) diff --git a/pkg/sessions/service_start.go b/pkg/sessions/service_start.go new file mode 100644 index 0000000..30a25a1 --- /dev/null +++ b/pkg/sessions/service_start.go @@ -0,0 +1,157 @@ +package sessions + +import ( + "context" + "errors" + "fmt" + "net/http" + "time" + + "github.com/flanksource/commons-db/cmd/query/profiles" + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/query/profilestore" + "github.com/flanksource/commons-db/types" +) + +func (h *sessionHandler) start(w http.ResponseWriter, r *http.Request, name string) { + if _, ok := r.URL.Query()["follow"]; ok { + http.Error(w, fmt.Sprintf("a session does not follow: live-tailing profile %q is a query read, GET %s/profile/%s?follow=true", name, h.prefix, name), http.StatusBadRequest) + return + } + // A surface key is the path the OpenAPI document hands out for a profile's + // session start, and the only one a name holding a "/" can take. + stored, err := profiles.StoredProfileName(r.Context(), h.store, name) + switch { + case errors.Is(err, profiles.ErrProfileSurfaceNotFound): + http.Error(w, err.Error(), http.StatusNotFound) + return + case err != nil: + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if !h.authorized(w, r, stored, ActionControl) { + return + } + resolved, err := profiles.Resolve(r.Context(), h.store, stored) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + p := resolved.Profile + if p, err = applySessionSpecOverrides(p, sessionSpec{ + Interval: r.URL.Query().Get("interval"), + Duration: r.URL.Query().Get("duration"), + }); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + params := map[string]any{} + for k, vs := range r.URL.Query() { + if sessionReservedParam(k) || len(vs) == 0 { + continue + } + params[k] = vs[0] + } + + session, err := query.ExecuteStream(h.sessionContext(r), h.registry, p, params) + if err != nil { + http.Error(w, err.Error(), sessionStartErrorStatus(err)) + return + } + h.writeLive(w, r, http.StatusCreated, session) +} + +func sessionStartErrorStatus(err error) int { + switch { + case errors.Is(err, query.ErrMaxSessions): + return http.StatusConflict + case errors.Is(err, query.ErrPrepareRead): + // The registry's BeforeRead failed: answered by its cause, exactly as + // the profile service answers its own hook. + status, _ := profiles.PrepareErrorStatus(err) + return status + } + return http.StatusBadRequest +} + +// writeLive answers with a live session's info and its overlays. +func (h *sessionHandler) writeLive(w http.ResponseWriter, r *http.Request, status int, session *query.Session) { + infos, err := h.overlay(r.Context(), []query.SessionRecord{session.Snapshot().SessionRecord}, h.readAllow(r)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + writeSessionJSON(w, status, infos[0]) +} + +// sessionContext is the context a session r starts runs under, and the one its +// registry's BeforeRead prepares the read with. It carries r's values — the +// tenant or environment a routed record store reads from the context — but not +// r's cancellation, because a session outlives the request that started it; its +// own duration bound and Stop end it. The server context's capabilities +// (connection resolver, logger, tracer, namespace, DB) are laid over them by +// Wrap, as every profile read does, and `profiles.` resolves from the +// handler's store. +func (h *sessionHandler) sessionContext(r *http.Request) dbcontext.Context { + return withSessionProfiles(h.ctx.Wrap(context.WithoutCancel(r.Context())), h.store, h.registry) +} + +// withSessionProfiles binds the `profiles.` references a top or trace +// session started under execCtx reads to store. Each is prepared by registry's +// BeforeRead under the context the reference is read in — the session's run +// context, which keeps the starting caller's values — as the session's own +// read is, at its start and before every later sample. +func withSessionProfiles(execCtx dbcontext.Context, store profilestore.Store, registry *query.SessionRegistry) dbcontext.Context { + return query.WithProfileResolver(execCtx, query.ProfileResolver{ + Resolve: func(ctx context.Context, name string) (query.Profile, error) { + resolved, err := profiles.Resolve(ctx, store, name) + return resolved.Profile, err + }, + Prepare: registry.PrepareRead, + }) +} + +// sessionSpec is the transport's side of a session request: the HTTP query +// params, or the equivalent CLI flags. +type sessionSpec struct { + Interval string + Duration string +} + +// applySessionSpecOverrides maps the transport inputs onto the profile: +// interval samples a plain profile as top (or overrides a declared interval), +// and duration lowers the session bound (the registry still clamps it). +// Live-tailing a plain profile is not a session but a query read, served by +// the profile route's ?follow. +func applySessionSpecOverrides(p query.Profile, spec sessionSpec) (query.Profile, error) { + if spec.Interval != "" { + d, err := time.ParseDuration(spec.Interval) + if err != nil { + return p, fmt.Errorf("invalid interval %q: %w", spec.Interval, err) + } + if p.Kind() == query.KindTrace { + return p, fmt.Errorf("profile %q is a trace; interval does not apply", p.Name) + } + if p.Top == nil { + p.Top = &query.TopSpec{} + } + p.Top.Interval = types.Duration{Duration: d} + } + if p.Kind() == query.KindQuery { + return p, fmt.Errorf("profile %q declares neither trace nor top; pass ?interval to sample it, or GET the profile with ?follow=true to tail it", p.Name) + } + if spec.Duration != "" { + d, err := time.ParseDuration(spec.Duration) + if err != nil { + return p, fmt.Errorf("invalid duration %q: %w", spec.Duration, err) + } + if p.Trace != nil { + p.Trace.MaxDuration = types.Duration{Duration: d} + } else { + p.Top.MaxDuration = types.Duration{Duration: d} + } + } + return p, nil +} diff --git a/pkg/sessions/service_start_follow_test.go b/pkg/sessions/service_start_follow_test.go new file mode 100644 index 0000000..1aa327c --- /dev/null +++ b/pkg/sessions/service_start_follow_test.go @@ -0,0 +1,48 @@ +package sessions + +import ( + "context" + "net/http" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + profilepkg "github.com/flanksource/commons-db/cmd/query/profiles" + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" +) + +var _ = ginkgo.Describe("a session start asked to follow", func() { + const profileName = "plain-follow" + var ( + handler *sessionHandler + registry *query.SessionRegistry + ) + + ginkgo.BeforeEach(func() { + query.RegisterProvider(&sessionStreamMock{typ: "sess-api-follow", rows: []query.Row{{"n": 1.0}}, block: true}) + store, err := profilepkg.NewFileStore(ginkgo.GinkgoT().TempDir()) + Expect(err).ToNot(HaveOccurred()) + p := execProfile(profileName) + p.Provider.Type = "sess-api-follow" + Expect(store.Save(context.Background(), p)).To(Succeed()) + registry = query.NewSessionRegistry(query.RegistryOptions{MaxSessions: 5}) + ginkgo.DeferCleanup(registry.StopAll) + handler = newSessionHandler(sessionHandlerOptions{ + Prefix: "/api/v1", Ctx: dbcontext.New(), Store: store, Registry: registry, Next: &nextMarker{}, + }) + }) + + ginkgo.DescribeTable("is refused 400 naming the query follow endpoint, starting nothing", + func(query string) { + response := doReq(handler, http.MethodPost, "/api/v1/profile/"+profileName+"/sessions?"+query) + Expect(response.Code).To(Equal(http.StatusBadRequest), response.Body.String()) + Expect(response.Body.String()).To(ContainSubstring("GET /api/v1/profile/" + profileName + "?follow=true")) + Expect(registry.List()).To(BeEmpty()) + }, + ginkgo.Entry("bare flag", "follow"), + ginkgo.Entry("true", "follow=true"), + ginkgo.Entry("with an interval", "follow=true&interval=1s"), + ginkgo.Entry("false still names a removed control", "follow=false"), + ) +}) diff --git a/pkg/sessions/service_table.go b/pkg/sessions/service_table.go new file mode 100644 index 0000000..916f5d6 --- /dev/null +++ b/pkg/sessions/service_table.go @@ -0,0 +1,119 @@ +package sessions + +import ( + "fmt" + "maps" + "net/http" + "slices" + "strings" + "time" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" + + "github.com/flanksource/commons-db/query" +) + +const clickyTableMediaType = "application/json+clicky" + +// sessionRow presents a session as a row of the sessions table. +type sessionRow query.SessionInfo + +// sessionStateColors are the badge colours of each state. +var sessionStateColors = map[query.SessionState][2]string{ + query.SessionStarting: {"bg-sky-500/10", "text-sky-700"}, + query.SessionRunning: {"bg-sky-500/10", "text-sky-700"}, + query.SessionStopping: {"bg-yellow-500/10", "text-yellow-700"}, + query.SessionCompleted: {"bg-emerald-500/10", "text-emerald-700"}, + query.SessionStopped: {"bg-slate-500/10", "text-slate-700"}, + query.SessionFailed: {"bg-red-500/10", "text-red-700"}, + query.SessionInterrupted: {"bg-yellow-500/10", "text-yellow-700"}, +} + +func (sessionRow) Columns() []api.ColumnDef { + return []api.ColumnDef{ + api.Column("_id").Hidden().Build(), + api.Column("startedAt").Label("Started").Kind("timestamp").SortKey("startedAt").Build(), + api.Column("session").Label("Session").MinWidthPixels(240).Build(), + api.Column("profile").Label("Profile").FilterKey("profile").SortKey("profile").Build(), + api.Column("target").Label("Target").FilterKey("label.target").Build(), + api.Column("state").Label("State").FilterKey("state").SortKey("state").Build(), + api.Column("eventCount").Label("Events").SortKey("eventCount").Build(), + api.Column("stoppedAt").Label("Stopped").Kind("timestamp").SortKey("stoppedAt").Build(), + api.Column("origin").Label("Origin").FilterKey("label.origin").Build(), + api.Column("principal").Label("Principal").FilterKey("principal").SortKey("principal").Build(), + api.Column("lineage").Label("Lineage").Build(), + api.Column("labels").Label("Labels").Kind("tags").Build(), + api.Column("owner").Label("Owner").Build(), + } +} + +func (s sessionRow) Row() map[string]any { + colors := sessionStateColors[s.State] + state := string(s.State) + if s.Unresponsive { + state += " · unresponsive" + } + label := s.Labels["label"] + if label == "" { + label = shortSessionID(s.ID) + } + labels := api.DescriptionList{} + for _, key := range slices.Sorted(maps.Keys(s.Labels)) { + if key != "label" { + labels.Items = append(labels.Items, api.KeyValuePair{Key: key, Value: s.Labels[key]}) + } + } + stopped := "" + if s.StoppedAt != nil { + stopped = s.StoppedAt.Format(time.RFC3339Nano) + } + return map[string]any{ + "_id": api.Text{Content: s.ID}, + "state": api.TableCell{Value: api.LabelBadge{Value: state, Color: colors[0], TextColor: colors[1], Shape: "pill"}, FilterValue: string(s.State)}, + "session": api.Text{Content: label}, + "profile": api.Text{Content: s.Profile}, + "target": api.Text{Content: s.Labels["target"]}, + "origin": api.Text{Content: s.Labels["origin"]}, + "principal": api.Text{Content: s.Principal}, + "startedAt": api.Text{Content: s.StartedAt.Format(time.RFC3339Nano)}, + "stoppedAt": api.Text{Content: stopped}, + "eventCount": s.EventCount, + "lineage": api.Text{Content: sessionLineage(query.SessionInfo(s))}, + "labels": labels, + "owner": api.Text{Content: fmt.Sprintf("%s · pid %d", s.Owner.Host, s.Owner.PID)}, + } +} + +func shortSessionID(id string) string { return id[:min(len(id), 8)] } + +func sessionLineage(info query.SessionInfo) string { + var parts []string + if info.RestartOf != "" { + parts = append(parts, "restart of "+shortSessionID(info.RestartOf)) + } + if len(info.RestartedAs) > 0 { + short := make([]string, len(info.RestartedAs)) + for i, id := range info.RestartedAs { + short[i] = shortSessionID(id) + } + parts = append(parts, "restarted as "+strings.Join(short, ", ")) + } + return strings.Join(parts, " · ") +} + +// writeSessionTable serves infos as the clicky table document. +func writeSessionTable(w http.ResponseWriter, infos []query.SessionInfo) { + rows := make([]sessionRow, len(infos)) + for i, info := range infos { + rows[i] = sessionRow(info) + } + document, err := clicky.Format(api.NewTableFrom(rows), clicky.FormatOptions{Format: query.ClickyPageFormat}) + if err != nil { + http.Error(w, fmt.Sprintf("render sessions table: %v", err), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", clickyTableMediaType) + w.Header().Set("Access-Control-Allow-Origin", "*") + _, _ = w.Write([]byte(document)) +} diff --git a/pkg/sessions/service_table_test.go b/pkg/sessions/service_table_test.go new file mode 100644 index 0000000..b5a7ee7 --- /dev/null +++ b/pkg/sessions/service_table_test.go @@ -0,0 +1,37 @@ +package sessions + +import ( + "github.com/flanksource/clicky/api" + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/commons-db/query" +) + +var _ = ginkgo.Describe("session table presentation", func() { + ginkgo.DescribeTable("gives every state its labelled semantic pill", + func(state query.SessionState, background, text string) { + row := sessionRow(query.SessionInfo{SessionRecord: query.SessionRecord{ + SessionStart: query.SessionStart{ID: "session-1", Labels: map[string]string{}}, + SessionStatus: query.SessionStatus{State: state}, + }}).Row() + cell, ok := row["state"].(api.TableCell) + Expect(ok).To(BeTrue()) + badge, ok := cell.Value.(api.LabelBadge) + Expect(ok).To(BeTrue()) + Expect(badge).To(And( + HaveField("Value", string(state)), + HaveField("Color", background), + HaveField("TextColor", text), + HaveField("Shape", "pill"), + )) + }, + ginkgo.Entry("starting", query.SessionStarting, "bg-sky-500/10", "text-sky-700"), + ginkgo.Entry("running", query.SessionRunning, "bg-sky-500/10", "text-sky-700"), + ginkgo.Entry("stopping", query.SessionStopping, "bg-yellow-500/10", "text-yellow-700"), + ginkgo.Entry("completed", query.SessionCompleted, "bg-emerald-500/10", "text-emerald-700"), + ginkgo.Entry("stopped", query.SessionStopped, "bg-slate-500/10", "text-slate-700"), + ginkgo.Entry("failed", query.SessionFailed, "bg-red-500/10", "text-red-700"), + ginkgo.Entry("interrupted", query.SessionInterrupted, "bg-yellow-500/10", "text-yellow-700"), + ) +}) diff --git a/pkg/sessions/service_test.go b/pkg/sessions/service_test.go new file mode 100644 index 0000000..6e0c213 --- /dev/null +++ b/pkg/sessions/service_test.go @@ -0,0 +1,253 @@ +package sessions + +import ( + "bufio" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + profilepkg "github.com/flanksource/commons-db/cmd/query/profiles" + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" + "github.com/stretchr/testify/require" +) + +type nextMarker struct{ hit bool } + +func (n *nextMarker) ServeHTTP(http.ResponseWriter, *http.Request) { n.hit = true } + +type execMock struct{ rows []query.Row } + +func (m *execMock) Type() string { return "session-exec-mock" } +func (m *execMock) Execute(_ dbcontext.Context, _ query.ProviderRequest) ([]query.Row, error) { + return m.rows, nil +} + +func execProfile(name string) query.Profile { + return query.Profile{ + Name: name, Provider: query.ProviderConfig{Type: "session-exec-mock"}, + Query: "select * where region = '{{.params.region}}'", + Params: []query.ParamDef{{Name: "region", Type: query.ParamTypeEnum, Options: []string{"US", "EU"}}}, + } +} + +// sessionStreamMock is a StreamProvider that emits fixed rows and then either +// ends or blocks until stopped. +type sessionStreamMock struct { + typ string + rows []query.Row + block bool +} + +func (m *sessionStreamMock) Type() string { return m.typ } + +func (m *sessionStreamMock) Execute(_ dbcontext.Context, _ query.ProviderRequest) ([]query.Row, error) { + return m.rows, nil +} + +func (m *sessionStreamMock) Stream(ctx dbcontext.Context, _ query.ProviderRequest, emit func(query.Row)) error { + for _, row := range m.rows { + emit(row) + } + if m.block { + <-ctx.Done() + return ctx.Err() + } + return nil +} + +func newSessionAPITest(t *testing.T, opts query.RegistryOptions, profiles ...query.Profile) (*sessionHandler, *query.SessionRegistry) { + t.Helper() + store, err := profilepkg.NewFileStore(t.TempDir()) + require.NoError(t, err) + for _, p := range profiles { + require.NoError(t, store.Save(context.Background(), p)) + } + registry := query.NewSessionRegistry(opts) + t.Cleanup(func() { require.NoError(t, registry.StopAll(context.Background())) }) + h := newSessionHandler(sessionHandlerOptions{ + Prefix: "/api/v1", + Ctx: dbcontext.New(), + Store: store, + Registry: registry, + Next: &nextMarker{}, + }) + return h, registry +} + +func traceTestProfile(name, providerType string) query.Profile { + return query.Profile{ + Name: name, + Provider: query.ProviderConfig{Type: providerType}, + Trace: &query.TraceSpec{}, + } +} + +func doReq(h http.Handler, method, path string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(method, path, nil)) + return rec +} + +func startSession(t *testing.T, h http.Handler, path string) query.SessionInfo { + t.Helper() + rec := doReq(h, http.MethodPost, path) + require.Equal(t, http.StatusCreated, rec.Code, rec.Body.String()) + var info query.SessionInfo + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &info)) + require.NotEmpty(t, info.ID) + return info +} + +func waitSessionState(t *testing.T, reg *query.SessionRegistry, id string, state query.SessionState) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if s, ok := reg.Get(id); ok && s.Snapshot().State == state { + return + } + time.Sleep(10 * time.Millisecond) + } + s, _ := reg.Get(id) + t.Fatalf("session %s never reached %s (now %s)", id, state, s.Snapshot().State) +} + +func TestSessionAPIStartsTraceSession(t *testing.T) { + query.RegisterProvider(&sessionStreamMock{typ: "sess-api-trace", rows: []query.Row{{"n": 1.0}}}) + h, reg := newSessionAPITest(t, query.RegistryOptions{MaxSessions: 5}, traceTestProfile("exec trace", "sess-api-trace")) + + info := startSession(t, h, "/api/v1/profile/exec-trace/sessions") + require.Equal(t, query.KindTrace, info.Kind) + waitSessionState(t, reg, info.ID, query.SessionCompleted) +} + +func TestSessionAPIRejectsPlainProfileWithoutInterval(t *testing.T) { + h, _ := newSessionAPITest(t, query.RegistryOptions{MaxSessions: 5}, execProfile("plain")) + rec := doReq(h, http.MethodPost, "/api/v1/profile/plain/sessions?region=EU") + require.Equal(t, http.StatusBadRequest, rec.Code) +} + +func TestSessionAPISynthesizesTopForPlainProfile(t *testing.T) { + query.RegisterProvider(&execMock{rows: []query.Row{{"id": 1}}}) + h, reg := newSessionAPITest(t, query.RegistryOptions{MaxSessions: 5}, execProfile("plain-top")) + + info := startSession(t, h, "/api/v1/profile/plain-top/sessions?interval=1s®ion=EU") + require.Equal(t, query.KindTop, info.Kind) + require.Equal(t, "EU", info.Params["region"], "filter params flow through, interval does not") + require.NotContains(t, info.Params, "interval") + + s, ok := reg.Get(info.ID) + require.True(t, ok) + s.Stop("test over") + waitSessionState(t, reg, info.ID, query.SessionStopped) +} + +func TestSessionAPIReturnsConflictAtCapacity(t *testing.T) { + query.RegisterProvider(&sessionStreamMock{typ: "sess-api-block", block: true}) + h, reg := newSessionAPITest(t, query.RegistryOptions{MaxSessions: 1}, traceTestProfile("blocker", "sess-api-block")) + + info := startSession(t, h, "/api/v1/profile/blocker/sessions") + waitSessionState(t, reg, info.ID, query.SessionRunning) + + rec := doReq(h, http.MethodPost, "/api/v1/profile/blocker/sessions") + require.Equal(t, http.StatusConflict, rec.Code, rec.Body.String()) +} + +func TestSessionAPIStopAndList(t *testing.T) { + query.RegisterProvider(&sessionStreamMock{typ: "sess-api-stop", rows: []query.Row{{"n": 1.0}}, block: true}) + h, reg := newSessionAPITest(t, query.RegistryOptions{MaxSessions: 5}, traceTestProfile("stoppable", "sess-api-stop")) + + info := startSession(t, h, "/api/v1/profile/stoppable/sessions") + waitSessionState(t, reg, info.ID, query.SessionRunning) + + rec := doReq(h, http.MethodGet, "/api/v1/sessions") + require.Equal(t, http.StatusOK, rec.Code) + var list sessionListResponse + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &list)) + require.Len(t, list.Items, 1) + + rec = doReq(h, http.MethodPost, "/api/v1/sessions/"+info.ID+"/stop") + require.Equal(t, http.StatusOK, rec.Code) + waitSessionState(t, reg, info.ID, query.SessionStopped) + + rec = doReq(h, http.MethodGet, "/api/v1/sessions/"+info.ID) + require.Equal(t, http.StatusOK, rec.Code) + var got query.SessionInfo + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got)) + require.Equal(t, query.SessionStopped, got.State) + + rec = doReq(h, http.MethodGet, "/api/v1/sessions/nope") + require.Equal(t, http.StatusNotFound, rec.Code) +} + +func TestSessionAPIStreamsEventsAsSSE(t *testing.T) { + query.RegisterProvider(&sessionStreamMock{typ: "sess-api-sse", rows: []query.Row{{"n": 1.0}, {"n": 2.0}}}) + h, reg := newSessionAPITest(t, query.RegistryOptions{MaxSessions: 5}, traceTestProfile("sse trace", "sess-api-sse")) + srv := httptest.NewServer(h) + t.Cleanup(srv.Close) + + info := startSession(t, h, "/api/v1/profile/sse-trace/sessions") + waitSessionState(t, reg, info.ID, query.SessionCompleted) + + resp, err := http.Get(srv.URL + "/api/v1/sessions/" + info.ID + "/events") + require.NoError(t, err) + t.Cleanup(func() { _ = resp.Body.Close() }) + require.Equal(t, "text/event-stream", resp.Header.Get("Content-Type")) + + var eventLines, doneLines int + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "event: event") { + eventLines++ + } + if strings.HasPrefix(line, "event: done") { + doneLines++ + } + } + require.Equal(t, 2, eventLines, "both buffered events are replayed") + require.Equal(t, 1, doneLines, "terminal done event closes the stream") +} + +func TestSessionAPIExportsNDJSON(t *testing.T) { + query.RegisterProvider(&sessionStreamMock{typ: "sess-api-ndjson", rows: []query.Row{{"n": 1.0}, {"n": 2.0}}}) + h, reg := newSessionAPITest(t, query.RegistryOptions{MaxSessions: 5}, traceTestProfile("ndjson trace", "sess-api-ndjson")) + + info := startSession(t, h, "/api/v1/profile/ndjson-trace/sessions") + waitSessionState(t, reg, info.ID, query.SessionCompleted) + + rec := doReq(h, http.MethodGet, "/api/v1/sessions/"+info.ID+"/events?format=ndjson") + require.Equal(t, http.StatusOK, rec.Code) + require.Equal(t, "application/x-ndjson", rec.Header().Get("Content-Type")) + + lines := strings.Split(strings.TrimSpace(rec.Body.String()), "\n") + require.Len(t, lines, 2) + var e query.Event + require.NoError(t, json.Unmarshal([]byte(lines[1]), &e)) + require.Equal(t, int64(2), e.Sequence) +} + +func TestSessionAPIServesResult(t *testing.T) { + query.RegisterProvider(&sessionStreamMock{typ: "sess-api-result", rows: []query.Row{{"n": 1.0}, {"n": 2.0}}}) + h, reg := newSessionAPITest(t, query.RegistryOptions{MaxSessions: 5}, traceTestProfile("result trace", "sess-api-result")) + + info := startSession(t, h, "/api/v1/profile/result-trace/sessions") + waitSessionState(t, reg, info.ID, query.SessionCompleted) + + rec := doReq(h, http.MethodGet, "/api/v1/sessions/"+info.ID+"/result") + require.Equal(t, http.StatusOK, rec.Code) + var rows []query.Row + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &rows)) + require.Len(t, rows, 2) +} + +func TestSessionAPIDelegatesUnrelatedPaths(t *testing.T) { + h, _ := newSessionAPITest(t, query.RegistryOptions{MaxSessions: 5}) + next := h.next.(*nextMarker) + _ = doReq(h, http.MethodGet, "/api/v1/profile/anything") + require.True(t, next.hit, "non-session paths fall through") +} diff --git a/pkg/sessions/sessions_openapi.go b/pkg/sessions/sessions_openapi.go new file mode 100644 index 0000000..940768c --- /dev/null +++ b/pkg/sessions/sessions_openapi.go @@ -0,0 +1,244 @@ +// The sessions API in the OpenAPI document: the list with its filters, window, +// sort and paging, one session and its actions, and the session start of each +// trace or top profile. + +package sessions + +import ( + "context" + "fmt" + "strings" + + "github.com/flanksource/clicky/entity" + "github.com/flanksource/clicky/rpc" + + "github.com/flanksource/commons-db/cmd/query/profiles" + "github.com/flanksource/commons-db/query" +) + +const ( + sessionsSurface = "sessions" + sessionsPath = "/api/v1/sessions" + sessionPath = sessionsPath + "/{id}" + profileSessionsPath = "/api/v1/profile/" +) + +// sessionFilter is one filter parameter of the sessions list. +type sessionFilter struct { + name, label string + value bool // a single typed value rather than a multi-select + enum []string +} + +func sessionFilters() []sessionFilter { + states := []string{string(query.SessionStarting), string(query.SessionRunning), string(query.SessionStopping), + string(query.SessionCompleted), string(query.SessionFailed), string(query.SessionStopped), string(query.SessionInterrupted)} + filters := []sessionFilter{ + {name: "profile", label: "Profile"}, + {name: "kind", label: "Kind", enum: []string{string(query.KindTrace), string(query.KindTop), string(query.KindCapture)}}, + {name: "state", label: "State", enum: states}, + {name: "principal", label: "Principal"}, + {name: "restartOf", label: "Restart of", value: true}, + } + for _, key := range []string{"target", "origin", "via", "runId", "plan", "step", "environment"} { + filters = append(filters, sessionFilter{name: "label." + key, label: key}) + } + return filters +} + +// AddSessionsOpenAPI emits the sessions API — the list with its filters, +// window, sort and paging, one session, and its stop, extend and restart +// actions — as the "sessions" surface a clicky-ui OperationCatalog renders. It +// is a profiles.OpenAPIExtension and an rpc.Config extension alike. +func AddSessionsOpenAPI(spec *rpc.OpenAPISpec) { + if spec.Paths == nil { + spec.Paths = map[string]rpc.OpenAPIPath{} + } + if spec.Clicky == nil { + spec.Clicky = &rpc.ClickySpecMeta{} + } + spec.Clicky.Surfaces = append(spec.Clicky.Surfaces, rpc.ClickySurface{Key: sessionsSurface, Entity: sessionsSurface, Title: "Sessions"}) + spec.Paths[sessionsPath] = rpc.OpenAPIPath{"get": sessionListOperation(spec)} + spec.Paths[sessionPath] = rpc.OpenAPIPath{"get": { + OperationID: "get-session", Summary: "Get a session", Parameters: []rpc.OpenAPIParameter{sessionIDParameter()}, + Responses: map[string]rpc.OpenAPIResponse{"200": {Description: "SessionInfo"}, "404": {Description: "No such session"}}, + Clicky: &rpc.ClickyOperationMeta{Command: sessionsSurface, Surface: sessionsSurface, Verb: "get", Scope: "entity", IDParam: "id"}, + }} + spec.Paths[sessionPath+"/stop"] = rpc.OpenAPIPath{"post": sessionAction("stop-session", "Stop a session", nil)} + spec.Paths[sessionPath+"/extend"] = rpc.OpenAPIPath{"post": sessionAction("extend-session", "Move a session's deadline later", + sessionDurationParameter(true, "Go duration added to stopAt, e.g. 900000ms or 15m; clamped to the registry's MaxDuration from the start"))} + restart := sessionAction("restart-session", "Start a new session from an ended one's params", + sessionDurationParameter(false, "Go duration for the new run, recorded as its params.durationMs; absent replays the ended "+ + "session's params.durationMs — the duration it was first requested for, not a run an extension lengthened")) + restart.Responses["201"] = rpc.OpenAPIResponse{Description: "The new session (SessionInfo) with restartOf set"} + restart.Responses["409"] = rpc.OpenAPIResponse{Description: "Not terminal, or not restartable here"} + delete(restart.Responses, "200") + spec.Paths[sessionPath+"/restart"] = rpc.OpenAPIPath{"post": restart} +} + +// AddProfileSessionsOpenAPI emits POST /api/v1/profile/{key}/sessions for +// every trace or top profile the service's store holds now, replacing the +// starts an earlier document held. It is an rpc.Config request extension. A +// query profile gets none: live-tailing it is the profile read's ?follow. +func (s *Service) AddProfileSessionsOpenAPI(ctx context.Context, spec *rpc.OpenAPISpec) error { + store, err := s.options.Profiles() + if err != nil { + return fmt.Errorf("profile session starts: %w", err) + } + saved, err := store.List(ctx) + if err != nil { + return fmt.Errorf("profile session starts: %w", err) + } + if spec.Paths == nil { + spec.Paths = map[string]rpc.OpenAPIPath{} + } + for path := range spec.Paths { + if strings.HasPrefix(path, profileSessionsPath) && strings.HasSuffix(path, "/sessions") { + delete(spec.Paths, path) + } + } + for _, p := range saved { + resolved, err := profiles.ResolveWithoutTouch(ctx, store, p.Name) + if err != nil { + return fmt.Errorf("profile session start %q: %w", p.Name, err) + } + if resolved.Profile.Kind() == query.KindQuery { + continue + } + key := profiles.SurfaceKey(resolved.Profile.Name) + spec.Paths[profileSessionsPath+key+"/sessions"] = rpc.OpenAPIPath{"post": profileSessionStart(resolved.Profile, key)} + } + return nil +} + +// profileSessionStart is the start operation of a trace or top profile: its +// params, a top's interval override and the duration bound. +func profileSessionStart(profile query.Profile, key string) rpc.OpenAPIOperation { + parameters := make([]rpc.OpenAPIParameter, 0, len(profile.Params)+2) + for _, param := range profile.Params { + parameters = append(parameters, profileSessionParameter(param)) + } + if profile.Kind() == query.KindTop { + parameters = append(parameters, rpc.OpenAPIParameter{Name: "interval", In: "query", + Description: "Go duration between samples, e.g. 5s; overrides the profile's", Schema: &rpc.OpenAPISchema{Type: "string"}}) + } + parameters = append(parameters, sessionDurationParameter(false, "Go duration the session runs for, e.g. 15m; clamped to the registry's MaxDuration")...) + return rpc.OpenAPIOperation{ + Summary: "Start a " + string(profile.Kind()) + " session for " + profile.Name, + Description: "Start a live session; follow it via GET /api/v1/sessions/{id}/events (SSE) and stop it via POST /api/v1/sessions/{id}/stop. " + + "Each event frame's data is a session event: a trace event's row is the raw row, and its clickyRow is the same row " + + "exactly as this profile's application/json+clicky table page presents it (node.rows[n])", + OperationID: "start-" + key + "-session", + Parameters: parameters, + Responses: map[string]rpc.OpenAPIResponse{ + "201": {Description: "Session started (SessionInfo)"}, + "400": {Description: "Invalid params, interval or duration"}, + "409": {Description: "The registry's session cap is reached"}, + }, + } +} + +func profileSessionParameter(param query.ParamDef) rpc.OpenAPIParameter { + schema := &rpc.OpenAPISchema{Type: "string", Title: param.DisplayLabel(), Description: param.Description, Default: param.Default} + switch param.Type { + case query.ParamTypeNumber: + schema.Type = "number" + case query.ParamTypeBoolean: + schema.Type = "boolean" + case query.ParamTypeDate: + schema.Format = "date" + case query.ParamTypeDateTime: + schema.Format = "date-time" + } + if param.Type != query.ParamTypeList && param.Type != query.ParamTypeLabels { + for _, option := range param.Options { + schema.Enum = append(schema.Enum, option) + } + } + return rpc.OpenAPIParameter{Name: param.Name, In: "query", Description: param.Description, Required: param.Required, Schema: schema} +} + +func sessionListOperation(spec *rpc.OpenAPISpec) rpc.OpenAPIOperation { + var parameters []rpc.OpenAPIParameter + for _, filter := range sessionFilters() { + component := "sessions-" + strings.ReplaceAll(filter.name, ".", "-") + shape := "multi-filter" + if filter.value { + shape = "value" + } + ensureFilterComponent(spec, entity.FilterSpec{Name: component, Label: filter.label, Type: shape, Multi: !filter.value}) + schema := &rpc.OpenAPISchema{Type: "string"} + for _, value := range filter.enum { + schema.Enum = append(schema.Enum, value) + } + parameters = append(parameters, rpc.OpenAPIParameter{ + Name: filter.name, In: "query", Schema: schema, Clicky: &rpc.ClickyParameterMeta{Role: "filter"}, + Description: "Include or exclude " + filter.label + " values: comma-separated, a leading ! excludes, * matches " + + "any run of characters. Repeated keys (?state=running&state=stopping) are the same selection.", + Lookup: &rpc.ClickyLookupMeta{ + Ref: "#/components/x-clicky-filters/" + component, URL: sessionsPath, Filter: filter.name, + SearchParam: "__lookup_q", Multi: !filter.value, + }, + }) + } + return rpc.OpenAPIOperation{ + OperationID: "list-sessions", Summary: "List sessions", + Description: "Capture sessions live in this process or persisted in the session store, authorized per profile " + + "before totals, paging and lookups.", + Parameters: append(parameters, sessionListPagingParameters()...), + Responses: map[string]rpc.OpenAPIResponse{"200": { + Description: "application/json: {items: SessionInfo[], total, shared}; application/json+clicky: a table document. " + + "Paging travels in X-Total-Count / X-Page-Limit / X-Page-Offset, and X-Sessions-Shared says whether the store is shared.", + }}, + Clicky: &rpc.ClickyOperationMeta{Command: sessionsSurface, Surface: sessionsSurface, Verb: "list", Scope: "collection"}, + } +} + +func ensureFilterComponent(spec *rpc.OpenAPISpec, filter entity.FilterSpec) { + if spec.Components == nil { + spec.Components = &rpc.OpenAPIComponents{} + } + if spec.Components.ClickyFilters == nil { + spec.Components.ClickyFilters = map[string]entity.FilterSpec{} + } + spec.Components.ClickyFilters[filter.Name] = filter +} + +func sessionListPagingParameters() []rpc.OpenAPIParameter { + param := func(name, role, description string, schema *rpc.OpenAPISchema) rpc.OpenAPIParameter { + return rpc.OpenAPIParameter{Name: name, In: "query", Description: description, Schema: schema, Clicky: &rpc.ClickyParameterMeta{Role: role}} + } + sortColumns := make([]any, len(query.SessionSortFields)) + for i, field := range query.SessionSortFields { + sortColumns[i] = field + } + return []rpc.OpenAPIParameter{ + param("from", "time-from", "Started from", &rpc.OpenAPISchema{Type: "string", Format: "date-time"}), + param("to", "time-to", "Started until", &rpc.OpenAPISchema{Type: "string", Format: "date-time"}), + param("sort", "sort", "Column to order by; id always breaks ties", &rpc.OpenAPISchema{Type: "string", Enum: sortColumns, Default: "startedAt"}), + param("order", "order", "Direction for sort", &rpc.OpenAPISchema{Type: "string", Enum: []any{"asc", "desc"}, Default: "desc"}), + param("limit", "limit", "Sessions per page (maximum 500)", &rpc.OpenAPISchema{Type: "integer", Default: 50}), + param("offset", "offset", "Sessions to skip", &rpc.OpenAPISchema{Type: "integer", Default: 0}), + } +} + +func sessionIDParameter() rpc.OpenAPIParameter { + return rpc.OpenAPIParameter{Name: "id", In: "path", Required: true, Schema: &rpc.OpenAPISchema{Type: "string"}} +} + +func sessionDurationParameter(required bool, description string) []rpc.OpenAPIParameter { + return []rpc.OpenAPIParameter{{Name: "duration", In: "query", Required: required, Description: description, Schema: &rpc.OpenAPISchema{Type: "string"}}} +} + +func sessionAction(operationID, summary string, parameters []rpc.OpenAPIParameter) rpc.OpenAPIOperation { + return rpc.OpenAPIOperation{ + OperationID: operationID, Summary: summary, + Parameters: append([]rpc.OpenAPIParameter{sessionIDParameter()}, parameters...), + Responses: map[string]rpc.OpenAPIResponse{ + "200": {Description: "The session as it stands after the request (SessionInfo)"}, + "403": {Description: "Authorize(r, profile, control) refused"}, + "404": {Description: "No such session"}, + "409": {Description: "The session's state does not allow this action"}, + }, + } +} diff --git a/pkg/sessions/sessions_openapi_test.go b/pkg/sessions/sessions_openapi_test.go new file mode 100644 index 0000000..cd35fd9 --- /dev/null +++ b/pkg/sessions/sessions_openapi_test.go @@ -0,0 +1,144 @@ +package sessions + +import ( + "context" + "slices" + + "github.com/flanksource/clicky/rpc" + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + profilepkg "github.com/flanksource/commons-db/cmd/query/profiles" + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/query/profilestore" +) + +func openAPIParameterNames(operation rpc.OpenAPIOperation) []string { + names := make([]string, 0, len(operation.Parameters)) + for _, parameter := range operation.Parameters { + names = append(names, parameter.Name) + } + return names +} + +func openAPIParameter(operation rpc.OpenAPIOperation, name string) rpc.OpenAPIParameter { + index := slices.IndexFunc(operation.Parameters, func(p rpc.OpenAPIParameter) bool { return p.Name == name }) + Expect(index).To(BeNumerically(">=", 0), "parameter %s", name) + return operation.Parameters[index] +} + +var _ = ginkgo.Describe("the sessions API's OpenAPI operations", func() { + var spec *rpc.OpenAPISpec + + ginkgo.BeforeEach(func() { + spec = &rpc.OpenAPISpec{Paths: map[string]rpc.OpenAPIPath{}} + AddSessionsOpenAPI(spec) + }) + + ginkgo.It("emits the list as the sessions surface's collection, with lookups on every filter", func() { + list := spec.Paths["/api/v1/sessions"]["get"] + Expect(list.OperationID).To(Equal("list-sessions")) + Expect(*list.Clicky).To(Equal(rpc.ClickyOperationMeta{Command: "sessions", Surface: "sessions", Verb: "list", Scope: "collection"})) + Expect(spec.Clicky.Surfaces).To(ContainElement(rpc.ClickySurface{Key: "sessions", Entity: "sessions", Title: "Sessions"})) + + Expect(openAPIParameterNames(list)).To(Equal([]string{ + "profile", "kind", "state", "principal", "restartOf", "label.target", "label.origin", "label.via", + "label.runId", "label.plan", "label.step", "label.environment", "from", "to", "sort", "order", "limit", "offset", + })) + Expect(*openAPIParameter(list, "label.runId").Lookup).To(Equal(rpc.ClickyLookupMeta{ + Ref: "#/components/x-clicky-filters/sessions-label-runId", URL: "/api/v1/sessions", + Filter: "label.runId", SearchParam: "__lookup_q", Multi: true, + })) + Expect(openAPIParameter(list, "restartOf").Lookup.Multi).To(BeFalse()) + Expect(openAPIParameter(list, "state").Schema.Enum).To(Equal([]any{"starting", "running", "stopping", "completed", "failed", "stopped", "interrupted"})) + Expect(openAPIParameter(list, "from").Clicky.Role).To(Equal("time-from")) + Expect(openAPIParameter(list, "sort").Schema.Enum).To(Equal([]any{"startedAt", "stoppedAt", "updatedAt", "state", "profile", "principal", "eventCount"})) + Expect(openAPIParameter(list, "limit").Schema.Default).To(Equal(50)) + + Expect(spec.Components.ClickyFilters["sessions-restartOf"]).To(And( + HaveField("Label", "Restart of"), HaveField("Type", "value"), HaveField("Multi", false))) + Expect(spec.Components.ClickyFilters["sessions-label-target"]).To(And( + HaveField("Label", "target"), HaveField("Type", "multi-filter"), HaveField("Multi", true))) + }) + + ginkgo.It("emits the detail and the stop, extend and restart actions", func() { + detail := spec.Paths["/api/v1/sessions/{id}"]["get"] + Expect(detail.OperationID).To(Equal("get-session")) + Expect(*detail.Clicky).To(Equal(rpc.ClickyOperationMeta{Command: "sessions", Surface: "sessions", Verb: "get", Scope: "entity", IDParam: "id"})) + + Expect(spec.Paths["/api/v1/sessions/{id}/stop"]["post"].OperationID).To(Equal("stop-session")) + extend := spec.Paths["/api/v1/sessions/{id}/extend"]["post"] + Expect(extend.OperationID).To(Equal("extend-session")) + Expect(openAPIParameter(extend, "duration").Required).To(BeTrue()) + restart := spec.Paths["/api/v1/sessions/{id}/restart"]["post"] + Expect(restart.OperationID).To(Equal("restart-session")) + Expect(openAPIParameter(restart, "duration").Required).To(BeFalse()) + Expect(restart.Responses).To(HaveKey("201")) + Expect(restart.Responses).ToNot(HaveKey("200")) + }) +}) + +var _ = ginkgo.Describe("the session start each profile's OpenAPI document offers", func() { + var ( + service *Service + spec *rpc.OpenAPISpec + ) + region := query.ParamDef{Name: "region", Type: query.ParamTypeEnum, Options: []string{"US", "EU"}} + profile := func(name string, shape func(*query.Profile)) query.Profile { + p := query.Profile{Name: name, Provider: query.ProviderConfig{Type: "session-exec-mock"}, Query: "q", Params: []query.ParamDef{region}} + shape(&p) + return p + } + + ginkgo.BeforeEach(func() { + store, err := profilepkg.NewFileStore(ginkgo.GinkgoT().TempDir()) + Expect(err).ToNot(HaveOccurred()) + for _, p := range []query.Profile{ + profile("declared trace", func(p *query.Profile) { p.Trace = &query.TraceSpec{} }), + profile("declared top", func(p *query.Profile) { p.Top = &query.TopSpec{} }), + profile("answers once", func(*query.Profile) {}), + } { + Expect(store.Save(context.Background(), p)).To(Succeed()) + } + service, err = New(Options{ + Profiles: func() (profilestore.Store, error) { return store, nil }, + Context: func() dbcontext.Context { return dbcontext.New() }, + Registry: query.NewSessionRegistry(query.RegistryOptions{}), + }) + Expect(err).ToNot(HaveOccurred()) + spec = &rpc.OpenAPISpec{Paths: map[string]rpc.OpenAPIPath{ + "/api/v1/profile/profile-removed/sessions": {"post": {OperationID: "start-profile-removed-session"}}, + }} + Expect(service.AddProfileSessionsOpenAPI(context.Background(), spec)).To(Succeed()) + }) + + start := func(name string) (rpc.OpenAPIOperation, bool) { + operation, ok := spec.Paths["/api/v1/profile/"+profilepkg.SurfaceKey(name)+"/sessions"]["post"] + return operation, ok + } + + ginkgo.It("starts a declared trace with its params and a duration, never a follow", func() { + operation, ok := start("declared trace") + Expect(ok).To(BeTrue()) + Expect(operation.OperationID).To(Equal("start-profile-declared-trace-session")) + Expect(operation.Description).To(ContainSubstring("POST /api/v1/sessions/{id}/stop")) + Expect(openAPIParameterNames(operation)).To(Equal([]string{"region", "duration"})) + Expect(openAPIParameter(operation, "region").Schema.Enum).To(Equal([]any{"US", "EU"})) + }) + + ginkgo.It("starts a declared top with an interval override", func() { + operation, ok := start("declared top") + Expect(ok).To(BeTrue()) + Expect(openAPIParameterNames(operation)).To(Equal([]string{"region", "interval", "duration"})) + }) + + ginkgo.It("offers a query profile no session start: following it is the profile read's ?follow", func() { + _, ok := start("answers once") + Expect(ok).To(BeFalse()) + }) + + ginkgo.It("drops the start of a profile the store no longer holds", func() { + Expect(spec.Paths).ToNot(HaveKey("/api/v1/profile/profile-removed/sessions")) + }) +}) diff --git a/pkg/sessions/sse.go b/pkg/sessions/sse.go new file mode 100644 index 0000000..34f4260 --- /dev/null +++ b/pkg/sessions/sse.go @@ -0,0 +1,93 @@ +package sessions + +import ( + "io" + "net/http" + "time" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/telltale/pkg/sessions/internal/sse" +) + +// sseStream is one client's view of a live session: where its replay begins — a +// resuming client already holds everything up to After — and how often an idle +// stream pings. +type sseStream struct { + session *query.Session + after int64 + keepalive time.Duration +} + +// run replays the buffered events the client has not seen, then follows the +// live channel until the session ends or the client disconnects. +// +// The keepalive is a case in this select rather than its own goroutine on a +// timer: every byte of the stream is written from here, so a ping can never +// land between the id and the data of an event frame. +func (s sseStream) run(w http.ResponseWriter, r *http.Request) { + replay, live, cancel := s.session.SubscribeFrom(s.after) + defer cancel() + + flusher, err := sse.Begin(w) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + for _, e := range replay { + if sse.WriteFrame(w, sse.Frame{Event: "event", ID: e.Sequence, Data: e}) != nil { + return + } + } + flusher.Flush() + + ticker := time.NewTicker(s.keepalive) + defer ticker.Stop() + for { + select { + case <-r.Context().Done(): + return + case <-ticker.C: + if _, err := io.WriteString(w, sse.KeepaliveFrame); err != nil { + return + } + flusher.Flush() + case e, open := <-live: + if !open { + _ = sse.WriteFrame(w, sse.Frame{Event: "done", Data: s.session.Snapshot()}) + flusher.Flush() + return + } + if sse.WriteFrame(w, sse.Frame{Event: "event", ID: e.Sequence, Data: e}) != nil { + return + } + flusher.Flush() + } + } +} + +// sseHistory serves a session that is already over. There is nothing to wait +// for, so it neither subscribes nor keeps the connection alive: replay, then +// done. +type sseHistory struct { + events []query.Event + info query.SessionInfo + after int64 +} + +func (h sseHistory) write(w http.ResponseWriter) { + flusher, err := sse.Begin(w) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + for _, e := range h.events { + if e.Sequence <= h.after { + continue + } + if sse.WriteFrame(w, sse.Frame{Event: "event", ID: e.Sequence, Data: e}) != nil { + return + } + } + _ = sse.WriteFrame(w, sse.Frame{Event: "done", Data: h.info}) + flusher.Flush() +} diff --git a/pkg/sessions/sse_test.go b/pkg/sessions/sse_test.go new file mode 100644 index 0000000..dcea0d5 --- /dev/null +++ b/pkg/sessions/sse_test.go @@ -0,0 +1,111 @@ +package sessions + +import ( + stdcontext "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/flanksource/commons-db/query" + "github.com/stretchr/testify/require" +) + +func doSSEReq(h http.Handler, path string, header map[string]string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodGet, path, nil) + for k, v := range header { + req.Header.Set(k, v) + } + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + return rec +} + +func TestSessionAPIFramesCarryTheirSequenceAsID(t *testing.T) { + query.RegisterProvider(&sessionStreamMock{typ: "sess-api-sse-id", rows: []query.Row{{"n": 1.0}, {"n": 2.0}}}) + h, reg := newSessionAPITest(t, query.RegistryOptions{MaxSessions: 5}, traceTestProfile("sse ids", "sess-api-sse-id")) + + info := startSession(t, h, "/api/v1/profile/sse-ids/sessions") + waitSessionState(t, reg, info.ID, query.SessionCompleted) + + body := doSSEReq(h, "/api/v1/sessions/"+info.ID+"/events", nil).Body.String() + require.Contains(t, body, "id: 1\nevent: event\n") + require.Contains(t, body, "id: 2\nevent: event\n") + require.NotContains(t, body, "id: 0", "the terminal done frame is not a resume point") +} + +func TestSessionAPIResumesFromLastEventID(t *testing.T) { + query.RegisterProvider(&sessionStreamMock{typ: "sess-api-sse-resume", rows: []query.Row{{"n": 1.0}, {"n": 2.0}, {"n": 3.0}}}) + h, reg := newSessionAPITest(t, query.RegistryOptions{MaxSessions: 5}, traceTestProfile("sse resume", "sess-api-sse-resume")) + + info := startSession(t, h, "/api/v1/profile/sse-resume/sessions") + waitSessionState(t, reg, info.ID, query.SessionCompleted) + + body := doSSEReq(h, "/api/v1/sessions/"+info.ID+"/events", + map[string]string{"Last-Event-ID": "2"}).Body.String() + require.NotContains(t, body, "id: 1\n") + require.NotContains(t, body, "id: 2\n") + require.Contains(t, body, "id: 3\n", "a resume replays the ring from the named sequence forward") + require.Contains(t, body, "event: done") +} + +func TestSessionAPIRejectsAnUnreadableLastEventID(t *testing.T) { + query.RegisterProvider(&sessionStreamMock{typ: "sess-api-sse-badid", rows: []query.Row{{"n": 1.0}}}) + h, reg := newSessionAPITest(t, query.RegistryOptions{MaxSessions: 5}, traceTestProfile("sse bad id", "sess-api-sse-badid")) + + info := startSession(t, h, "/api/v1/profile/sse-bad-id/sessions") + waitSessionState(t, reg, info.ID, query.SessionCompleted) + + rec := doSSEReq(h, "/api/v1/sessions/"+info.ID+"/events", map[string]string{"Last-Event-ID": "abc"}) + require.Equal(t, http.StatusBadRequest, rec.Code) + require.Contains(t, rec.Body.String(), "abc") +} + +// A tail can sit silent for minutes; the comment frames are what keeps the +// proxies in between from calling the connection dead. +func TestSSEKeepalivePingsAnIdleStream(t *testing.T) { + session, err := query.NewSession(query.SessionOptions{ + ID: "keepalive", Profile: traceTestProfile("idle", "sess-api-idle"), + Kind: query.KindTrace, Role: query.SessionRoleCapture, + }) + require.NoError(t, err) + + ctx, cancel := stdcontext.WithTimeout(stdcontext.Background(), 80*time.Millisecond) + defer cancel() + rec := httptest.NewRecorder() + sseStream{session: session, keepalive: 10 * time.Millisecond}.run(rec, httptest.NewRequest(http.MethodGet, "/", nil).WithContext(ctx)) + + require.GreaterOrEqual(t, strings.Count(rec.Body.String(), ": ping\n\n"), 2) +} + +// The keepalive shares the stream's one goroutine, so a ping can never land in +// the middle of an event frame. +func TestSSEKeepaliveNeverSplitsAnEventFrame(t *testing.T) { + session, err := query.NewSession(query.SessionOptions{ + ID: "interleave", Profile: traceTestProfile("busy", "sess-api-busy"), + Kind: query.KindTrace, Role: query.SessionRoleCapture, + }) + require.NoError(t, err) + go func() { + for i := 0; i < 50; i++ { + session.Emit(query.Event{Row: query.Row{"n": i}}) + time.Sleep(time.Millisecond) + } + }() + + ctx, cancel := stdcontext.WithTimeout(stdcontext.Background(), 80*time.Millisecond) + defer cancel() + rec := httptest.NewRecorder() + sseStream{session: session, keepalive: time.Millisecond}.run(rec, httptest.NewRequest(http.MethodGet, "/", nil).WithContext(ctx)) + + frames := strings.Split(strings.TrimSpace(rec.Body.String()), "\n\n") + require.NotEmpty(t, frames) + for _, frame := range frames { + if strings.HasPrefix(frame, ":") { + require.Equal(t, ": ping", frame) + continue + } + require.Regexp(t, `^id: \d+\nevent: event\ndata: \{.*\}$`, frame) + } +} diff --git a/pkg/sessions/store.go b/pkg/sessions/store.go new file mode 100644 index 0000000..a644250 --- /dev/null +++ b/pkg/sessions/store.go @@ -0,0 +1,416 @@ +package sessions + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "time" + + "gorm.io/gorm" + "gorm.io/gorm/clause" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/types" +) + +// sessionEventBatchSize triggers a synchronous flush; a background ticker +// flushes partial batches every sessionEventFlushInterval. +const ( + sessionEventBatchSize = 100 + sessionEventFlushInterval = time.Second +) + +// sessionRecord is one row of the sessions table: the start and status JSON a +// SessionRecord is read back from, and the scalar columns List filters, sorts +// and pages by in SQL. +type sessionRecord struct { + ID string `gorm:"column:id;primaryKey"` + ProfileName string `gorm:"column:profile_name"` + Kind string `gorm:"column:kind"` + Role string `gorm:"column:role;default:capture"` + Principal string `gorm:"column:principal"` + RestartOf string `gorm:"column:restart_of"` + State string `gorm:"column:state"` + Error string `gorm:"column:error"` + EventCount int64 `gorm:"column:event_count"` + StartedAt time.Time `gorm:"column:started_at"` + StoppedAt *time.Time `gorm:"column:stopped_at"` + Start types.JSON `gorm:"column:start;type:jsonb;not null;default:'{}'"` + Status types.JSON `gorm:"column:status;type:jsonb;not null;default:'{}'"` + CreatedAt time.Time `gorm:"column:created_at;default:now();<-:create"` + UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime:false"` +} + +func (sessionRecord) TableName() string { return "sessions" } + +type sessionEventRecord struct { + SessionID string `gorm:"column:session_id;primaryKey"` + Sequence int64 `gorm:"column:sequence;primaryKey"` + Time time.Time `gorm:"column:time"` + Payload types.JSON `gorm:"column:payload;type:jsonb"` +} + +func (sessionEventRecord) TableName() string { return "session_events" } + +// Store is the database-backed query.SessionStore and query.EventSink of the +// query server: session records in the sessions table, and the events of +// capture stream sessions batch-written to session_events, which is what a +// session's events are replayed from after it left the registry. +type Store struct { + db *gorm.DB + retention time.Duration + + mu sync.Mutex + pending map[string][]sessionEventRecord + failed map[string]error + + // resolve finds the live session so a background flush failure can fail it + // loudly. + resolve func(id string) (*query.Session, bool) + + stop chan struct{} + done chan struct{} +} + +var ( + _ query.SessionStore = (*Store)(nil) + _ query.EventSink = (*Store)(nil) +) + +// NewStore creates the store and starts its background flusher. +func NewStore(db *gorm.DB, retention time.Duration) (*Store, error) { + if db == nil { + return nil, fmt.Errorf("session database is required") + } + if retention <= 0 { + return nil, fmt.Errorf("session retention must be positive") + } + s := &Store{ + db: db, + retention: retention, + pending: map[string][]sessionEventRecord{}, + failed: map[string]error{}, + stop: make(chan struct{}), + done: make(chan struct{}), + } + go s.flushLoop() + return s, nil +} + +// BindResolver wires the live-session lookup used to fail sessions whose +// events cannot be persisted by the background flusher. +func (s *Store) BindResolver(resolve func(id string) (*query.Session, bool)) { + s.mu.Lock() + defer s.mu.Unlock() + s.resolve = resolve +} + +func newSessionRow(rec query.SessionRecord) (sessionRecord, error) { + start, err := json.Marshal(rec.SessionStart) + if err != nil { + return sessionRecord{}, fmt.Errorf("encode session %s start: %w", rec.ID, err) + } + row := sessionRecord{ + ID: rec.ID, ProfileName: rec.Profile, Kind: string(rec.Kind), Role: string(rec.Role), + Principal: rec.Principal, RestartOf: rec.RestartOf, StartedAt: rec.StartedAt, Start: types.JSON(start), + } + return row, row.setStatus(rec.ID, rec.SessionStatus) +} + +func (r *sessionRecord) setStatus(id string, status query.SessionStatus) error { + data, err := json.Marshal(status) + if err != nil { + return fmt.Errorf("encode session %s status: %w", id, err) + } + r.State, r.Error, r.EventCount = string(status.State), status.Error, status.EventCount + r.StoppedAt, r.UpdatedAt, r.Status = status.StoppedAt, status.UpdatedAt, types.JSON(data) + return nil +} + +func (s *Store) Begin(ctx context.Context, rec query.SessionRecord) error { + if rec.ID == "" || rec.StartedAt.IsZero() { + return fmt.Errorf("begin session: id %q and startedAt are both required", rec.ID) + } + row, err := newSessionRow(rec) + if err != nil { + return err + } + result := s.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(&row) + switch { + case result.Error != nil: + return fmt.Errorf("begin session %s: %w", rec.ID, result.Error) + case result.RowsAffected == 0: + return fmt.Errorf("begin session %s: a record already exists", rec.ID) + } + return nil +} + +// Update writes the status. The registry settles the session's events through +// Close before writing a terminal status, so the record never ends ahead of +// its events. +func (s *Store) Update(ctx context.Context, id string, status query.SessionStatus) error { + var row sessionRecord + if err := row.setStatus(id, status); err != nil { + return err + } + result := s.db.WithContext(ctx).Model(&sessionRecord{}).Where("id = ?", id).Updates(map[string]any{ + "state": row.State, "error": row.Error, "event_count": row.EventCount, + "stopped_at": row.StoppedAt, "updated_at": row.UpdatedAt, "status": row.Status, + }) + switch { + case result.Error != nil: + return fmt.Errorf("update session %s: %w", id, result.Error) + case result.RowsAffected == 0: + return fmt.Errorf("update session %s: no record; it was never begun or has been pruned", id) + } + return nil +} + +func (s *Store) Get(ctx context.Context, id string) (query.SessionRecord, bool, error) { + var row sessionRecord + err := s.db.WithContext(ctx).Where("id = ?", id).First(&row).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return query.SessionRecord{}, false, nil + } + if err != nil { + return query.SessionRecord{}, false, fmt.Errorf("get session %s: %w", id, err) + } + rec, err := row.record() + return rec, err == nil, err +} + +func (s *Store) Lineage(ctx context.Context, ids []string) (map[string][]string, error) { + lineage := map[string][]string{} + if len(ids) == 0 { + return lineage, nil + } + var rows []sessionRecord + err := s.db.WithContext(ctx).Select("id", "restart_of").Where("restart_of IN ?", ids). + Order("started_at, id").Find(&rows).Error + if err != nil { + return nil, fmt.Errorf("session lineage: %w", err) + } + for _, row := range rows { + lineage[row.RestartOf] = append(lineage[row.RestartOf], row.ID) + } + return lineage, nil +} + +func (r sessionRecord) record() (query.SessionRecord, error) { + var rec query.SessionRecord + if err := json.Unmarshal(r.Start, &rec.SessionStart); err != nil { + return rec, fmt.Errorf("decode session %s start: %w", r.ID, err) + } + if err := json.Unmarshal(r.Status, &rec.SessionStatus); err != nil { + return rec, fmt.Errorf("decode session %s status: %w", r.ID, err) + } + if rec.ID != r.ID { + return rec, fmt.Errorf("session %s: start record names id %q", r.ID, rec.ID) + } + return rec, nil +} + +// Append buffers the event, flushing synchronously once a full batch has +// accumulated. It returns the failure of an earlier background flush of the +// session, so the session emitting it fails. +func (s *Store) Append(_ context.Context, e query.Event) error { + payload, err := json.Marshal(e) + if err != nil { + return fmt.Errorf("encode session %s event %d: %w", e.SessionID, e.Sequence, err) + } + s.mu.Lock() + if failure, failed := s.failed[e.SessionID]; failed { + delete(s.failed, e.SessionID) + s.mu.Unlock() + return failure + } + s.pending[e.SessionID] = append(s.pending[e.SessionID], sessionEventRecord{ + SessionID: e.SessionID, Sequence: e.Sequence, Time: e.Time, Payload: types.JSON(payload), + }) + full := len(s.pending[e.SessionID]) >= sessionEventBatchSize + s.mu.Unlock() + + if full { + return s.flushSession(e.SessionID) + } + return nil +} + +// CloseSession writes whatever the session still has buffered, along with any +// failure the background flusher recorded for it while it had no live handle. +// The registry calls it as the session ends, before its terminal status is +// written, so the record a reader finds is never ahead of the events table. +func (s *Store) CloseSession(_ context.Context, id string) error { + s.mu.Lock() + recorded, failed := s.failed[id] + delete(s.failed, id) + s.mu.Unlock() + + err := s.flushSession(id) + if err == nil && !failed { + return nil + } + return fmt.Errorf("close session %s: %w", id, errors.Join(recorded, err)) +} + +// flushSession writes the events buffered for one session. A batch that fails +// is buffered again, ahead of anything appended since, so the retried terminal +// status write retries it too rather than landing without its events. +func (s *Store) flushSession(id string) error { + s.mu.Lock() + records := s.pending[id] + delete(s.pending, id) + s.mu.Unlock() + err := s.writeEvents(id, records) + if err != nil { + s.mu.Lock() + s.pending[id] = append(records, s.pending[id]...) + s.mu.Unlock() + } + return err +} + +func (s *Store) writeEvents(id string, records []sessionEventRecord) error { + if len(records) == 0 { + return nil + } + err := s.db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(records, sessionEventBatchSize).Error + if err != nil { + return fmt.Errorf("persist %d events for session %s: %w", len(records), id, err) + } + return nil +} + +// Flush writes all buffered events, joining any errors (including flush +// failures recorded for sessions with no live handle). +func (s *Store) Flush() error { + flushErr := s.flushPending() + s.mu.Lock() + errs := make([]error, 0, len(s.failed)+1) + for id, err := range s.failed { + errs = append(errs, fmt.Errorf("session %s: %w", id, err)) + } + s.failed = map[string]error{} + s.mu.Unlock() + return errors.Join(append(errs, flushErr)...) +} + +// flushPending writes every buffered batch, failing each session whose batch +// cannot be written. +func (s *Store) flushPending() error { + s.mu.Lock() + batches := s.pending + s.pending = map[string][]sessionEventRecord{} + s.mu.Unlock() + + var errs []error + for id, records := range batches { + if err := s.writeEvents(id, records); err != nil { + s.failSession(id, err) + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +// Close stops the background flusher after a final flush. +func (s *Store) Close() error { + close(s.stop) + <-s.done + return s.Flush() +} + +// Events returns the persisted events for a session, in sequence order. +func (s *Store) Events(ctx context.Context, id string) ([]query.Event, error) { + var records []sessionEventRecord + if err := s.db.WithContext(ctx).Where("session_id = ?", id).Order("sequence").Find(&records).Error; err != nil { + return nil, fmt.Errorf("list events for session %s: %w", id, err) + } + out := make([]query.Event, len(records)) + for i := range records { + if err := json.Unmarshal(records[i].Payload, &out[i]); err != nil { + return nil, fmt.Errorf("decode event %s/%d: %w", id, records[i].Sequence, err) + } + } + return out, nil +} + +// HasEvents reports whether any event of session id is buffered or persisted. +func (s *Store) HasEvents(ctx context.Context, id string) (bool, error) { + s.mu.Lock() + buffered := len(s.pending[id]) > 0 + s.mu.Unlock() + if buffered { + return true, nil + } + var held bool + err := s.db.WithContext(ctx).Raw("SELECT EXISTS (SELECT 1 FROM session_events WHERE session_id = ?)", id).Scan(&held).Error + if err != nil { + return false, fmt.Errorf("find events for session %s: %w", id, err) + } + return held, nil +} + +// Prune atomically deletes sessions stopped before the retention window and +// their events. +func (s *Store) Prune(ctx context.Context) error { + err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + var sessionIDs []string + if err := tx.Model(&sessionRecord{}). + Where("stopped_at IS NOT NULL AND stopped_at < ?", time.Now().Add(-s.retention)). + Pluck("id", &sessionIDs).Error; err != nil { + return fmt.Errorf("find expired sessions: %w", err) + } + if len(sessionIDs) == 0 { + return nil + } + if err := tx.Where("session_id IN ?", sessionIDs).Delete(&sessionEventRecord{}).Error; err != nil { + return fmt.Errorf("delete expired session events: %w", err) + } + if err := tx.Where("id IN ?", sessionIDs).Delete(&sessionRecord{}).Error; err != nil { + return fmt.Errorf("delete expired sessions: %w", err) + } + return nil + }) + if err != nil { + return fmt.Errorf("prune sessions: %w", err) + } + return nil +} + +func (s *Store) flushLoop() { + defer close(s.done) + ticker := time.NewTicker(sessionEventFlushInterval) + defer ticker.Stop() + for { + select { + case <-s.stop: + return + case <-ticker.C: + // A failed batch has already failed its session, or is held for + // the session's next Append, its CloseSession, or the next Flush. + _ = s.flushPending() + } + } +} + +// failSession fails the live session loudly; without a live handle the error +// is held for the session's next Append, CloseSession, or the next Flush. +func (s *Store) failSession(id string, err error) { + s.mu.Lock() + resolve := s.resolve + s.mu.Unlock() + if resolve != nil { + if session, ok := resolve(id); ok { + session.Abort(err) + return + } + } + s.mu.Lock() + defer s.mu.Unlock() + if _, exists := s.failed[id]; !exists { + s.failed[id] = err + } +} diff --git a/pkg/sessions/store_contract_test.go b/pkg/sessions/store_contract_test.go new file mode 100644 index 0000000..dd2e42c --- /dev/null +++ b/pkg/sessions/store_contract_test.go @@ -0,0 +1,171 @@ +package sessions + +import ( + "context" + "encoding/json" + "time" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/commons-db/query" +) + +// contractEpoch anchors the contract fixture an hour back, well inside every +// store's retention window, so window filters read as minute offsets. +func contractEpoch() time.Time { return time.Now().Add(-time.Hour).Truncate(time.Millisecond).UTC() } + +func contractRecord(epoch time.Time, id, profile string, state query.SessionState, startedMin int, events int64, labels map[string]string) query.SessionRecord { + started := epoch.Add(time.Duration(startedMin) * time.Minute) + return query.SessionRecord{ + SessionStart: query.SessionStart{ + SchemaVersion: query.SessionSchemaVersion, ID: id, Profile: profile, Kind: query.KindCapture, + Role: query.SessionRoleCapture, Labels: labels, Principal: "admin", + Params: map[string]any{"class": "com.example.Listener"}, + Owner: query.SessionOwner{Host: "pod-a", PID: 1, Boot: "boot-a"}, StartedAt: started, + }, + SessionStatus: query.SessionStatus{State: state, EventCount: events, UpdatedAt: started, HeartbeatAt: started}, + } +} + +// contractFixture mirrors the query package's filter fixture: b and c start at +// the same minute so a startedAt sort needs the id tie-breaker, and d has no +// labels at all. +func contractFixture(epoch time.Time) []query.SessionRecord { + restarted := contractRecord(epoch, "f", "trace-capture/jvm_trace", query.SessionRunning, 5, 1, map[string]string{"target": "cycle", "origin": "web"}) + restarted.RestartOf = "c" + restarted.Principal = "operator" + return []query.SessionRecord{ + contractRecord(epoch, "c", "trace-capture/jvm_trace", query.SessionStopped, 2, 30, map[string]string{"target": "cycle", "origin": "web"}), + contractRecord(epoch, "a", "trace-capture/jvm_trace", query.SessionRunning, 0, 10, map[string]string{"target": "oipa", "origin": "testplan"}), + contractRecord(epoch, "b", "trace-capture/sql_xevent", query.SessionCompleted, 2, 20, map[string]string{"target": "cycle", "origin": "web"}), + contractRecord(epoch, "d", "trace-capture/jvm_trace", query.SessionFailed, 3, 5, nil), + restarted, + } +} + +func pageIDs(page query.SessionPage) []string { + ids := make([]string, len(page.Items)) + for i, item := range page.Items { + ids[i] = item.ID + } + return ids +} + +// expectSameRecord compares records as their JSON, which is what every store +// persists and what the API serves. +func expectSameRecord(got, want query.SessionRecord) { + gotJSON, err := json.Marshal(got) + Expect(err).ToNot(HaveOccurred()) + wantJSON, err := json.Marshal(want) + Expect(err).ToNot(HaveOccurred()) + Expect(gotJSON).To(MatchJSON(wantJSON)) +} + +// describeSessionStoreContract registers the specs every query.SessionStore +// must pass; newStore is called before each spec for an empty store. +func describeSessionStoreContract(newStore func() query.SessionStore) { + var store query.SessionStore + var epoch time.Time + ctx := context.Background() + + ginkgo.BeforeEach(func() { + store, epoch = newStore(), contractEpoch() + for _, rec := range contractFixture(epoch) { + Expect(store.Begin(ctx, rec)).To(Succeed()) + } + }) + + ginkgo.It("returns a begun record exactly as it was written", func() { + want := contractFixture(epoch)[0] + got, found, err := store.Get(ctx, want.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeTrue()) + expectSameRecord(got, want) + + _, found, err = store.Get(ctx, "missing") + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeFalse()) + }) + + ginkgo.It("refuses to begin a record twice", func() { + Expect(store.Begin(ctx, contractFixture(epoch)[0])).To(MatchError(ContainSubstring("already exists"))) + }) + + ginkgo.It("overwrites the status of a begun record and refuses one never begun", func() { + rec := contractFixture(epoch)[1] + stopped := epoch.Add(9 * time.Minute) + rec.State, rec.StoppedAt, rec.StopReason, rec.EventCount = query.SessionStopped, &stopped, "stopped by admin", 99 + rec.Events = &query.EventsRef{Stream: "stream-a", Kind: "jvm_trace", From: 1, To: 99, Total: 99} + rec.Summary = json.RawMessage(`{"calls":99}`) + rec.Metadata = []query.SessionMetadata{{Name: "xe.statements", Label: "Started with", Language: "sql", Value: "CREATE EVENT SESSION [t] ON SERVER;"}} + rec.Exports = map[string]string{"HTTPS_PROXY": "http://127.0.0.1:41234"} + Expect(store.Update(ctx, rec.ID, rec.SessionStatus)).To(Succeed()) + + got, _, err := store.Get(ctx, rec.ID) + Expect(err).ToNot(HaveOccurred()) + expectSameRecord(got, rec) + Expect(store.Update(ctx, "never-begun", rec.SessionStatus)).To(MatchError(ContainSubstring("never-begun"))) + }) + + ginkgo.DescribeTable("lists records selected, authorized, sorted and paged", + func(filter func(epoch time.Time) query.SessionFilter, wantIDs []string, wantTotal int) { + page, err := store.List(ctx, filter(epoch)) + Expect(err).ToNot(HaveOccurred()) + Expect(pageIDs(page)).To(Equal(wantIDs)) + Expect(page.Total).To(Equal(wantTotal)) + }, + ginkgo.Entry("everything by startedAt ascending, id breaking the b/c tie", + func(time.Time) query.SessionFilter { return query.SessionFilter{} }, []string{"a", "b", "c", "d", "f"}, 5), + ginkgo.Entry("descending reverses both keys", + func(time.Time) query.SessionFilter { return query.SessionFilter{Desc: true} }, []string{"f", "d", "c", "b", "a"}, 5), + ginkgo.Entry("a profile suffix wildcard and a state exclusion", + func(time.Time) query.SessionFilter { + return query.SessionFilter{Profile: []string{"*jvm_trace"}, State: []string{"!failed"}} + }, []string{"a", "c", "f"}, 3), + ginkgo.Entry("states included case-insensitively", + func(time.Time) query.SessionFilter { + return query.SessionFilter{State: []string{"RUNNING", "stopped"}} + }, []string{"a", "c", "f"}, 3), + ginkgo.Entry("a label value, which a record without the label never matches", + func(time.Time) query.SessionFilter { + return query.SessionFilter{Labels: map[string][]string{"target": {"cycle"}}} + }, []string{"b", "c", "f"}, 3), + ginkgo.Entry("a label exclusion, which a record without the label matches", + func(time.Time) query.SessionFilter { + return query.SessionFilter{Labels: map[string][]string{"origin": {"!web"}}} + }, []string{"a", "d"}, 2), + ginkgo.Entry("principal and restartOf", + func(time.Time) query.SessionFilter { + return query.SessionFilter{Principal: []string{"oper*"}, RestartOf: []string{"c"}} + }, []string{"f"}, 1), + ginkgo.Entry("the startedAt window, inclusive at both ends", + func(epoch time.Time) query.SessionFilter { + return query.SessionFilter{From: epoch.Add(2 * time.Minute), To: epoch.Add(3 * time.Minute)} + }, []string{"b", "c", "d"}, 3), + ginkgo.Entry("Allow before the total, so refused profiles are never counted", + func(time.Time) query.SessionFilter { + return query.SessionFilter{Allow: func(profile string) bool { return profile == "trace-capture/sql_xevent" }} + }, []string{"b"}, 1), + ginkgo.Entry("eventCount descending with the page after the first", + func(time.Time) query.SessionFilter { + return query.SessionFilter{Sort: "eventCount", Desc: true, Limit: 2, Offset: 1} + }, []string{"b", "a"}, 5), + ginkgo.Entry("stoppedAt ascending puts the never-stopped first, by id", + func(time.Time) query.SessionFilter { return query.SessionFilter{Sort: "stoppedAt", Limit: 2} }, + []string{"a", "b"}, 5), + ginkgo.Entry("an offset past the end is an empty page with the whole total", + func(time.Time) query.SessionFilter { return query.SessionFilter{Offset: 10} }, []string{}, 5), + ) + + ginkgo.It("refuses a sort outside the allowlist", func() { + _, err := store.List(ctx, query.SessionFilter{Sort: "error"}) + Expect(err).To(MatchError(ContainSubstring(`session sort "error" is not one of`))) + }) + + ginkgo.It("maps each id to the sessions restarted from it", func() { + lineage, err := store.Lineage(ctx, []string{"c", "a"}) + Expect(err).ToNot(HaveOccurred()) + Expect(lineage).To(Equal(map[string][]string{"c": {"f"}})) + }) +} diff --git a/pkg/sessions/store_kv.go b/pkg/sessions/store_kv.go new file mode 100644 index 0000000..eab4a25 --- /dev/null +++ b/pkg/sessions/store_kv.go @@ -0,0 +1,296 @@ +package sessions + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/flanksource/clicky/cache" + + "github.com/flanksource/commons-db/query" +) + +// DefaultSessionTTL is how long a kv session record lives from its start. +const DefaultSessionTTL = 30 * 24 * time.Hour + +// kvExpiryMargin keeps List and Begin away from records about to expire, so a +// start and its status never race each other out of the store mid-read. +const kvExpiryMargin = time.Minute + +// KVResolver names the cache a request's session records live in and the key +// prefix that isolates its environment (e.g. "oipa.lab:"). +type KVResolver func(ctx context.Context) (cache.Store, string, error) + +// KVStoreOptions configures a KVStore. +type KVStoreOptions struct { + // TTL is how long a record lives from its StartedAt; zero is + // DefaultSessionTTL. + TTL time.Duration +} + +// KVStore is a query.SessionStore over a clicky cache.Store: +// +// sessions:v1::start immutable SessionStart JSON +// sessions:v1::status SessionStatus JSON, expiry pinned to start +// sessions:v1:index ZSET id → startedAt (unix ms) +type KVStore struct { + resolve KVResolver + ttl time.Duration + shared bool +} + +var _ query.SessionStore = (*KVStore)(nil) + +// NewKVStore stores records in the cache resolve names for each call; the +// cache is shared with every process using it. +func NewKVStore(resolve KVResolver, options KVStoreOptions) (*KVStore, error) { + if resolve == nil { + return nil, errors.New("kv session store: a resolver is required") + } + return newKVStore(resolve, options, true) +} + +// NewMemoryKVStore stores records in this process's memory for its lifetime. +func NewMemoryKVStore(options KVStoreOptions) (*KVStore, error) { + memory := cache.NewMemory() + return newKVStore(func(context.Context) (cache.Store, string, error) { return memory, "", nil }, options, false) +} + +func newKVStore(resolve KVResolver, options KVStoreOptions, shared bool) (*KVStore, error) { + switch { + case options.TTL < 0: + return nil, fmt.Errorf("kv session store: ttl %s must not be negative", options.TTL) + case options.TTL == 0: + options.TTL = DefaultSessionTTL + case options.TTL <= kvExpiryMargin: + return nil, fmt.Errorf("kv session store: ttl %s must exceed %s", options.TTL, kvExpiryMargin) + } + return &KVStore{resolve: resolve, ttl: options.TTL, shared: shared}, nil +} + +type kvKeys struct { + kv cache.Store + prefix string +} + +func (k kvKeys) start(id string) string { return k.prefix + "sessions:v1:" + id + ":start" } +func (k kvKeys) status(id string) string { return k.prefix + "sessions:v1:" + id + ":status" } +func (k kvKeys) index() string { return k.prefix + "sessions:v1:index" } + +func (s *KVStore) keys(ctx context.Context) (kvKeys, error) { + kv, prefix, err := s.resolve(ctx) + if err != nil { + return kvKeys{}, fmt.Errorf("resolve session store: %w", err) + } + if kv == nil { + return kvKeys{}, errors.New("resolve session store: resolver returned no cache") + } + return kvKeys{kv: kv, prefix: prefix}, nil +} + +// remaining is how long a record started at startedAt has left to live. +func (s *KVStore) remaining(startedAt time.Time) time.Duration { + return time.Until(startedAt.Add(s.ttl)) +} + +func (s *KVStore) Begin(ctx context.Context, rec query.SessionRecord) error { + if rec.ID == "" || rec.StartedAt.IsZero() { + return fmt.Errorf("begin session: id %q and startedAt are both required", rec.ID) + } + ttl := s.remaining(rec.StartedAt) + if ttl <= kvExpiryMargin { + return fmt.Errorf("begin session %s: started %s, outside the %s session ttl", rec.ID, rec.StartedAt.Format(time.RFC3339), s.ttl) + } + k, err := s.keys(ctx) + if err != nil { + return err + } + switch _, err := k.kv.Get(ctx, k.start(rec.ID)); { + case err == nil: + return fmt.Errorf("begin session %s: a record already exists", rec.ID) + case !errors.Is(err, cache.ErrKeyNotFound): + return fmt.Errorf("begin session %s: read start: %w", rec.ID, err) + } + start, err := json.Marshal(rec.SessionStart) + if err != nil { + return fmt.Errorf("begin session %s: encode start: %w", rec.ID, err) + } + if err := k.kv.Set(ctx, k.start(rec.ID), start, ttl); err != nil { + return fmt.Errorf("begin session %s: write start: %w", rec.ID, err) + } + if err := s.writeStatus(ctx, k, rec.ID, rec.SessionStatus, ttl); err != nil { + return fmt.Errorf("begin session %s: %w", rec.ID, err) + } + if err := k.kv.ZAdd(ctx, k.index(), float64(rec.StartedAt.UnixMilli()), rec.ID); err != nil { + return fmt.Errorf("begin session %s: index: %w", rec.ID, err) + } + cutoff := float64(time.Now().Add(-s.ttl).UnixMilli()) + if err := k.kv.ZRemRangeByScore(ctx, k.index(), cache.NegInf, cache.Exclusive(cutoff)); err != nil { + return fmt.Errorf("begin session %s: trim index: %w", rec.ID, err) + } + return nil +} + +func (s *KVStore) writeStatus(ctx context.Context, k kvKeys, id string, status query.SessionStatus, ttl time.Duration) error { + data, err := json.Marshal(status) + if err != nil { + return fmt.Errorf("encode status: %w", err) + } + if err := k.kv.Set(ctx, k.status(id), data, ttl); err != nil { + return fmt.Errorf("write status: %w", err) + } + return nil +} + +// Update overwrites the status with the expiry its start record has left. +func (s *KVStore) Update(ctx context.Context, id string, status query.SessionStatus) error { + k, err := s.keys(ctx) + if err != nil { + return err + } + data, err := k.kv.Get(ctx, k.start(id)) + if errors.Is(err, cache.ErrKeyNotFound) { + return fmt.Errorf("update session %s: no start record; it was never begun or has expired", id) + } + if err != nil { + return fmt.Errorf("update session %s: read start: %w", id, err) + } + var start query.SessionStart + if err := json.Unmarshal(data, &start); err != nil { + return fmt.Errorf("update session %s: decode start: %w", id, err) + } + ttl := s.remaining(start.StartedAt) + if ttl <= 0 { + return fmt.Errorf("update session %s: the record expired at %s", id, start.StartedAt.Add(s.ttl).Format(time.RFC3339)) + } + if err := s.writeStatus(ctx, k, id, status, ttl); err != nil { + return fmt.Errorf("update session %s: %w", id, err) + } + return nil +} + +// Delete removes a persisted session's start, status, and list index entry. +func (s *KVStore) Delete(ctx context.Context, id string) error { + if id == "" { + return fmt.Errorf("delete session: id is required") + } + k, err := s.keys(ctx) + if err != nil { + return err + } + if err := k.kv.ZRem(ctx, k.index(), id); err != nil { + return fmt.Errorf("delete session %s: remove index: %w", id, err) + } + if err := k.kv.Del(ctx, k.status(id)); err != nil { + return fmt.Errorf("delete session %s: remove status: %w", id, err) + } + if err := k.kv.Del(ctx, k.start(id)); err != nil { + return fmt.Errorf("delete session %s: remove start: %w", id, err) + } + return nil +} + +func (s *KVStore) Get(ctx context.Context, id string) (query.SessionRecord, bool, error) { + k, err := s.keys(ctx) + if err != nil { + return query.SessionRecord{}, false, err + } + records, missing, err := s.read(ctx, k, []string{id}) + if err != nil || len(missing) > 0 { + return query.SessionRecord{}, false, err + } + return records[0], true, nil +} + +// List reads every record in the filter's startedAt window (bounded by the +// ttl), dropping index entries whose start expired, then filters in Go. +func (s *KVStore) List(ctx context.Context, filter query.SessionFilter) (query.SessionPage, error) { + if err := filter.Validate(); err != nil { + return query.SessionPage{}, err + } + records, err := s.window(ctx, filter.From, filter.To) + if err != nil { + return query.SessionPage{}, err + } + page, err := query.ApplySessionFilter(records, filter) + page.Shared = s.shared + return page, err +} + +// Lineage reads only the records that could name ids: those started no earlier +// than the earliest of them, since a restart always starts after what it +// restarts. +func (s *KVStore) Lineage(ctx context.Context, ids []string) (map[string][]string, error) { + if len(ids) == 0 { + return map[string][]string{}, nil + } + k, err := s.keys(ctx) + if err != nil { + return nil, err + } + parents, _, err := s.read(ctx, k, ids) + if err != nil { + return nil, err + } + if len(parents) == 0 { + return map[string][]string{}, nil + } + earliest := parents[0].StartedAt + for _, parent := range parents[1:] { + if parent.StartedAt.Before(earliest) { + earliest = parent.StartedAt + } + } + records, err := s.window(ctx, earliest, time.Time{}) + if err != nil { + return nil, err + } + return lineageOf(records, ids), nil +} + +// lineageOf maps each of ids to the records restarted from it, in records' +// order. +func lineageOf(records []query.SessionRecord, ids []string) map[string][]string { + wanted := make(map[string]bool, len(ids)) + for _, id := range ids { + wanted[id] = true + } + lineage := map[string][]string{} + for _, rec := range records { + if rec.RestartOf != "" && wanted[rec.RestartOf] { + lineage[rec.RestartOf] = append(lineage[rec.RestartOf], rec.ID) + } + } + return lineage +} + +func (s *KVStore) window(ctx context.Context, from, to time.Time) ([]query.SessionRecord, error) { + k, err := s.keys(ctx) + if err != nil { + return nil, err + } + lower := time.Now().Add(-s.ttl + kvExpiryMargin) + if from.After(lower) { + lower = from + } + upper := cache.PosInf + if !to.IsZero() { + upper = cache.Inclusive(float64(to.UnixMilli())) + } + ids, err := k.kv.ZRangeByScore(ctx, k.index(), cache.Inclusive(float64(lower.UnixMilli())), upper) + if err != nil { + return nil, fmt.Errorf("list sessions: read index: %w", err) + } + records, missing, err := s.read(ctx, k, ids) + if err != nil { + return nil, err + } + for _, id := range missing { + if err := k.kv.ZRem(ctx, k.index(), id); err != nil { + return nil, fmt.Errorf("list sessions: drop expired %s from the index: %w", id, err) + } + } + return records, nil +} diff --git a/pkg/sessions/store_kv_read.go b/pkg/sessions/store_kv_read.go new file mode 100644 index 0000000..a576668 --- /dev/null +++ b/pkg/sessions/store_kv_read.go @@ -0,0 +1,198 @@ +package sessions + +import ( + "context" + "encoding/json" + "fmt" + "iter" + "slices" + "sync" + "sync/atomic" + "time" + + "github.com/flanksource/clicky/cache" + "github.com/flanksource/commons/logger" + + "github.com/flanksource/commons-db/query" +) + +// kvOrphanedRecords counts start records the kv store found without a status +// and removed. +var kvOrphanedRecords atomic.Int64 + +// KVOrphanedRecords counts the start records any KVStore in this process found +// without their status — evicted under maxmemory, say — and removed, with their +// index entries, rather than failing every list that reached them. +func KVOrphanedRecords() int64 { return kvOrphanedRecords.Load() } + +// kvReadMemo holds the records one request has already read, so the list, its +// lineage and its lookups read each record's keys at most once. +type kvReadMemo struct { + mu sync.Mutex + entries map[string]kvMemoEntry // prefix + id +} + +// kvMemoEntry is what reading one id found: its record, or that its start is +// gone, or neither — a start whose status is not written yet. +type kvMemoEntry struct { + record *query.SessionRecord + startGone bool +} + +type kvReadMemoKey struct{} + +// withSessionReadMemo returns ctx carrying a fresh read memo. A handler scopes +// it to one resolution: a record read again later, such as a stream's final +// status, must be read fresh. +func withSessionReadMemo(ctx context.Context) context.Context { + return context.WithValue(ctx, kvReadMemoKey{}, &kvReadMemo{entries: map[string]kvMemoEntry{}}) +} + +func readMemo(ctx context.Context) *kvReadMemo { + memo, _ := ctx.Value(kvReadMemoKey{}).(*kvReadMemo) + return memo +} + +// held returns the entries of ids the memo holds. A nil memo holds none. +func (m *kvReadMemo) held(prefix string, ids []string) map[string]kvMemoEntry { + held := map[string]kvMemoEntry{} + if m == nil { + return held + } + m.mu.Lock() + defer m.mu.Unlock() + for _, id := range ids { + if entry, ok := m.entries[prefix+id]; ok { + held[id] = entry + } + } + return held +} + +func (m *kvReadMemo) keep(prefix string, entries map[string]kvMemoEntry) { + if m == nil { + return + } + m.mu.Lock() + defer m.mu.Unlock() + for id, entry := range entries { + m.entries[prefix+id] = entry + } +} + +// read loads the records of ids in order, returning the ids whose start record +// is gone. Records ctx's memo holds are not read again. +func (s *KVStore) read(ctx context.Context, k kvKeys, ids []string) ([]query.SessionRecord, []string, error) { + memo := readMemo(ctx) + entries := memo.held(k.prefix, ids) + unread := slices.DeleteFunc(slices.Clone(ids), func(id string) bool { _, ok := entries[id]; return ok }) + fetched, err := s.fetch(ctx, k, unread) + if err != nil { + return nil, nil, err + } + memo.keep(k.prefix, fetched) + records := make([]query.SessionRecord, 0, len(ids)) + var missing []string + for _, id := range ids { + entry, ok := entries[id] + if !ok { + entry = fetched[id] + } + switch { + case entry.record != nil: + records = append(records, *entry.record) + case entry.startGone: + missing = append(missing, id) + } + } + return records, missing, nil +} + +// fetch reads ids' start and status keys in one MGet. A start without its +// status is an orphan once it is older than a Begin's two writes could be +// apart: it is removed and reported missing. +func (s *KVStore) fetch(ctx context.Context, k kvKeys, ids []string) (map[string]kvMemoEntry, error) { + keys := func(yield func(string) bool) { + for _, id := range ids { + if !yield(k.start(id)) || !yield(k.status(id)) { + return + } + } + } + next, stop := iter.Pull2(k.kv.MGet(ctx, keys)) + defer stop() + entries := make(map[string]kvMemoEntry, len(ids)) + for _, id := range ids { + start, err := pullEntry(next, id) + if err != nil { + return nil, err + } + status, err := pullEntry(next, id) + if err != nil { + return nil, err + } + switch { + case !start.Found: + entries[id] = kvMemoEntry{startGone: true} + case !status.Found: + if err := s.dropOrphan(ctx, k, id, start.Value); err != nil { + return nil, err + } + entries[id] = kvMemoEntry{} + default: + rec, err := decodeKVRecord(id, start.Value, status.Value) + if err != nil { + return nil, err + } + entries[id] = kvMemoEntry{record: &rec} + } + } + return entries, nil +} + +// dropOrphan removes a start record whose status is gone, and its index entry. +// A start younger than kvExpiryMargin may be a Begin between its two writes and +// is left alone; either way the record is not listed. +func (s *KVStore) dropOrphan(ctx context.Context, k kvKeys, id string, start []byte) error { + var rec query.SessionStart + if err := json.Unmarshal(start, &rec); err != nil { + return fmt.Errorf("decode session %s start: %w", id, err) + } + if time.Since(rec.StartedAt) < kvExpiryMargin { + return nil + } + logger.Errorf("session %s (%s): start record has no status, likely evicted; removing it from the session store", id, rec.Profile) + if err := k.kv.Del(ctx, k.start(id)); err != nil { + return fmt.Errorf("session %s: remove start record orphaned by its missing status: %w", id, err) + } + if err := k.kv.ZRem(ctx, k.index(), id); err != nil { + return fmt.Errorf("session %s: remove orphaned record from the index: %w", id, err) + } + kvOrphanedRecords.Add(1) + return nil +} + +func pullEntry(next func() (cache.Entry, error, bool), id string) (cache.Entry, error) { + entry, err, ok := next() + switch { + case err != nil: + return cache.Entry{}, fmt.Errorf("read session %s: %w", id, err) + case !ok: + return cache.Entry{}, fmt.Errorf("read session %s: the cache returned fewer entries than keys", id) + } + return entry, nil +} + +func decodeKVRecord(id string, start, status []byte) (query.SessionRecord, error) { + var rec query.SessionRecord + if err := json.Unmarshal(start, &rec.SessionStart); err != nil { + return rec, fmt.Errorf("decode session %s start: %w", id, err) + } + if err := json.Unmarshal(status, &rec.SessionStatus); err != nil { + return rec, fmt.Errorf("decode session %s status: %w", id, err) + } + if rec.ID != id { + return rec, fmt.Errorf("session %s: start record names id %q", id, rec.ID) + } + return rec, nil +} diff --git a/pkg/sessions/store_kv_test.go b/pkg/sessions/store_kv_test.go new file mode 100644 index 0000000..3e3e1a9 --- /dev/null +++ b/pkg/sessions/store_kv_test.go @@ -0,0 +1,261 @@ +package sessions + +import ( + "context" + "encoding/json" + "errors" + "iter" + "strings" + "sync" + "time" + + "github.com/flanksource/clicky/cache" + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/commons-db/query" +) + +// ttlRecordingCache is a memory cache that remembers the ttl of every Set, so +// a spec can see the expiry a store pinned without waiting for it. +type ttlRecordingCache struct { + cache.Store + mu sync.Mutex + ttls map[string]time.Duration +} + +func newTTLRecordingCache() *ttlRecordingCache { + return &ttlRecordingCache{Store: cache.NewMemory(), ttls: map[string]time.Duration{}} +} + +func (c *ttlRecordingCache) Set(ctx context.Context, key string, value []byte, ttl time.Duration) error { + c.mu.Lock() + c.ttls[key] = ttl + c.mu.Unlock() + return c.Store.Set(ctx, key, value, ttl) +} + +func (c *ttlRecordingCache) ttl(key string) time.Duration { + c.mu.Lock() + defer c.mu.Unlock() + return c.ttls[key] +} + +// countingCache counts how often each session's start key is read, so a spec +// can see which records a request read and how many times. +type countingCache struct { + cache.Store + mu sync.Mutex + reads map[string]int +} + +func (c *countingCache) MGet(ctx context.Context, keys iter.Seq[string]) iter.Seq2[cache.Entry, error] { + c.mu.Lock() + for key := range keys { + if strings.HasSuffix(key, ":start") { + c.reads[key]++ + } + } + c.mu.Unlock() + return c.Store.MGet(ctx, keys) +} + +func (c *countingCache) Get(ctx context.Context, key string) ([]byte, error) { + c.mu.Lock() + if strings.HasSuffix(key, ":start") { + c.reads[key]++ + } + c.mu.Unlock() + return c.Store.Get(ctx, key) +} + +// readsOf is how often session id's start key was read under any prefix. +func (c *countingCache) readsOf(id string) int { + c.mu.Lock() + defer c.mu.Unlock() + total := 0 + for key, reads := range c.reads { + if strings.HasSuffix(key, "sessions:v1:"+id+":start") { + total += reads + } + } + return total +} + +func (c *countingCache) reset() { + c.mu.Lock() + defer c.mu.Unlock() + c.reads = map[string]int{} +} + +var _ = ginkgo.Describe("KVStore", func() { + ginkgo.Describe("over the memory cache", func() { + describeSessionStoreContract(func() query.SessionStore { + store, err := NewMemoryKVStore(KVStoreOptions{}) + Expect(err).ToNot(HaveOccurred()) + return store + }) + }) + + const prefix = "oipa.lab:" + var ( + ctx context.Context + backing *ttlRecordingCache + store *KVStore + epoch time.Time + ) + + ginkgo.BeforeEach(func() { + ctx, backing, epoch = context.Background(), newTTLRecordingCache(), contractEpoch() + var err error + store, err = NewKVStore(func(context.Context) (cache.Store, string, error) { return backing, prefix, nil }, KVStoreOptions{TTL: 48 * time.Hour}) + Expect(err).ToNot(HaveOccurred()) + }) + + ginkgo.It("writes the start, the status and the index under the environment prefix", func() { + rec := contractFixture(epoch)[0] + Expect(store.Begin(ctx, rec)).To(Succeed()) + + start, err := backing.Get(ctx, prefix+"sessions:v1:c:start") + Expect(err).ToNot(HaveOccurred()) + wantStart, _ := json.Marshal(rec.SessionStart) + Expect(start).To(MatchJSON(wantStart)) + status, err := backing.Get(ctx, prefix+"sessions:v1:c:status") + Expect(err).ToNot(HaveOccurred()) + wantStatus, _ := json.Marshal(rec.SessionStatus) + Expect(status).To(MatchJSON(wantStatus)) + + indexed, err := backing.ZRangeByScore(ctx, prefix+"sessions:v1:index", + cache.Inclusive(float64(rec.StartedAt.UnixMilli())), cache.Inclusive(float64(rec.StartedAt.UnixMilli()))) + Expect(err).ToNot(HaveOccurred()) + Expect(indexed).To(Equal([]string{"c"})) + }) + + ginkgo.It("deletes one session record and removes it from the index", func() { + records := contractFixture(epoch) + Expect(store.Begin(ctx, records[0])).To(Succeed()) + Expect(store.Begin(ctx, records[1])).To(Succeed()) + + Expect(store.Delete(ctx, records[0].ID)).To(Succeed()) + _, found, err := store.Get(ctx, records[0].ID) + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeFalse()) + page, err := store.List(ctx, query.SessionFilter{}) + Expect(err).NotTo(HaveOccurred()) + Expect(page.Items).To(HaveLen(1)) + Expect(page.Items[0].ID).To(Equal(records[1].ID)) + }) + + ginkgo.It("pins the status expiry to the start record's", func() { + rec := contractFixture(epoch)[0] + Expect(store.Begin(ctx, rec)).To(Succeed()) + startTTL := backing.ttl(prefix + "sessions:v1:c:start") + Expect(startTTL).To(BeNumerically("~", 47*time.Hour+2*time.Minute, time.Minute), "48h less the 58 minutes it has run") + + rec.State = query.SessionStopped + Expect(store.Update(ctx, rec.ID, rec.SessionStatus)).To(Succeed()) + Expect(backing.ttl(prefix + "sessions:v1:c:status")).To(BeNumerically("~", startTTL, time.Second)) + }) + + ginkgo.It("refuses a record that has already outlived the ttl", func() { + rec := contractFixture(epoch)[0] + rec.StartedAt = time.Now().Add(-49 * time.Hour) + Expect(store.Begin(ctx, rec)).To(MatchError(ContainSubstring("outside the 48h0m0s session ttl"))) + }) + + ginkgo.It("drops index entries whose start record is gone, and trims those older than the ttl", func() { + for _, rec := range contractFixture(epoch) { + Expect(store.Begin(ctx, rec)).To(Succeed()) + } + index := prefix + "sessions:v1:index" + Expect(backing.ZAdd(ctx, index, float64(time.Now().Add(-72*time.Hour).UnixMilli()), "ancient")).To(Succeed()) + Expect(backing.Del(ctx, prefix+"sessions:v1:b:start")).To(Succeed()) + + page, err := store.List(ctx, query.SessionFilter{}) + Expect(err).ToNot(HaveOccurred()) + Expect(pageIDs(page)).To(Equal([]string{"a", "c", "d", "f"})) + Expect(page.Shared).To(BeTrue()) + members, err := backing.ZRangeByScore(ctx, index, cache.NegInf, cache.PosInf) + Expect(err).ToNot(HaveOccurred()) + Expect(members).ToNot(ContainElement("b"), "a missing start removes its index entry") + Expect(members).To(ContainElement("ancient"), "only a write trims by age") + + Expect(store.Begin(ctx, contractRecord(epoch, "g", "trace-capture/jvm_trace", query.SessionRunning, 6, 0, nil))).To(Succeed()) + members, err = backing.ZRangeByScore(ctx, index, cache.NegInf, cache.PosInf) + Expect(err).ToNot(HaveOccurred()) + Expect(members).ToNot(ContainElement("ancient")) + }) + + ginkgo.It("removes a start record whose status was evicted, counts it, and lists the rest", func() { + for _, rec := range contractFixture(epoch) { + Expect(store.Begin(ctx, rec)).To(Succeed()) + } + orphansBefore := KVOrphanedRecords() + Expect(backing.Del(ctx, prefix+"sessions:v1:c:status")).To(Succeed()) + + page, err := store.List(ctx, query.SessionFilter{}) + Expect(err).ToNot(HaveOccurred()) + Expect(pageIDs(page)).To(Equal([]string{"a", "b", "d", "f"})) + Expect(KVOrphanedRecords()).To(Equal(orphansBefore + 1)) + _, err = backing.Get(ctx, prefix+"sessions:v1:c:start") + Expect(err).To(MatchError(cache.ErrKeyNotFound)) + members, err := backing.ZRangeByScore(ctx, prefix+"sessions:v1:index", cache.NegInf, cache.PosInf) + Expect(err).ToNot(HaveOccurred()) + Expect(members).ToNot(ContainElement("c")) + _, found, err := store.Get(ctx, "c") + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeFalse()) + }) + + ginkgo.It("leaves a just-begun start whose status is not written yet, unlisted but in place", func() { + rec := contractRecord(time.Now(), "fresh", jvmProfile, query.SessionStarting, 0, 0, nil) + Expect(store.Begin(ctx, rec)).To(Succeed()) + Expect(backing.Del(ctx, prefix+"sessions:v1:fresh:status")).To(Succeed()) + + page, err := store.List(ctx, query.SessionFilter{}) + Expect(err).ToNot(HaveOccurred()) + Expect(pageIDs(page)).To(BeEmpty()) + _, err = backing.Get(ctx, prefix+"sessions:v1:fresh:start") + Expect(err).ToNot(HaveOccurred()) + members, err := backing.ZRangeByScore(ctx, prefix+"sessions:v1:index", cache.NegInf, cache.PosInf) + Expect(err).ToNot(HaveOccurred()) + Expect(members).To(ContainElement("fresh")) + }) + + ginkgo.It("reads only records started no earlier than the ids it maps, for lineage", func() { + counting := &countingCache{Store: backing, reads: map[string]int{}} + store, err := NewKVStore(func(context.Context) (cache.Store, string, error) { return counting, prefix, nil }, KVStoreOptions{TTL: 48 * time.Hour}) + Expect(err).ToNot(HaveOccurred()) + for _, rec := range contractFixture(epoch) { + Expect(store.Begin(ctx, rec)).To(Succeed()) + } + counting.reset() + + lineage, err := store.Lineage(ctx, []string{"c"}) + Expect(err).ToNot(HaveOccurred()) + Expect(lineage).To(Equal(map[string][]string{"c": {"f"}})) + Expect(counting.readsOf("a")).To(BeZero(), "a started before c and cannot be its restart") + }) + + ginkgo.It("surfaces a resolver failure", func() { + broken, err := NewKVStore(func(context.Context) (cache.Store, string, error) { + return nil, "", errors.New("no L2 for environment x") + }, KVStoreOptions{}) + Expect(err).ToNot(HaveOccurred()) + _, err = broken.List(ctx, query.SessionFilter{}) + Expect(err).To(MatchError(ContainSubstring("no L2 for environment x"))) + }) + + ginkgo.It("reports a memory store as not shared", func() { + memory, err := NewMemoryKVStore(KVStoreOptions{}) + Expect(err).ToNot(HaveOccurred()) + page, err := memory.List(ctx, query.SessionFilter{}) + Expect(err).ToNot(HaveOccurred()) + Expect(page.Shared).To(BeFalse()) + }) + + ginkgo.It("rejects a negative ttl", func() { + _, err := NewMemoryKVStore(KVStoreOptions{TTL: -time.Second}) + Expect(err).To(MatchError(ContainSubstring("ttl"))) + }) +}) diff --git a/pkg/sessions/store_list.go b/pkg/sessions/store_list.go new file mode 100644 index 0000000..3707773 --- /dev/null +++ b/pkg/sessions/store_list.go @@ -0,0 +1,201 @@ +package sessions + +import ( + "context" + "fmt" + "net/url" + "strings" + + "gorm.io/gorm" + + "github.com/flanksource/commons-db/query" +) + +// List filters the scalar fields and the window in SQL, and authorizes by +// narrowing to the allowed profiles among those that match. Without a label +// filter SQL also counts, sorts and pages; labels live in the start JSON, so +// with one the matches are paged in Go. +func (s *Store) List(ctx context.Context, filter query.SessionFilter) (query.SessionPage, error) { + if err := filter.Validate(); err != nil { + return query.SessionPage{}, err + } + empty := query.SessionPage{Items: []query.SessionRecord{}, Shared: true} + scoped := func() *gorm.DB { return scalarWhere(s.db.WithContext(ctx).Model(&sessionRecord{}), filter) } + if filter.Allow != nil { + allowed, err := allowedProfiles(scoped(), filter.Allow) + if err != nil || len(allowed) == 0 { + return empty, err + } + base := scoped + scoped = func() *gorm.DB { return base().Where("profile_name IN ?", allowed) } + } + if len(filter.Labels) > 0 { + return s.listInGo(scoped(), filter) + } + var total int64 + if err := scoped().Count(&total).Error; err != nil { + return query.SessionPage{}, fmt.Errorf("count sessions: %w", err) + } + page := scoped().Order(sessionOrder(filter.Sort, filter.Desc)).Offset(filter.Offset) + if filter.Limit > 0 { + page = page.Limit(filter.Limit) + } + records, err := findRecords(page) + if err != nil { + return query.SessionPage{}, err + } + return query.SessionPage{Items: records, Total: int(total), Shared: true}, nil +} + +func (s *Store) listInGo(scoped *gorm.DB, filter query.SessionFilter) (query.SessionPage, error) { + records, err := findRecords(scoped) + if err != nil { + return query.SessionPage{}, err + } + filter.Allow = nil + page, err := query.ApplySessionFilter(records, filter) + page.Shared = true + return page, err +} + +func findRecords(db *gorm.DB) ([]query.SessionRecord, error) { + var rows []sessionRecord + if err := db.Find(&rows).Error; err != nil { + return nil, fmt.Errorf("list sessions: %w", err) + } + records := make([]query.SessionRecord, 0, len(rows)) + for _, row := range rows { + rec, err := row.record() + if err != nil { + return nil, err + } + records = append(records, rec) + } + return records, nil +} + +func allowedProfiles(db *gorm.DB, allow func(string) bool) ([]string, error) { + var profiles []string + if err := db.Distinct("profile_name").Pluck("profile_name", &profiles).Error; err != nil { + return nil, fmt.Errorf("list session profiles: %w", err) + } + allowed := make([]string, 0, len(profiles)) + for _, profile := range profiles { + if allow(profile) { + allowed = append(allowed, profile) + } + } + return allowed, nil +} + +// sessionOrder is filter.Sort as SQL, with the id breaking ties in the same +// direction. Text sorts by byte order, as the Go filter does. +func sessionOrder(field string, desc bool) string { + direction, nulls := "ASC", "" + if desc { + direction = "DESC" + } + column := map[string]string{ + "": "started_at", "startedAt": "started_at", "updatedAt": "updated_at", "eventCount": "event_count", + "stoppedAt": "stopped_at", "state": `state COLLATE "C"`, "profile": `profile_name COLLATE "C"`, + "principal": `COALESCE(principal, '') COLLATE "C"`, + }[field] + if field == "stoppedAt" { + // A session that has not stopped sorts before any that has, as in Go. + nulls = map[bool]string{false: " NULLS FIRST", true: " NULLS LAST"}[desc] + } + return fmt.Sprintf(`%s %s%s, id COLLATE "C" %s`, column, direction, nulls, direction) +} + +// scalarWhere adds the filter's scalar patterns and startedAt window. +func scalarWhere(db *gorm.DB, filter query.SessionFilter) *gorm.DB { + for column, patterns := range map[string][]string{ + "id": filter.IDs, "profile_name": filter.Profile, "kind": filter.Kind, "role": filter.Role, + "state": filter.State, "COALESCE(principal, '')": filter.Principal, "COALESCE(restart_of, '')": filter.RestartOf, + } { + if len(patterns) > 0 { + clause, args := matchItemsSQL(column, patterns) + db = db.Where(clause, args...) + } + } + if !filter.From.IsZero() { + db = db.Where("started_at >= ?", filter.From) + } + if !filter.To.IsZero() { + db = db.Where("started_at <= ?", filter.To) + } + return db +} + +// matchItemsSQL is collections.MatchItems as a SQL predicate over column: any +// exclusion refuses, then any inclusion accepts, and a list of only exclusions +// accepts what none refused. Matching ignores case; `*` is a wildcard only at +// either end. +func matchItemsSQL(column string, patterns []string) (string, []any) { + var includes, excludes []string + var includeArgs, excludeArgs []any + for _, pattern := range normalizeMatchPatterns(patterns) { + negated := strings.HasPrefix(pattern, "!") + clause, args := matchPatternSQL(column, strings.TrimPrefix(pattern, "!")) + if negated { + excludes, excludeArgs = append(excludes, "NOT ("+clause+")"), append(excludeArgs, args...) + } else { + includes, includeArgs = append(includes, clause), append(includeArgs, args...) + } + } + switch { + case len(includes) == 0 && len(excludes) == 0: + return "FALSE", nil + case len(includes) == 0: + return strings.Join(excludes, " AND "), excludeArgs + case len(excludes) == 0: + return "(" + strings.Join(includes, " OR ") + ")", includeArgs + } + return "(" + strings.Join(includes, " OR ") + ") AND " + strings.Join(excludes, " AND "), + append(includeArgs, excludeArgs...) +} + +// matchPatternSQL mirrors collections' matchPattern for one pattern. +func matchPatternSQL(column, pattern string) (string, []any) { + if pattern == "*" { + return "TRUE", nil + } + clauses := []string{"LOWER(" + column + ") = LOWER(?)"} + args := []any{pattern} + prefixed, suffixed := strings.HasPrefix(pattern, "*"), strings.HasSuffix(pattern, "*") + if prefixed && suffixed { + clauses, args = append(clauses, column+" ILIKE ?"), append(args, "%"+likeEscape(strings.TrimPrefix(strings.TrimSuffix(pattern, "*"), "*"))+"%") + } + if prefixed { + clauses, args = append(clauses, column+" ILIKE ?"), append(args, "%"+likeEscape(strings.TrimPrefix(pattern, "*"))) + } + if suffixed { + clauses, args = append(clauses, column+" ILIKE ?"), append(args, likeEscape(strings.TrimSuffix(pattern, "*"))+"%") + } + return strings.Join(clauses, " OR "), args +} + +func likeEscape(value string) string { + return strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(value) +} + +// normalizeMatchPatterns splits comma lists and unescapes each part exactly as +// collections.MatchItems does before matching. +func normalizeMatchPatterns(patterns []string) []string { + var normalized []string + for _, pattern := range patterns { + parts := strings.Split(pattern, ",") + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" && len(parts) > 1 { + continue + } + decoded, err := url.QueryUnescape(part) + if err != nil { + continue + } + normalized = append(normalized, decoded) + } + } + return normalized +} diff --git a/pkg/sessions/store_test.go b/pkg/sessions/store_test.go new file mode 100644 index 0000000..7da1111 --- /dev/null +++ b/pkg/sessions/store_test.go @@ -0,0 +1,146 @@ +package sessions + +import ( + "context" + "errors" + "sync/atomic" + "time" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" + + "github.com/flanksource/commons-db/dbtest" + "github.com/flanksource/commons-db/query" +) + +func sessionStoreDB() *gorm.DB { + gdb := dbtest.ForGinkgo(dbtest.Options{Name: "query_sessions", LogName: "query-sessions-test"}).Gorm() + Expect(gdb.AutoMigrate(&sessionRecord{}, &sessionEventRecord{})).To(Succeed()) + return gdb +} + +func newGormStore(gdb *gorm.DB, retention time.Duration) *Store { + store, err := NewStore(gdb, retention) + Expect(err).ToNot(HaveOccurred()) + ginkgo.DeferCleanup(store.Close) + return store +} + +var _ = ginkgo.Describe("Store (gorm)", func() { + ginkgo.Describe("as a session store", func() { + describeSessionStoreContract(func() query.SessionStore { + return newGormStore(sessionStoreDB(), 7*24*time.Hour) + }) + }) + + var ( + ctx context.Context + store *Store + epoch time.Time + ) + + ginkgo.BeforeEach(func() { + ctx, epoch = context.Background(), contractEpoch() + store = newGormStore(sessionStoreDB(), time.Hour) + }) + + appendEvents := func(id string, count int) { + for i := 1; i <= count; i++ { + Expect(store.Append(ctx, query.Event{SessionID: id, Sequence: int64(i), Time: epoch, Row: query.Row{"n": float64(i)}})).To(Succeed()) + } + } + + ginkgo.It("flushes a session's buffered events when the session is closed", func() { + rec := contractFixture(epoch)[1] + Expect(store.Begin(ctx, rec)).To(Succeed()) + appendEvents(rec.ID, 3) + + // The registry closes the sink before writing the terminal status, so + // by the time the record reads as ended the events are already here. + Expect(store.CloseSession(ctx, rec.ID)).To(Succeed()) + rec.State = query.SessionCompleted + Expect(store.Update(ctx, rec.ID, rec.SessionStatus)).To(Succeed()) + events, err := store.Events(ctx, rec.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(events).To(HaveLen(3)) + Expect(events[1].Row).To(Equal(query.Row{"n": 2.0})) + }) + + ginkgo.It("flushes a full batch as it is appended", func() { + rec := contractFixture(epoch)[1] + Expect(store.Begin(ctx, rec)).To(Succeed()) + appendEvents(rec.ID, sessionEventBatchSize) + events, err := store.Events(ctx, rec.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(events).To(HaveLen(sessionEventBatchSize)) + }) + + ginkgo.It("prunes sessions stopped before the retention window, with their events", func() { + old := contractFixture(epoch)[0] + longAgo := time.Now().Add(-2 * time.Hour) + old.StoppedAt = &longAgo + fresh := contractFixture(epoch)[1] + Expect(store.Begin(ctx, old)).To(Succeed()) + Expect(store.Begin(ctx, fresh)).To(Succeed()) + appendEvents(old.ID, 1) + Expect(store.Flush()).To(Succeed()) + + Expect(store.Prune(ctx)).To(Succeed()) + _, found, err := store.Get(ctx, old.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeFalse()) + events, err := store.Events(ctx, old.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(events).To(BeEmpty()) + _, found, err = store.Get(ctx, fresh.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeTrue()) + }) + + ginkgo.It("keeps a batch whose write failed, so a later close lands with its events", func() { + gdb := sessionStoreDB() + store := newGormStore(gdb, time.Hour) + var refusals atomic.Int32 + refusals.Store(1) + const callback = "sessions-spec:refuse-events" + Expect(gdb.Callback().Create().Before("gorm:create").Register(callback, func(db *gorm.DB) { + if db.Statement.Table == "session_events" && refusals.Add(-1) >= 0 { + _ = db.AddError(errors.New("disk full")) + } + })).To(Succeed()) + ginkgo.DeferCleanup(func() { Expect(gdb.Callback().Create().Remove(callback)).To(Succeed()) }) + + rec := contractFixture(epoch)[1] + Expect(store.Begin(ctx, rec)).To(Succeed()) + for i := int64(1); i <= 3; i++ { + Expect(store.Append(ctx, query.Event{SessionID: rec.ID, Sequence: i, Time: epoch, Row: query.Row{"n": float64(i)}})).To(Succeed()) + } + Expect(store.CloseSession(ctx, rec.ID)).To(MatchError(ContainSubstring("disk full"))) + held, err := store.HasEvents(ctx, rec.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(held).To(BeTrue(), "the refused batch is still buffered") + + Expect(store.CloseSession(ctx, rec.ID)).To(Succeed()) + rec.State = query.SessionCompleted + Expect(store.Update(ctx, rec.ID, rec.SessionStatus)).To(Succeed()) + events, err := store.Events(ctx, rec.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(events).To(HaveLen(3)) + stored, _, err := store.Get(ctx, rec.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(stored.State).To(Equal(query.SessionCompleted)) + held, err = store.HasEvents(ctx, "never-emitted") + Expect(err).ToNot(HaveOccurred()) + Expect(held).To(BeFalse()) + }) + + ginkgo.It("lists nothing when Allow refuses every profile", func() { + for _, rec := range contractFixture(epoch) { + Expect(store.Begin(ctx, rec)).To(Succeed()) + } + page, err := store.List(ctx, query.SessionFilter{Allow: func(string) bool { return false }}) + Expect(err).ToNot(HaveOccurred()) + Expect(page).To(Equal(query.SessionPage{Items: []query.SessionRecord{}, Total: 0, Shared: true})) + }) +}) diff --git a/pkg/sessions/suite_test.go b/pkg/sessions/suite_test.go new file mode 100644 index 0000000..17baaa6 --- /dev/null +++ b/pkg/sessions/suite_test.go @@ -0,0 +1,13 @@ +package sessions + +import ( + "testing" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +func TestSessions(t *testing.T) { + gomega.RegisterFailHandler(ginkgo.Fail) + ginkgo.RunSpecs(t, "Sessions Suite") +} diff --git a/pkg/sessions/top.go b/pkg/sessions/top.go new file mode 100644 index 0000000..0273b9d --- /dev/null +++ b/pkg/sessions/top.go @@ -0,0 +1,99 @@ +package sessions + +import ( + "context" + "fmt" + "os/signal" + "syscall" + + "github.com/flanksource/clicky/api" + "github.com/flanksource/clicky/task" + "github.com/flanksource/commons-db/cmd/query/profiles" + "github.com/flanksource/commons-db/query" + commonsContext "github.com/flanksource/commons/context" +) + +type TopOptions struct { + Interval string + Duration string + Params []string +} + +func (r *Runner) RunTop(ctx context.Context, name string, options TopOptions) error { + store, err := r.profiles() + if err != nil { + return err + } + resolved, err := profiles.Resolve(ctx, store, name) + if err != nil { + return err + } + p := resolved.Profile + if p.Kind() == query.KindTrace { + return fmt.Errorf("profile %q is a trace; use `query trace` to stream it", p.Name) + } + interval := options.Interval + if p.Top == nil && interval == "" { + interval = query.DefaultTopInterval.String() + } + if p, err = applySessionSpecOverrides(p, sessionSpec{Interval: interval, Duration: options.Duration}); err != nil { + return err + } + params, err := parseSessionParams(options.Params) + if err != nil { + return err + } + + session, err := r.startCLISession(store, p, params) + if err != nil { + return err + } + + sigCtx, cancelSig := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) + defer cancelSig() + go func() { + <-sigCtx.Done() + session.Stop("interrupted by signal") + }() + + // The clicky task keeps the render loop alive for the session's lifetime; + // the live renderer swaps the task tree for the latest snapshot table. + task.SetLiveRenderer(&topLiveRenderer{session: session, profile: p}) + defer task.SetLiveRenderer(nil) + t := task.StartTask("top "+p.Name, func(_ commonsContext.Context, _ *task.Task) (any, error) { + _, live, cancel := session.Subscribe() + defer cancel() + for range live { // drain until the session reaches a terminal state + } + return nil, nil + }) + _ = t.WaitFor() + + return r.finishCLISession(session) +} + +// topLiveRenderer renders the session's latest snapshot through the clicky +// table formatter on every live frame. +type topLiveRenderer struct { + session *query.Session + profile query.Profile +} + +func (r *topLiveRenderer) RenderLive(_ []*task.Task) api.Text { return r.render() } +func (r *topLiveRenderer) RenderFinal(_ []*task.Task) api.Text { return r.render() } + +func (r *topLiveRenderer) render() api.Text { + snap := r.session.Snapshot() + header := fmt.Sprintf("top %s every %s samples=%d state=%s\n", + r.profile.Name, r.profile.Top.TickInterval(), snap.EventCount, snap.State) + + latest := r.session.Latest() + if latest == nil { + return api.Text{Content: header + "waiting for the first sample…"} + } + table, err := latest.Render(r.profile.Columns, "pretty") + if err != nil { + return api.Text{Content: header + "render error: " + err.Error()} + } + return api.Text{Content: header + table} +} diff --git a/pkg/sessions/trace.go b/pkg/sessions/trace.go new file mode 100644 index 0000000..3bb9cce --- /dev/null +++ b/pkg/sessions/trace.go @@ -0,0 +1,200 @@ +package sessions + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os/signal" + "sort" + "strconv" + "strings" + "syscall" + "time" + + "github.com/flanksource/commons-db/cmd/query/profiles" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/query/profilestore" + "github.com/flanksource/telltale/pkg/sessions/internal/paramfile" +) + +type RunnerOptions struct { + Profiles ProfileStoreProvider + Context ContextProvider + Stdout io.Writer + Stderr io.Writer + + // BeforeExecute, when set, prepares a CLI session's data as it starts and + // before every later sample — the hook the profile service runs, so a + // session reads what a one-off run of the profile would. + BeforeExecute profilestore.BeforeExecuteFunc +} + +type Runner struct { + profiles ProfileStoreProvider + context ContextProvider + stdout io.Writer + stderr io.Writer + beforeExecute profilestore.BeforeExecuteFunc +} + +func NewRunner(options RunnerOptions) (*Runner, error) { + if options.Profiles == nil { + return nil, fmt.Errorf("profile store provider is required") + } + if options.Context == nil { + return nil, fmt.Errorf("context provider is required") + } + if options.Stdout == nil || options.Stderr == nil { + return nil, fmt.Errorf("session output writers are required") + } + return &Runner{ + profiles: options.Profiles, context: options.Context, stdout: options.Stdout, stderr: options.Stderr, + beforeExecute: options.BeforeExecute, + }, nil +} + +type TraceOptions struct { + Duration string + Params []string + Output string +} + +func (r *Runner) RunTrace(ctx context.Context, name string, options TraceOptions) error { + store, err := r.profiles() + if err != nil { + return err + } + resolved, err := profiles.Resolve(ctx, store, name) + if err != nil { + return err + } + p := resolved.Profile + if p.Kind() != query.KindTrace { + return fmt.Errorf("profile %q is not a trace; use `query top` to sample it", p.Name) + } + if p, err = applySessionSpecOverrides(p, sessionSpec{Duration: options.Duration}); err != nil { + return err + } + params, err := parseSessionParams(options.Params) + if err != nil { + return err + } + + session, err := r.startCLISession(store, p, params) + if err != nil { + return err + } + replay, live, cancelSub := session.Subscribe() + defer cancelSub() + + sigCtx, cancelSig := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) + defer cancelSig() + go func() { + <-sigCtx.Done() + session.Stop("interrupted by signal") + }() + + for _, e := range replay { + if err := r.printTraceEvent(e, options.Output); err != nil { + session.Stop("output failed") + return err + } + } + for e := range live { + if err := r.printTraceEvent(e, options.Output); err != nil { + session.Stop("output failed") + return err + } + } + return r.finishCLISession(session) +} + +// startCLISession runs the profile, resolved from store, through a single-slot +// in-memory registry; CLI sessions are not persisted. +func (r *Runner) startCLISession(store profilestore.Store, p query.Profile, params map[string]any) (*query.Session, error) { + registry := query.NewSessionRegistry(query.RegistryOptions{ + MaxSessions: 1, + MaxDuration: sessionSpecDuration(p), + BeforeRead: func(ctx context.Context, p query.Profile, params map[string]any) (func(), error) { + return profilestore.PrepareReads(ctx, r.beforeExecute, []profilestore.ReadRequest{{Profile: p, Params: params}}) + }, + }) + return query.ExecuteStream(withSessionProfiles(r.context(), store, registry), registry, p, params) +} + +func sessionSpecDuration(p query.Profile) time.Duration { + if p.Trace != nil { + return p.Trace.DurationLimit() + } + return p.Top.DurationLimit() +} + +// finishCLISession prints the session summary and surfaces a failure as a +// non-zero exit. +func (r *Runner) finishCLISession(session *query.Session) error { + snap := session.Snapshot() + elapsed := time.Since(snap.StartedAt).Round(time.Millisecond) + if snap.StoppedAt != nil { + elapsed = snap.StoppedAt.Sub(snap.StartedAt).Round(time.Millisecond) + } + if _, err := fmt.Fprintf(r.stderr, "session %s: %s, %d events in %s\n", snap.ID, snap.State, snap.EventCount, elapsed); err != nil { + return fmt.Errorf("write session %s summary: %w", snap.ID, err) + } + if snap.State == query.SessionFailed { + return errors.New(snap.Error) + } + return nil +} + +// parseSessionParams reads the repeatable --param flag. trace and top are +// plain cobra commands, so the value always came from the operator's own shell +// — which is what makes expanding an @file reference safe here. +func parseSessionParams(pairs []string) (map[string]any, error) { + return paramfile.Parse(pairs) +} + +// printTraceEvent writes e to stdout; an event that cannot be encoded or +// written ends the trace rather than leaving a gap in its output. +func (r *Runner) printTraceEvent(e query.Event, format string) error { + line := formatTraceEventLine(e) + if format == "json" { + data, err := json.Marshal(e) + if err != nil { + return fmt.Errorf("encode event %d: %w", e.Sequence, err) + } + line = string(data) + } + if _, err := fmt.Fprintln(r.stdout, line); err != nil { + return fmt.Errorf("write event %d: %w", e.Sequence, err) + } + return nil +} + +// formatTraceEventLine renders an event as one logfmt line: timestamp then +// sorted key=value pairs, quoting values with spaces. +func formatTraceEventLine(e query.Event) string { + var b strings.Builder + b.WriteString(e.Time.UTC().Format(time.RFC3339)) + if e.Error != "" { + fmt.Fprintf(&b, " error=%s", logfmtValue(e.Error)) + return b.String() + } + keys := make([]string, 0, len(e.Row)) + for k := range e.Row { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + fmt.Fprintf(&b, " %s=%s", k, logfmtValue(fmt.Sprint(e.Row[k]))) + } + return b.String() +} + +func logfmtValue(v string) string { + if strings.ContainsAny(v, " \t\"=") { + return strconv.Quote(v) + } + return v +} diff --git a/pkg/sessions/trace_export.go b/pkg/sessions/trace_export.go new file mode 100644 index 0000000..1240ed2 --- /dev/null +++ b/pkg/sessions/trace_export.go @@ -0,0 +1,141 @@ +// The trace export route: a window of a capture's records downloaded in one of +// its kind's formats. + +package sessions + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "strconv" + + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/telltale/pkg/tracing" +) + +// exportTrace downloads the window ?stream, ?afterSeq, ?toSeq of kind in +// ?format, authorized as reading its profile traces/. A refusal found +// before the first byte is answered with its status; a failure once the +// download has begun aborts it, so a client never keeps a file cut short. +func (h *sessionHandler) exportTrace(w http.ResponseWriter, r *http.Request, kind string) { + query := r.URL.Query() + format, err := h.traces.Kinds.Format(kind, query.Get("format")) + if err != nil { + http.Error(w, err.Error(), traceExportErrorStatus(err)) + return + } + if !h.authorized(w, r, tracing.ProfilePrefix+kind, ActionRead) { + return + } + ref, err := traceRef(kind, query) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + download := &attachment{w: w, format: format, filename: fmt.Sprintf("%s-%s.%s", kind, ref.Stream, format.Ext)} + err = h.traces.Export(h.ctx.Wrap(r.Context()), ref, format.Name, download) + switch { + case err != nil && download.started: + panic(http.ErrAbortHandler) + case err != nil: + http.Error(w, err.Error(), traceExportErrorStatus(err)) + default: + // An empty export is still answered as a download; an empty write cannot fail. + _, _ = download.Write(nil) + } +} + +// traceSeq answers {"seq": n}, the seq ?stream of kind has committed through +// (0 for one that never committed a row), authorized as reading traces/. +func (h *sessionHandler) traceSeq(w http.ResponseWriter, r *http.Request, kind string) { + if _, ok := h.traces.Kinds.Get(kind); !ok { + http.Error(w, fmt.Sprintf("%v %q", tracing.ErrUnknownKind, kind), http.StatusNotFound) + return + } + if !h.authorized(w, r, tracing.ProfilePrefix+kind, ActionRead) { + return + } + stream := r.URL.Query().Get("stream") + if stream == "" { + http.Error(w, "stream is required", http.StatusBadRequest) + return + } + var seq int64 + meta, err := h.traces.Results.Backend.Meta(r.Context(), stream) + switch { + case errors.Is(err, recordstore.ErrNotFound): + case err != nil: + http.Error(w, err.Error(), http.StatusInternalServerError) + return + default: + seq = meta.HighSeq + } + writeSessionJSON(w, http.StatusOK, map[string]int64{"seq": seq}) +} + +// traceRef is the window query names of kind's stream. +func traceRef(kind string, query url.Values) (tracing.TraceRef, error) { + ref := tracing.TraceRef{Kind: kind, Stream: query.Get("stream")} + if ref.Stream == "" { + return ref, errors.New("stream is required") + } + for name, seq := range map[string]*int64{"afterSeq": &ref.AfterSeq, "toSeq": &ref.ToSeq} { + if raw := query.Get(name); raw != "" { + parsed, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + return ref, fmt.Errorf("%s %q is not a seq", name, raw) + } + *seq = parsed + } + } + return ref, ref.Validate() +} + +func traceExportErrorStatus(err error) int { + switch { + case errors.Is(err, tracing.ErrUnknownKind), errors.Is(err, recordstore.ErrNotFound): + return http.StatusNotFound + case errors.Is(err, tracing.ErrUnknownFormat), errors.Is(err, tracing.ErrInvalidRef): + return http.StatusBadRequest + } + return http.StatusInternalServerError +} + +// attachment writes a download's headers with its first byte, so an export +// refused before it writes anything is still answered with its error. +type attachment struct { + w http.ResponseWriter + format tracing.ExportFormat + filename string + started bool +} + +// Write sets the download headers on the same writer it writes the body to, so +// the non-HTML content type is visibly bound to the untrusted bytes it carries. +func (a *attachment) Write(p []byte) (int, error) { + w := a.w + if !a.started { + a.started = true + header := w.Header() + // A plugin's format must not turn captured bytes into an active browser document. + switch a.format.MIME { + case "application/x-ndjson": + header.Set("Content-Type", "application/x-ndjson") + case "application/json": + header.Set("Content-Type", "application/json") + case "application/x-asciicast": + header.Set("Content-Type", "application/x-asciicast") + default: + header.Set("Content-Type", "application/octet-stream") + } + header.Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", a.filename)) + header.Set("X-Content-Type-Options", "nosniff") + header.Set("Content-Security-Policy", "sandbox") + w.WriteHeader(http.StatusOK) + } + if len(p) == 0 { + return 0, nil + } + return w.Write(p) +} diff --git a/pkg/sessions/trace_export_test.go b/pkg/sessions/trace_export_test.go new file mode 100644 index 0000000..08b78f0 --- /dev/null +++ b/pkg/sessions/trace_export_test.go @@ -0,0 +1,35 @@ +package sessions + +import ( + "net/http" + "net/http/httptest" + + "github.com/flanksource/telltale/pkg/tracing" + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = ginkgo.Describe("Trace attachment", func() { + ginkgo.DescribeTable("keeps untrusted export bytes in a non-executable download", func(format tracing.ExportFormat, contentType string) { + payload := []byte(``) + response := httptest.NewRecorder() + download := &attachment{w: response, format: format, filename: "trace." + format.Ext} + + written, err := download.Write(payload) + Expect(err).ToNot(HaveOccurred()) + Expect(written).To(Equal(len(payload))) + Expect(response.Result().StatusCode).To(Equal(http.StatusOK)) + Expect(response.Result().Header).To(SatisfyAll( + HaveKeyWithValue("Content-Type", []string{contentType}), + HaveKeyWithValue("Content-Disposition", []string{`attachment; filename="trace.` + format.Ext + `"`}), + HaveKeyWithValue("X-Content-Type-Options", []string{"nosniff"}), + HaveKeyWithValue("Content-Security-Policy", []string{"sandbox"}), + )) + Expect(response.Body.Bytes()).To(Equal(payload)) + }, + ginkgo.Entry("NDJSON", tracing.NDJSON, "application/x-ndjson"), + ginkgo.Entry("HAR JSON", tracing.ExportFormat{Name: "har", Ext: "har", MIME: "application/json"}, "application/json"), + ginkgo.Entry("terminal cast", tracing.ExportFormat{Name: "cast", Ext: "cast", MIME: "application/x-asciicast"}, "application/x-asciicast"), + ginkgo.Entry("HTML", tracing.ExportFormat{Name: "html", Ext: "html", MIME: "text/html"}, "application/octet-stream"), + ) +}) diff --git a/pkg/sessions/trace_test.go b/pkg/sessions/trace_test.go new file mode 100644 index 0000000..567aa41 --- /dev/null +++ b/pkg/sessions/trace_test.go @@ -0,0 +1,69 @@ +package sessions + +import ( + "testing" + "time" + + "github.com/flanksource/commons-db/query" + "github.com/stretchr/testify/require" +) + +func TestApplySessionSpecOverrides(t *testing.T) { + t.Run("synthesizes top for a plain profile", func(t *testing.T) { + p, err := applySessionSpecOverrides(execProfile("plain"), sessionSpec{Interval: "2s"}) + require.NoError(t, err) + require.Equal(t, query.KindTop, p.Kind()) + require.Equal(t, 2*time.Second, p.Top.Interval.Duration) + }) + + t.Run("rejects a plain profile without an interval", func(t *testing.T) { + _, err := applySessionSpecOverrides(execProfile("plain"), sessionSpec{}) + require.ErrorContains(t, err, "neither trace nor top") + }) + + t.Run("overrides a declared top interval and duration", func(t *testing.T) { + in := execProfile("declared") + in.Top = &query.TopSpec{} + p, err := applySessionSpecOverrides(in, sessionSpec{Interval: "10s", Duration: "1m"}) + require.NoError(t, err) + require.Equal(t, 10*time.Second, p.Top.Interval.Duration) + require.Equal(t, time.Minute, p.Top.MaxDuration.Duration) + }) + + t.Run("rejects an interval on a trace profile", func(t *testing.T) { + _, err := applySessionSpecOverrides(traceTestProfile("t", "any"), sessionSpec{Interval: "2s"}) + require.ErrorContains(t, err, "interval does not apply") + }) + + t.Run("overrides trace duration", func(t *testing.T) { + p, err := applySessionSpecOverrides(traceTestProfile("t", "any"), sessionSpec{Duration: "30s"}) + require.NoError(t, err) + require.Equal(t, 30*time.Second, p.Trace.MaxDuration.Duration) + }) + + t.Run("rejects malformed durations", func(t *testing.T) { + _, err := applySessionSpecOverrides(traceTestProfile("t", "any"), sessionSpec{Duration: "soon"}) + require.ErrorContains(t, err, "invalid duration") + _, err = applySessionSpecOverrides(execProfile("p"), sessionSpec{Interval: "fast"}) + require.ErrorContains(t, err, "invalid interval") + }) +} + +func TestParseSessionParams(t *testing.T) { + params, err := parseSessionParams([]string{"region=EU", "pod=api-0"}) + require.NoError(t, err) + require.Equal(t, map[string]any{"region": "EU", "pod": "api-0"}, params) + + _, err = parseSessionParams([]string{"no-equals"}) + require.ErrorContains(t, err, "expected key=value") +} + +func TestFormatTraceEventLine(t *testing.T) { + ts := time.Date(2026, 7, 13, 10, 0, 0, 0, time.UTC) + e := query.Event{Sequence: 3, Time: ts, Row: query.Row{"msg": "hello world", "level": "info"}} + line := formatTraceEventLine(e) + require.Equal(t, `2026-07-13T10:00:00Z level=info msg="hello world"`, line) + + errLine := formatTraceEventLine(query.Event{Time: ts, Error: "backend gone"}) + require.Equal(t, `2026-07-13T10:00:00Z error="backend gone"`, errLine) +} diff --git a/pkg/sessions/traces.go b/pkg/sessions/traces.go new file mode 100644 index 0000000..3258db0 --- /dev/null +++ b/pkg/sessions/traces.go @@ -0,0 +1,78 @@ +// The trace plugin routes: list the trace kinds a server serves, and start a +// capture of one as a session the rest of the session API controls. + +package sessions + +import ( + "encoding/json" + "errors" + "net/http" + "time" + + "github.com/flanksource/telltale/pkg/tracing" +) + +// traceStartInput is the body of a trace start: the kind's params, how long +// the capture runs (the registry's longest duration when empty), and labels a +// host lists its sessions by. +type traceStartInput struct { + Params json.RawMessage `json:"params,omitempty"` + Duration string `json:"duration,omitempty"` + Labels map[string]string `json:"labels,omitempty"` +} + +// traceKinds answers the kinds the caller may read, each with its params form +// and its columns. +func (h *sessionHandler) traceKinds(w http.ResponseWriter, r *http.Request) { + kinds, err := h.traces.Kinds.List() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + allowed := make([]tracing.KindInfo, 0, len(kinds)) + for _, kind := range kinds { + if h.refusal(r, tracing.ProfilePrefix+kind.Name, ActionRead) == nil { + allowed = append(allowed, kind) + } + } + writeSessionJSON(w, http.StatusOK, allowed) +} + +// startTrace starts a capture of kind, authorized as its profile traces/. +func (h *sessionHandler) startTrace(w http.ResponseWriter, r *http.Request, kind string) { + if !h.authorized(w, r, tracing.ProfilePrefix+kind, ActionControl) { + return + } + var input traceStartInput + decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&input); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + request := tracing.StartRequest{Kind: kind, Params: input.Params, Labels: input.Labels} + if input.Duration != "" { + duration, err := time.ParseDuration(input.Duration) + if err != nil || duration <= 0 { + http.Error(w, "duration must be positive (for example 5m)", http.StatusBadRequest) + return + } + stopAt := time.Now().Add(duration) + request.StopAt = &stopAt + } + if h.principal != nil { + request.Principal = h.principal(r) + } + managed, err := h.traces.Start(h.sessionContext(r), request) + switch { + case errors.Is(err, tracing.ErrUnknownKind): + http.Error(w, err.Error(), http.StatusNotFound) + return + case err != nil && h.traceStartError != nil && h.traceStartError(w, r, err): + return + case err != nil: + http.Error(w, err.Error(), sessionStartErrorStatus(err)) + return + } + h.writeLive(w, r, http.StatusCreated, managed.Session()) +} diff --git a/pkg/sessions/traces_openapi.go b/pkg/sessions/traces_openapi.go new file mode 100644 index 0000000..d1942e1 --- /dev/null +++ b/pkg/sessions/traces_openapi.go @@ -0,0 +1,109 @@ +// The trace plugin routes in the OpenAPI document: the kinds list, and one +// start operation per kind whose request body is that kind's params form. + +package sessions + +import ( + "github.com/flanksource/clicky/rpc" + "github.com/flanksource/commons/logger" + "github.com/flanksource/telltale/pkg/tracing" + + "github.com/flanksource/commons-db/cmd/query/profiles" +) + +const tracesPath = "/api/v1/traces" + +// AddTracesOpenAPI describes the trace routes of the runtime current returns, +// as it is when the document is built; a server serving no trace kinds yet +// (current returns nil) describes none. +func AddTracesOpenAPI(current func() *tracing.Runtime) profiles.OpenAPIExtension { + return func(spec *rpc.OpenAPISpec) { + runtime := current() + if runtime == nil { + return + } + kinds, err := runtime.Kinds.List() + if err != nil { + logger.Warnf("trace kinds openapi: %v", err) + return + } + if spec.Paths == nil { + spec.Paths = map[string]rpc.OpenAPIPath{} + } + spec.Paths[tracesPath+"/kinds"] = rpc.OpenAPIPath{"get": { + OperationID: "list-trace-kinds", Summary: "List trace kinds", + Description: "The trace plugin kinds this server serves: each kind's params form, columns and capabilities", + Responses: map[string]rpc.OpenAPIResponse{"200": {Description: "KindInfo[]"}}, + }} + for _, kind := range kinds { + spec.Paths[tracesPath+"/"+kind.Name+"/sessions"] = rpc.OpenAPIPath{"post": traceStartOperation(kind)} + spec.Paths[tracesPath+"/"+kind.Name+"/export"] = rpc.OpenAPIPath{"get": traceExportOperation(kind)} + spec.Paths[tracesPath+"/"+kind.Name+"/seq"] = rpc.OpenAPIPath{"get": traceSeqOperation(kind)} + } + } +} + +func traceSeqOperation(kind tracing.KindInfo) rpc.OpenAPIOperation { + return rpc.OpenAPIOperation{ + OperationID: kind.Name + "-trace-seq", + Summary: "Read how far a " + kind.Title + " trace committed", + Description: "The seq a session's events.stream committed through, as {\"seq\": n}; 0 for a stream that committed nothing", + Parameters: []rpc.OpenAPIParameter{ + {Name: "stream", In: "query", Required: true, Description: "The session's events.stream", Schema: &rpc.OpenAPISchema{Type: "string"}}, + }, + Responses: map[string]rpc.OpenAPIResponse{ + "200": {Description: "{\"seq\": n}"}, + "400": {Description: "No stream"}, + "404": {Description: "No such trace kind"}, + }, + } +} + +func traceExportOperation(kind tracing.KindInfo) rpc.OpenAPIOperation { + formats := make([]any, 0, len(kind.ExportFormats)) + for _, format := range kind.ExportFormats { + formats = append(formats, format.Name) + } + seq := func(name, description string) rpc.OpenAPIParameter { + return rpc.OpenAPIParameter{Name: name, In: "query", Description: description, Schema: &rpc.OpenAPISchema{Type: "integer"}} + } + return rpc.OpenAPIOperation{ + OperationID: "export-" + kind.Name + "-trace", + Summary: "Export a " + kind.Title + " trace", + Description: "Download the records of a session's events.stream after afterSeq through toSeq as an attachment", + Parameters: []rpc.OpenAPIParameter{ + {Name: "stream", In: "query", Required: true, Description: "The session's events.stream", Schema: &rpc.OpenAPISchema{Type: "string"}}, + {Name: "format", In: "query", Required: true, Description: "The format to download in", Schema: &rpc.OpenAPISchema{Type: "string", Enum: formats}}, + seq("afterSeq", "Export the records after this seq; 0 starts at the first"), + seq("toSeq", "Export the records through this seq; 0 runs to the last"), + }, + Responses: map[string]rpc.OpenAPIResponse{ + "200": {Description: "The window in the format asked"}, + "400": {Description: "No stream, a format the kind lacks, or a window that is not one"}, + "404": {Description: "No such trace kind or stream"}, + }, + } +} + +func traceStartOperation(kind tracing.KindInfo) rpc.OpenAPIOperation { + return rpc.OpenAPIOperation{ + OperationID: "start-" + kind.Name + "-trace", + Summary: "Start a " + kind.Title + " trace", + Description: "Start a capture as a session; stop it via POST /api/v1/sessions/{id}/stop. Its records are read " + + "through the profile " + tracing.ProfilePrefix + kind.Name + " with stream set to the session's events.stream.", + RequestBody: &rpc.OpenAPIRequestBody{Required: true, Content: map[string]rpc.OpenAPIMediaType{ + "application/json": {Schema: &rpc.OpenAPISchema{Type: "object", Properties: map[string]*rpc.OpenAPISchema{ + "params": kind.Params, + "duration": {Type: "string", Description: "Go duration the capture runs for, e.g. 5m; empty runs for the server's longest"}, + "labels": {Type: "object", Description: "Labels the session is listed by, e.g. label.target=cycle", + AdditionalProperties: &rpc.OpenAPISchema{Type: "string"}}, + }}}, + }}, + Responses: map[string]rpc.OpenAPIResponse{ + "201": {Description: "The started session (SessionInfo)"}, + "400": {Description: "Params the kind refuses"}, + "404": {Description: "No such trace kind"}, + "409": {Description: "Too many sessions"}, + }, + } +} diff --git a/pkg/sessions/traces_openapi_test.go b/pkg/sessions/traces_openapi_test.go new file mode 100644 index 0000000..ff2eca1 --- /dev/null +++ b/pkg/sessions/traces_openapi_test.go @@ -0,0 +1,57 @@ +// Specs for the trace plugin OpenAPI: the kinds list, and one start operation +// per kind whose request body is the kind's own params form. + +package sessions + +import ( + "time" + + "github.com/flanksource/clicky/rpc" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/target" + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/commons-db/recordstore/probe" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/commons-db/recordstore/recordresults/recordresultstest" +) + +var _ = ginkgo.Describe("Trace plugin OpenAPI", func() { + ginkgo.It("describes the kinds list and a start operation per kind taking its params form", func() { + kinds := tracing.NewKinds() + Expect(kinds.RegisterKind("pings", tracing.NewHandler[pingParams, ping](pings{}, tracing.Capabilities{Live: true}, target.Host))).To(Succeed()) + results := recordresultstest.OpenResults(recordresults.OpenOptions{ + Prefix: "traces", ConnectionName: "traces", + Settings: recordresultstest.LocalSettings(recordstore.BackendSQLite), Register: kinds.RegisterResultTypes, + }) + runtime := &tracing.Runtime{Kinds: kinds, Results: results, Probes: probe.NewManager(probe.ManagerOptions{}), PollEvery: time.Second} + + spec := &rpc.OpenAPISpec{} + AddTracesOpenAPI(func() *tracing.Runtime { return runtime })(spec) + + Expect(spec.Paths).To(HaveKey("/api/v1/traces/kinds")) + start := spec.Paths["/api/v1/traces/pings/sessions"]["post"] + Expect(start.OperationID).To(Equal("start-pings-trace")) + Expect(start.Responses).To(HaveKey("201")) + body := start.RequestBody.Content["application/json"].Schema + Expect(body.Properties["params"].Properties).To(HaveKey("count")) + Expect(body.Properties["params"].Properties["count"].Default).To(BeEquivalentTo(1)) + Expect(body.Properties["duration"].Type).To(Equal("string")) + + export := spec.Paths["/api/v1/traces/pings/export"]["get"] + Expect(export.OperationID).To(Equal("export-pings-trace")) + Expect(export.Parameters).To(ContainElement(HaveField("Schema.Enum", []any{"ndjson"}))) + + seq := spec.Paths["/api/v1/traces/pings/seq"]["get"] + Expect(seq.OperationID).To(Equal("pings-trace-seq")) + Expect(seq.Parameters).To(ConsistOf(And(HaveField("Name", "stream"), HaveField("Required", true)))) + }) + + ginkgo.It("describes nothing before the server serves trace kinds", func() { + spec := &rpc.OpenAPISpec{} + AddTracesOpenAPI(func() *tracing.Runtime { return nil })(spec) + Expect(spec.Paths).To(BeEmpty()) + }) +}) diff --git a/pkg/sessions/traces_test.go b/pkg/sessions/traces_test.go new file mode 100644 index 0000000..7964c72 --- /dev/null +++ b/pkg/sessions/traces_test.go @@ -0,0 +1,353 @@ +// Specs for the trace plugin routes: listing the kinds a server serves, and +// starting a capture of one as a session the session API then controls. + +package sessions + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "time" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + profilepkg "github.com/flanksource/commons-db/cmd/query/profiles" + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/commons-db/recordstore/probe" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/commons-db/recordstore/recordresults/recordresultstest" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/target" +) + +type pingParams struct { + Count int `json:"count"` +} + +func (p pingParams) Validate() error { + if p.Count < 0 { + return fmt.Errorf("count must not be negative") + } + return nil +} + +type ping struct { + At time.Time `json:"at"` + N int `json:"n"` +} + +// pings emits Count pings, then runs until it is stopped. +type pings struct{} + +func (pings) Params() pingParams { return pingParams{Count: 1} } + +func (pings) Schema() recordresults.ResultType[ping] { + return recordresults.ResultType[ping]{Title: "Pings", TimeColumn: "at"} +} + +func (pings) Handle(ctx dbcontext.Context, p pingParams, records tracing.Emitter[ping], _ tracing.Records[ping]) error { + for n := 1; n <= p.Count; n++ { + if err := records.Emit(ctx, ping{At: time.Now().UTC(), N: n}); err != nil { + return err + } + } + <-ctx.Done() + return nil +} + +// lockedOut refuses every start in Prepare, with an error a host recognises. +type lockedOut struct{} + +type loginDenied struct{ login string } + +func (e loginDenied) Error() string { return "login " + e.login + " lacks ALTER ANY EVENT SESSION" } + +func (lockedOut) Params() pingParams { return pingParams{} } + +func (lockedOut) Schema() recordresults.ResultType[ping] { + return recordresults.ResultType[ping]{Title: "Locked out", TimeColumn: "at"} +} + +func (lockedOut) Prepare(ctx dbcontext.Context, _ pingParams, _ tracing.Emitter[ping]) (dbcontext.Context, func() error, error) { + return ctx, nil, fmt.Errorf("prepare: %w", loginDenied{login: "oipa"}) +} + +func (lockedOut) Handle(dbcontext.Context, pingParams, tracing.Emitter[ping], tracing.Records[ping]) error { + return nil +} + +type traceHarness struct { + handler *sessionHandler + registry *query.SessionRegistry + results *recordresults.Results + + mu sync.Mutex + refused map[string]Action +} + +// newTraceHarness serves the pings kind, and whatever kinds extra registers. +func newTraceHarness(extra ...func(*tracing.Kinds)) *traceHarness { + h := &traceHarness{refused: map[string]Action{}} + kinds := tracing.NewKinds() + Expect(kinds.RegisterKind("pings", tracing.NewHandler[pingParams, ping](pings{}, tracing.Capabilities{Live: true}, target.Host))).To(Succeed()) + for _, register := range extra { + register(kinds) + } + h.results = recordresultstest.OpenResults(recordresults.OpenOptions{ + Prefix: "traces", ConnectionName: "traces", + Settings: recordresultstest.LocalSettings(recordstore.BackendSQLite), Register: kinds.RegisterResultTypes, + }) + runtime := &tracing.Runtime{ + Kinds: kinds, Results: h.results, Probes: probe.NewManager(probe.ManagerOptions{}), PollEvery: 20 * time.Millisecond, + } + h.registry = query.NewSessionRegistry(query.RegistryOptions{}) + runtime.Sessions = h.registry + ginkgo.DeferCleanup(func() { Expect(h.registry.StopAll(context.Background())).To(Succeed()) }) + profiles, err := profilepkg.NewFileStore(ginkgo.GinkgoT().TempDir()) + Expect(err).ToNot(HaveOccurred()) + h.handler = newSessionHandler(sessionHandlerOptions{ + Prefix: "/api/v1", Ctx: dbcontext.New(), Store: profiles, Registry: h.registry, + Records: h.results.Backend, Traces: runtime, Authorize: h.authorize, Next: &nextMarker{}, + Principal: func(*http.Request) string { return "tester" }, + TraceStartError: func(w http.ResponseWriter, _ *http.Request, err error) bool { + var denied loginDenied + if !errors.As(err, &denied) { + return false + } + writeSessionJSON(w, http.StatusForbidden, map[string]string{"code": "login_denied", "login": denied.login}) + return true + }, + }) + return h +} + +func (h *traceHarness) authorize(_ *http.Request, profile string, action Action) error { + h.mu.Lock() + defer h.mu.Unlock() + if refused, ok := h.refused[profile]; ok && (refused == ActionRead || action == refused) { + return fmt.Errorf("profile %q is not %s-able by this caller", profile, action) + } + return nil +} + +func (h *traceHarness) do(method, path, body string) *httptest.ResponseRecorder { + request := httptest.NewRequest(method, path, bytes.NewBufferString(body)) + response := httptest.NewRecorder() + h.handler.ServeHTTP(response, request) + return response +} + +var _ = ginkgo.Describe("Trace plugin routes", func() { + ginkgo.It("lists the kinds it serves with their params form and columns", func() { + h := newTraceHarness() + response := h.do(http.MethodGet, "/api/v1/traces/kinds", "") + Expect(response.Code).To(Equal(http.StatusOK)) + var kinds []tracing.KindInfo + Expect(json.Unmarshal(response.Body.Bytes(), &kinds)).To(Succeed()) + Expect(kinds).To(HaveLen(1)) + Expect(kinds[0].Name).To(Equal("pings")) + Expect(kinds[0].Params.Properties).To(HaveKey("count")) + Expect(kinds[0].Columns).ToNot(BeEmpty()) + Expect(kinds[0].ExportFormats).To(Equal([]tracing.ExportFormat{tracing.NDJSON})) + }) + + ginkgo.It("answers the seq a stream committed through, 0 for one never written, and refuses what it cannot read", func() { + h := newTraceHarness() + const stream = "pings-1" + _, err := recordstore.AppendTyped(context.Background(), h.results.Backend, stream, "pings", []ping{{N: 1}, {N: 2}, {N: 3}}) + Expect(err).ToNot(HaveOccurred()) + + for path, want := range map[string]string{"stream=" + stream: `{"seq": 3}`, "stream=never-written": `{"seq": 0}`} { + response := h.do(http.MethodGet, "/api/v1/traces/pings/seq?"+path, "") + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + Expect(response.Body.String()).To(MatchJSON(want), path) + } + Expect(h.do(http.MethodGet, "/api/v1/traces/nope/seq?stream="+stream, "").Code).To(Equal(http.StatusNotFound)) + Expect(h.do(http.MethodGet, "/api/v1/traces/pings/seq", "").Code).To(Equal(http.StatusBadRequest)) + h.refused["traces/pings"] = ActionRead + Expect(h.do(http.MethodGet, "/api/v1/traces/pings/seq?stream="+stream, "").Code).To(Equal(http.StatusForbidden)) + }) + + ginkgo.It("leaves out the kinds a caller may not read", func() { + h := newTraceHarness() + h.refused["traces/pings"] = ActionRead + response := h.do(http.MethodGet, "/api/v1/traces/kinds", "") + Expect(response.Code).To(Equal(http.StatusOK)) + Expect(strings.TrimSpace(response.Body.String())).To(Equal("[]")) + }) + + ginkgo.It("starts a capture as a session the session API stops, its records committed and sealed", func() { + h := newTraceHarness() + response := h.do(http.MethodPost, "/api/v1/traces/pings/sessions", `{"params": {"count": 2}, "duration": "5m"}`) + Expect(response.Code).To(Equal(http.StatusCreated), response.Body.String()) + var info query.SessionInfo + Expect(json.Unmarshal(response.Body.Bytes(), &info)).To(Succeed()) + Expect(info.Profile).To(Equal("traces/pings")) + Expect(info.Principal).To(Equal("tester")) + Expect(info.StopAt).ToNot(BeNil()) + Expect(*info.StopAt).To(BeTemporally("~", time.Now().Add(5*time.Minute), time.Minute)) + + session, ok := h.registry.Get(info.ID) + Expect(ok).To(BeTrue()) + Eventually(func() int64 { return session.Snapshot().EventCount }).WithTimeout(5 * time.Second).Should(Equal(int64(2))) + + Expect(h.do(http.MethodPost, "/api/v1/sessions/"+info.ID+"/stop", "").Code).To(Equal(http.StatusOK)) + Eventually(session.Done()).WithTimeout(10 * time.Second).Should(BeClosed()) + ended := session.Snapshot() + Expect(ended.State).To(Equal(query.SessionStopped)) + meta, err := h.results.Backend.Meta(context.Background(), ended.Events.Stream) + Expect(err).ToNot(HaveOccurred()) + Expect(meta.Sealed).To(BeTrue()) + Expect(meta.Total).To(Equal(int64(2))) + + events := h.do(http.MethodGet, "/api/v1/sessions/"+info.ID+"/events?format=ndjson", "") + Expect(events.Code).To(Equal(http.StatusOK)) + Expect(strings.Count(strings.TrimSpace(events.Body.String()), "\n")).To(Equal(1)) + }) + + ginkgo.It("labels a capture as its start asks, so a host lists its sessions by label", func() { + h := newTraceHarness() + response := h.do(http.MethodPost, "/api/v1/traces/pings/sessions", `{"labels": {"target": "cycle", "label": "nightly"}}`) + Expect(response.Code).To(Equal(http.StatusCreated), response.Body.String()) + var info query.SessionInfo + Expect(json.Unmarshal(response.Body.Bytes(), &info)).To(Succeed()) + Expect(info.Labels).To(HaveKeyWithValue("target", "cycle")) + Expect(info.Labels).To(HaveKeyWithValue("label", "nightly")) + }) + + ginkgo.DescribeTable("refuses a start it cannot serve", + func(path, body string, status int, message string) { + h := newTraceHarness() + response := h.do(http.MethodPost, path, body) + Expect(response.Code).To(Equal(status)) + Expect(response.Body.String()).To(ContainSubstring(message)) + Expect(h.registry.List()).To(BeEmpty()) + }, + ginkgo.Entry("an unknown kind", "/api/v1/traces/nope/sessions", `{}`, http.StatusNotFound, "unknown trace kind"), + ginkgo.Entry("params the kind refuses", "/api/v1/traces/pings/sessions", `{"params": {"count": -1}}`, http.StatusBadRequest, "must not be negative"), + ginkgo.Entry("a params field the kind lacks", "/api/v1/traces/pings/sessions", `{"params": {"cnt": 1}}`, http.StatusBadRequest, "cnt"), + ginkgo.Entry("a body field the route lacks", "/api/v1/traces/pings/sessions", `{"parms": {}}`, http.StatusBadRequest, "parms"), + ginkgo.Entry("a duration that is not one", "/api/v1/traces/pings/sessions", `{"duration": "soon"}`, http.StatusBadRequest, "duration"), + ) + + ginkgo.It("answers a refused start as the host describes the refusal", func() { + h := newTraceHarness(func(kinds *tracing.Kinds) { + Expect(kinds.RegisterKind("locked_out", tracing.NewHandler[pingParams, ping](lockedOut{}, tracing.Capabilities{Live: true}, target.Host))).To(Succeed()) + }) + response := h.do(http.MethodPost, "/api/v1/traces/locked_out/sessions", `{}`) + Expect(response.Code).To(Equal(http.StatusForbidden), response.Body.String()) + Expect(response.Body.String()).To(MatchJSON(`{"code": "login_denied", "login": "oipa"}`)) + }) + + ginkgo.It("answers a refused start the host does not describe with its error", func() { + h := newTraceHarness() + response := h.do(http.MethodPost, "/api/v1/traces/pings/sessions", `{"params": {"count": -1}}`) + Expect(response.Code).To(Equal(http.StatusBadRequest)) + Expect(response.Body.String()).To(ContainSubstring("must not be negative")) + }) + + ginkgo.Context("exporting a window of a capture", func() { + var h *traceHarness + var stream string + + ginkgo.BeforeEach(func() { + h = newTraceHarness() + response := h.do(http.MethodPost, "/api/v1/traces/pings/sessions", `{"params": {"count": 2}}`) + Expect(response.Code).To(Equal(http.StatusCreated), response.Body.String()) + var info query.SessionInfo + Expect(json.Unmarshal(response.Body.Bytes(), &info)).To(Succeed()) + session, _ := h.registry.Get(info.ID) + Eventually(func() int64 { return session.Snapshot().EventCount }).WithTimeout(5 * time.Second).Should(Equal(int64(2))) + stream = info.Events.Stream + }) + + ginkgo.It("downloads the window as an attachment named for its kind and stream", func() { + response := h.do(http.MethodGet, "/api/v1/traces/pings/export?format=ndjson&afterSeq=1&stream="+stream, "") + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + Expect(response.Header().Get("Content-Type")).To(Equal("application/x-ndjson")) + Expect(response.Header().Get("X-Content-Type-Options")).To(Equal("nosniff")) + Expect(response.Header().Get("Content-Security-Policy")).To(Equal("sandbox")) + Expect(response.Header().Get("Content-Disposition")).To(Equal(`attachment; filename="pings-` + stream + `.ndjson"`)) + var row map[string]any + Expect(json.Unmarshal(response.Body.Bytes(), &row)).To(Succeed(), "one row: the window after seq 1") + Expect(row).To(HaveKeyWithValue("n", float64(2))) + }) + + ginkgo.DescribeTable("refuses an export it cannot serve", + func(path func(stream string) string, refuse string, status int) { + if refuse != "" { + h.refused[refuse] = ActionRead + } + Expect(h.do(http.MethodGet, path(stream), "").Code).To(Equal(status)) + }, + ginkgo.Entry("an unknown kind", func(s string) string { return "/api/v1/traces/nope/export?format=ndjson&stream=" + s }, "", http.StatusNotFound), + ginkgo.Entry("no stream", func(string) string { return "/api/v1/traces/pings/export?format=ndjson" }, "", http.StatusBadRequest), + ginkgo.Entry("a stream that does not exist", func(string) string { return "/api/v1/traces/pings/export?format=ndjson&stream=nope" }, "", http.StatusNotFound), + ginkgo.Entry("a format the kind lacks", func(s string) string { return "/api/v1/traces/pings/export?format=har&stream=" + s }, "", http.StatusBadRequest), + ginkgo.Entry("no format", func(s string) string { return "/api/v1/traces/pings/export?stream=" + s }, "", http.StatusBadRequest), + ginkgo.Entry("a seq that is not one", func(s string) string { return "/api/v1/traces/pings/export?format=ndjson&toSeq=x&stream=" + s }, "", http.StatusBadRequest), + ginkgo.Entry("a kind the caller may not read", func(s string) string { return "/api/v1/traces/pings/export?format=ndjson&stream=" + s }, "traces/pings", http.StatusForbidden), + ) + }) + + ginkgo.Context("checkpointing a running capture", func() { + var h *traceHarness + var info query.SessionInfo + + ginkgo.BeforeEach(func() { + h = newTraceHarness() + response := h.do(http.MethodPost, "/api/v1/traces/pings/sessions", `{"params": {"count": 2}}`) + Expect(response.Code).To(Equal(http.StatusCreated), response.Body.String()) + Expect(json.Unmarshal(response.Body.Bytes(), &info)).To(Succeed()) + }) + + ginkgo.It("commits what the capture emitted before it answers with the session", func() { + response := h.do(http.MethodPost, "/api/v1/sessions/"+info.ID+"/checkpoint", "") + Expect(response.Code).To(Equal(http.StatusOK), response.Body.String()) + var checkpointed query.SessionInfo + Expect(json.Unmarshal(response.Body.Bytes(), &checkpointed)).To(Succeed()) + Expect(checkpointed.ID).To(Equal(info.ID)) + Eventually(func() int64 { + Expect(h.do(http.MethodPost, "/api/v1/sessions/"+info.ID+"/checkpoint", "").Code).To(Equal(http.StatusOK)) + session, _ := h.registry.Get(info.ID) + return session.Snapshot().EventCount + }).WithTimeout(5 * time.Second).Should(Equal(int64(2))) + }) + + ginkgo.It("refuses checkpoint params the capture's kind does not read as a bad request", func() { + response := h.do(http.MethodPost, "/api/v1/sessions/"+info.ID+"/checkpoint", `{"key": "1@1", "reason": "manual"}`) + Expect(response.Code).To(Equal(http.StatusBadRequest), response.Body.String()) + Expect(response.Body.String()).To(ContainSubstring("reads no checkpoint params")) + }) + + ginkgo.It("refuses a checkpoint the caller may not take, though it may stop the capture", func() { + h.refused["traces/pings"] = ActionCheckpoint + Expect(h.do(http.MethodPost, "/api/v1/sessions/"+info.ID+"/checkpoint", "").Code).To(Equal(http.StatusForbidden)) + Expect(h.do(http.MethodPost, "/api/v1/sessions/"+info.ID+"/stop", "").Code).To(Equal(http.StatusOK)) + }) + + ginkgo.It("answers a checkpoint of an ended capture as a conflict", func() { + session, _ := h.registry.Get(info.ID) + Expect(h.do(http.MethodPost, "/api/v1/sessions/"+info.ID+"/stop", "").Code).To(Equal(http.StatusOK)) + Eventually(session.Done()).WithTimeout(10 * time.Second).Should(BeClosed()) + Expect(h.do(http.MethodPost, "/api/v1/sessions/"+info.ID+"/checkpoint", "").Code).To(Equal(http.StatusConflict)) + }) + }) + + ginkgo.It("refuses a start the caller may not control", func() { + h := newTraceHarness() + h.refused["traces/pings"] = ActionControl + response := h.do(http.MethodPost, "/api/v1/traces/pings/sessions", `{}`) + Expect(response.Code).To(Equal(http.StatusForbidden)) + Expect(h.registry.List()).To(BeEmpty()) + }) +}) diff --git a/pkg/traces/builtin.go b/pkg/traces/builtin.go index 09b924d..3ba6770 100644 --- a/pkg/traces/builtin.go +++ b/pkg/traces/builtin.go @@ -6,57 +6,40 @@ import ( "github.com/flanksource/telltale/pkg/tracing" "github.com/flanksource/telltale/pkg/tracing/goprof" - "github.com/flanksource/telltale/pkg/tracing/groups" "github.com/flanksource/telltale/pkg/tracing/httpproxy" "github.com/flanksource/telltale/pkg/tracing/pgproxy" - "github.com/flanksource/telltale/pkg/tracing/process" + "github.com/flanksource/telltale/pkg/tracing/processcapture" tracesql "github.com/flanksource/telltale/pkg/tracing/sql" "github.com/flanksource/telltale/pkg/tracing/stacks" "github.com/flanksource/telltale/pkg/tracing/target" "github.com/flanksource/telltale/pkg/tracing/terminal" ) +// The environment variables the `http` and `sql` proxies, and their +// http_proxy and pg_proxy presets, hand a command. +var ( + httpExports = []string{"HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy", "NO_PROXY", "no_proxy", "SSL_CERT_FILE", "CURL_CA_BUNDLE", "NODE_EXTRA_CA_CERTS", "REQUESTS_CA_BUNDLE", "GIT_SSL_CAINFO", "AWS_CA_BUNDLE"} + sqlExports = []string{"DATABASE_URL", "GAVEL_DB_DSN", "PGHOST", "PGPORT", "PGSSLMODE"} +) + // The builtin kinds, registered before any command or run reads the catalog. // The `http` and `sql` kinds default to telltale's own traffic: -// its HTTP feature and its own database. goprof reads gops agents, at the +// its HTTP feature and its own database; their http_proxy and pg_proxy +// presets arm them only on their proxy seam. goprof reads gops agents, at the // address their port files give. func init() { for _, registration := range []Registration{ { - Name: process.TreeKind, Title: "Process tree", Seams: []target.Seam{target.Process}, DefaultSeam: target.Process, - Summary: "Record every process of a tree as it starts, changes and exits", - Description: "Follows the process tree under its root: telltale's own tree, which holds the command, or the process --pid attaches to. " + - "Every run records this tree already; arm it again for another interval or to read each process's environment.", + Name: processcapture.Kind, Title: "Processes", Seams: []target.Seam{target.Process, target.Host}, DefaultSeam: target.Process, + Summary: "Record process identities, resource changes and numeric metrics", + Description: "Follows a process tree or the host. Stores startup metadata, environment and grouping tags once, files and receiving sockets as added/removed events, and numeric metrics in a separate wide table. TCP connections are unique peers per process, with inferred or unknown direction.", Examples: []Example{ {Args: "--environment-prefixes AWS_ -- make deploy", Says: "record the tree with each process's AWS_ variables"}, {Args: "--pid 1234 --duration 30s", Says: "follow process 1234's tree for 30 seconds"}, }, New: func(deps KindDeps) tracing.TracePlugin { - return process.NewTreeKind(deps.Sampler, process.TreeKindOptions{Dumpers: deps.Dumpers}) - }, - }, - { - Name: process.SamplesKind, Title: "Process samples", Seams: []target.Seam{target.Process}, DefaultSeam: target.Process, - Summary: "Sample the CPU, memory, disk IO and open descriptors of every process of a tree", - Description: "Samples each process under its root at every interval: telltale's own tree, which holds the command, or the process --pid attaches to. " + - "Facets add each process's open files, sockets, listening ports or connections, read with lsof.", - Examples: []Example{ - {Args: "--facets ports,connections -- ./server", Says: "sample the server and the ports and connections it opens"}, - {Args: "--seam process --pid 1234 --interval 250ms --duration 1m", Says: "sample process 1234's tree four times a second for a minute"}, - }, - New: func(deps KindDeps) tracing.TracePlugin { return process.NewSamplesKind(deps.Sampler) }, - }, - { - Name: groups.Kind, Title: "Process groups", Seams: []target.Seam{target.Host}, DefaultSeam: target.Host, - Summary: "Record the CPU, memory, disk, sockets and network of each tool's processes on the host", - Description: "Sorts every process on the host into the tool that owns it — claude, codex, cmux, gavel, go test, vitest, agent-browser, chrome and more — " + - "as a tree of groups, each with its working directory, checkout, agent session and cmux workspace, and records what each group and those under it use. " + - "Rules come from the built-in catalog and groups.yaml in ./.telltale, else the global config dir. Network bytes are read with nettop on darwin only when asked for, as it costs over a core.", - Examples: []Example{ - {Args: "--duration 10m", Says: "record every tool's groups for ten minutes"}, - {Args: "--groups claude,vitest --net", Says: "record claude sessions and vitest runs with their network bytes"}, + return processcapture.NewKind(deps.Sampler, processcapture.KindOptions{Dumpers: deps.Dumpers}) }, - New: func(deps KindDeps) tracing.TracePlugin { return groups.NewKind(deps.Sampler) }, }, { Name: stacks.Kind, Title: "Stacks", Seams: []target.Seam{target.Process, target.Agent, target.Self}, DefaultSeam: target.Process, @@ -97,7 +80,7 @@ func init() { Name: "http", Title: "HTTP", Seams: []target.Seam{target.Self, target.Agent, target.Proxy}, DefaultSeam: target.Self, Armable: true, Summary: "Record HTTP exchanges from an agent or a local proxy", Description: "Observes instrumented HTTP in telltale or an agent, or routes a command through a recording proxy.", - Exports: []string{"HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy", "NO_PROXY", "no_proxy", "SSL_CERT_FILE", "CURL_CA_BUNDLE", "NODE_EXTRA_CA_CERTS", "REQUESTS_CA_BUNDLE", "GIT_SSL_CAINFO", "AWS_CA_BUNDLE"}, + Exports: httpExports, Examples: []Example{{Args: "--features http -- true", Says: "record telltale HTTP"}, {Args: "--seam proxy --mode mitm -- curl -sS https://example.com", Says: "record a command through the proxy"}}, New: func(KindDeps) tracing.TracePlugin { return httpproxy.UnifiedKind(nil) }, }, @@ -105,10 +88,26 @@ func init() { Name: tracesql.RowKind, Title: "SQL", Seams: []target.Seam{target.Self, target.Agent, target.Proxy}, DefaultSeam: target.Self, Armable: true, Summary: "Record SQL statements from an agent or a local Postgres proxy", Description: "Observes instrumented connections in telltale or an agent, or routes a command through a Postgres proxy.", - Exports: []string{"DATABASE_URL", "GAVEL_DB_DSN", "PGHOST", "PGPORT", "PGSSLMODE"}, + Exports: sqlExports, Examples: []Example{{Args: "--connections self -- true", Says: "record telltale's database statements"}, {Args: "--seam proxy --dsn postgres://app@localhost:5432/orders -- ./migrate up", Says: "record a command through the Postgres proxy"}}, New: func(KindDeps) tracing.TracePlugin { return pgproxy.UnifiedKind(nil) }, }, + { + Name: httpproxy.ProxyKind, Base: httpproxy.Kind, Title: "HTTP proxy", Seams: []target.Seam{target.Proxy}, DefaultSeam: target.Proxy, Armable: true, + Summary: "Record a command's HTTP exchanges through a local recording proxy", + Description: "The http kind armed only on its proxy seam: routes a command through a recording proxy and records its exchanges as http does.", + Exports: httpExports, + Examples: []Example{{Args: "-- curl -sS http://example.com", Says: "record a command through the proxy"}, {Args: "--mode mitm -- curl -sS https://example.com", Says: "decrypt and record a command's HTTPS"}}, + New: func(KindDeps) tracing.TracePlugin { return httpproxy.UnifiedKind(nil).Preset(httpproxy.ProxyPreset()) }, + }, + { + Name: pgproxy.ProxyKind, Base: tracesql.RowKind, Title: "Postgres proxy", Seams: []target.Seam{target.Proxy}, DefaultSeam: target.Proxy, Armable: true, + Summary: "Record a command's SQL statements through a local Postgres proxy", + Description: "The sql kind armed only on its proxy seam: routes a command through a Postgres proxy and records its statements as sql does.", + Exports: sqlExports, + Examples: []Example{{Args: "--dsn postgres://app@localhost:5432/orders -- ./migrate up", Says: "record a command through the Postgres proxy"}}, + New: func(KindDeps) tracing.TracePlugin { return pgproxy.UnifiedKind(nil).Preset(pgproxy.ProxyPreset()) }, + }, { Name: terminal.Kind, Title: "Terminal", Seams: []target.Seam{target.Self}, DefaultSeam: target.Self, Summary: "Record terminal events a host feeds in, not the command's output", diff --git a/pkg/traces/builtin_clicky.go b/pkg/traces/builtin_clicky.go new file mode 100644 index 0000000..2ce970a --- /dev/null +++ b/pkg/traces/builtin_clicky.go @@ -0,0 +1,27 @@ +package traces + +import ( + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/entity" + "github.com/flanksource/telltale/pkg/tracing/target" + "github.com/flanksource/telltale/pkg/tracing/task" +) + +// The clicky kinds read the probes github.com/flanksource/clicky/telltale +// registers in a clicky process; telltale itself registers neither. +func init() { + MustRegister(Registration{ + Name: entity.Kind, Title: "Entity operations", Summary: "Record a clicky agent's entity operations", + Description: "Records the clicky entity operations (list, get, create, actions) an agent runs over the CLI, HTTP or MCP, with masked parameters and never their results.", + Examples: []Example{{Args: "--pid 1234 --entity widget --errors-only --duration 1m", Says: "record failed widget operations in process 1234"}}, + Seams: []target.Seam{target.Agent}, DefaultSeam: target.Agent, Armable: true, + New: func(KindDeps) tracing.TracePlugin { return entity.NewKind(nil) }, + }) + MustRegister(Registration{ + Name: task.Kind, Title: "Tasks", Summary: "Record a clicky agent's task transitions and progress", + Description: "Records the clicky tasks an agent already runs as it arms, then each status transition and throttled progress. Task logs and output are left to the logs and exec kinds.", + Examples: []Example{{Args: "--pid 1234 --duration 1m", Says: "record the tasks of process 1234"}}, + Seams: []target.Seam{target.Agent}, DefaultSeam: target.Agent, Armable: true, + New: func(KindDeps) tracing.TracePlugin { return task.NewKind(nil) }, + }) +} diff --git a/pkg/traces/builtin_jfr.go b/pkg/traces/builtin_jfr.go index d66c307..d13b723 100644 --- a/pkg/traces/builtin_jfr.go +++ b/pkg/traces/builtin_jfr.go @@ -7,5 +7,7 @@ import ( ) func init() { - MustRegister(Registration{Name: jfr.KindName, Title: "JVM JFR", Summary: "Profile a JVM with Flight Recorder or async-profiler", Description: "Attach to a local JVM process or select a connected Java agent. Checkpoints decode new profiling samples into the trace store.", Seams: []target.Seam{target.Process, target.Agent}, DefaultSeam: target.Process, Armable: true, Checkpoint: map[string]any{}, New: func(KindDeps) tracing.TracePlugin { return jfr.Kind() }, Examples: []Example{{Args: "--pid 1234 --duration 30s --every 10s", Says: "profile a local JVM"}, {Args: "--seam agent --selector service.name=example --duration 30s", Says: "profile connected Java agents"}}}) + MustRegister(Registration{Name: jfr.KindName, Title: "JVM JFR", Summary: "Profile a JVM with Flight Recorder or async-profiler", Description: "Attach to a local JVM process or select a connected Java agent. Checkpoints decode new profiling samples into the trace store.", Seams: []target.Seam{target.Process, target.Agent}, DefaultSeam: target.Process, Armable: true, Checkpoint: map[string]any{}, New: func(KindDeps) tracing.TracePlugin { return jfr.Kind() }, Examples: []Example{{Args: "--pid 1234 --duration 30s --every 10s", Says: "profile a local JVM"}, {Args: "--seam agent --selector service.name=example --duration 30s", Says: "profile connected Java agents"}}, Commands: []KindCommand{ + {Path: []string{"jvm", "profiler"}, Summary: "Profile a connected JVM with Flight Recorder or async-profiler", Defaults: map[string]any{"target": map[string]any{"seam": "agent"}}}, + }}) } diff --git a/pkg/traces/builtin_jvm.go b/pkg/traces/builtin_jvm.go new file mode 100644 index 0000000..13ae1cf --- /dev/null +++ b/pkg/traces/builtin_jvm.go @@ -0,0 +1,17 @@ +package traces + +import ( + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/jvmtrace" + "github.com/flanksource/telltale/pkg/tracing/target" +) + +func init() { + MustRegister(Registration{Name: jvmtrace.NotificationsKind, Title: "JMX notifications / GC", Summary: "Subscribe to JVM notifications", Description: "Record notifications from an MBean or ObjectName pattern, including decoded garbage collections and dropped-event counts.", Seams: []target.Seam{target.Agent}, DefaultSeam: target.Agent, Armable: true, Checkpoint: jvmtrace.CheckpointParams{}, New: func(KindDeps) tracing.TracePlugin { return jvmtrace.NewNotificationKind(nil) }}) + MustRegister(Registration{Name: jvmtrace.ExpressionKind, Title: "Periodic OGNL", Summary: "Sample an expression in a selected application loader", Description: "Evaluate an OGNL expression periodically and record bounded object graphs.", Seams: []target.Seam{target.Agent}, DefaultSeam: target.Agent, Armable: true, Checkpoint: jvmtrace.CheckpointParams{}, New: func(KindDeps) tracing.TracePlugin { return jvmtrace.NewExpressionKind(nil) }}) + MustRegister(Registration{Name: jvmtrace.RedefineKind, Title: "JVM redefinition", Summary: "Replace Java code for a capture and restore its original bytes", Description: "Compiles replacement source against a selected live class, applies it under a lease, and restores it when the capture ends.", Seams: []target.Seam{target.Agent}, DefaultSeam: target.Agent, Armable: true, Checkpoint: jvmtrace.CheckpointParams{}, New: func(KindDeps) tracing.TracePlugin { return jvmtrace.NewRedefineKind(nil) }}) + MustRegister(Registration{Name: jvmtrace.Kind, Title: "JVM methods", Summary: "Watch method values or trace nested calls", Description: "Instrument selected methods in a connected JVM, record values and call timings, and restore the original bytecode when the capture ends.", Seams: []target.Seam{target.Agent}, DefaultSeam: target.Agent, Armable: true, Checkpoint: jvmtrace.CheckpointParams{}, New: func(KindDeps) tracing.TracePlugin { return jvmtrace.NewKind(nil) }, Commands: []KindCommand{ + {Path: []string{"jvm", "watch"}, Summary: "Watch method arguments, return values and exceptions", Defaults: map[string]any{"args": true, "return": true, "exception": true, "calls": false}}, + {Path: []string{"jvm", "trace"}, Summary: "Trace methods with their nested calls, values and timings", Defaults: map[string]any{"args": true, "return": true, "exception": true, "calls": true}}, + }}) +} diff --git a/pkg/traces/builtin_observer.go b/pkg/traces/builtin_observer.go new file mode 100644 index 0000000..6013cb5 --- /dev/null +++ b/pkg/traces/builtin_observer.go @@ -0,0 +1,27 @@ +package traces + +import ( + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/httptop" + "github.com/flanksource/telltale/pkg/tracing/jms" + "github.com/flanksource/telltale/pkg/tracing/jmstop" + "github.com/flanksource/telltale/pkg/tracing/target" +) + +func init() { + MustRegister(Registration{Name: jmstop.Kind, Title: "JMS top", Summary: "Map JMS receive handlers and their activity", Description: "Retains discovered destination-to-handler mappings, including idle handlers.", Seams: []target.Seam{target.Agent}, DefaultSeam: target.Agent, Armable: true, Checkpoint: map[string]any{}, New: func(KindDeps) tracing.TracePlugin { return jmstop.NewKind(nil) }}) + MustRegister(Registration{ + Name: httptop.Kind, Title: "HTTP top", Summary: "Sample active incoming and outgoing HTTP exchanges", + Description: "Samples requests observed by an attached JVM, including Spring and asynchronous response bodies. Each complete snapshot replaces the previous view; an empty snapshot clears it.", + Seams: []target.Seam{target.Agent}, DefaultSeam: target.Agent, Armable: true, Checkpoint: map[string]any{}, + Examples: []Example{{Args: "--agent java/checkout --allow-mutation --direction both --poll 5s --top 50", Says: "watch the longest active JVM HTTP exchanges"}}, + New: func(KindDeps) tracing.TracePlugin { return httptop.NewKind(nil) }, + }) + MustRegister(Registration{ + Name: jms.Kind, Title: "JMS", Summary: "Record incoming and outgoing JVM messaging operations", + Description: "Records actual sent and received messages, including handler outcomes and failed sends with a known message. Empty receive polls are excluded. Message bodies are disabled by default.", + Seams: []target.Seam{target.Agent}, DefaultSeam: target.Agent, Armable: true, Checkpoint: map[string]any{}, + Examples: []Example{{Args: "--agent java/checkout --allow-mutation --direction both", Says: "record JVM JMS operations without reading message bodies"}}, + New: func(KindDeps) tracing.TracePlugin { return jms.NewKind(nil) }, + }) +} diff --git a/pkg/traces/builtin_prometheus.go b/pkg/traces/builtin_prometheus.go index caf60a8..8a2c94c 100644 --- a/pkg/traces/builtin_prometheus.go +++ b/pkg/traces/builtin_prometheus.go @@ -9,15 +9,18 @@ import ( func init() { MustRegister(Registration{ Name: prometheus.Kind, Title: "Prometheus metrics", Seams: []target.Seam{target.Host}, DefaultSeam: target.Host, - Summary: "Scrape Prometheus metrics endpoints into a Prometheus telltale manages", - Description: "Hands each endpoint to the data dir's managed Prometheus " + prometheus.Version + " as a file_sd target of job " + prometheus.Job + - ", scraped every --interval, and records one row per completed scrape: its endpoint, health, duration, error and sample count. " + - "The first capture downloads and starts that Prometheus through deps (state and TSDB under /services/prometheus) as telltale's sub-process: the run's captures share it, it stops when the run ends, and the next run starts it again on the same TSDB and port. " + - "The samples stay in its TSDB, labelled telltale_run, telltale_stream and telltale_endpoint plus --labels: query them with PromQL at the URL the run shows, or as the read-only " + - prometheus.ConnectionName + " connection of telltale serve. Ending the capture stops the scraping, not the samples.", + Summary: "Scrape Prometheus metrics endpoints into VictoriaMetrics", + Description: "Adds endpoints to the data directory's managed VictoriaMetrics " + prometheus.Version + ", which scrapes them at --interval and stores samples for 15 days. " + + "Captures report target health and emit zero scrape records. Run telltale serve --prometheus first to keep the instance available between runs using the same data directory. " + + "Query samples with PromQL using the URL shown by the capture or the read-only " + prometheus.ConnectionName + + " connection. Samples have telltale_run, telltale_stream and telltale_endpoint labels plus --labels. " + + "Inline Kubernetes endpoints use Telltale's Kubernetes identity. Replace the example service, namespace and port with yours. portforward:// and port-forward:// are equivalent.", Examples: []Example{ {Args: "--endpoints http://localhost:8080/metrics --interval 5s --duration 1m", Says: "scrape one endpoint every 5 seconds for a minute"}, {Args: "--endpoints http://localhost:9100/metrics,http://localhost:9090/metrics --labels env=dev", Says: "scrape two exporters every 15 seconds, labelled env=dev, until Ctrl+C"}, + {Args: "--endpoints port-forward://metrics-exporter.monitoring:9100/metrics --duration 1m", Says: "scrape a Kubernetes Service through a port-forward managed by Telltale"}, + {Args: "--endpoints proxy://metrics-exporter.monitoring:9100/metrics --duration 1m", Says: "scrape a Kubernetes Service through the API server proxy"}, + {Args: "--endpoints https://metrics.example.com/metrics --duration 1m", Says: "scrape an exporter reachable over HTTPS"}, }, New: func(deps KindDeps) tracing.TracePlugin { managed := prometheus.NewManaged(deps.DataDir) diff --git a/pkg/traces/builtin_query.go b/pkg/traces/builtin_query.go index a20d5fb..e328a08 100644 --- a/pkg/traces/builtin_query.go +++ b/pkg/traces/builtin_query.go @@ -12,7 +12,7 @@ import ( dbcontext "github.com/flanksource/commons-db/context" dbquery "github.com/flanksource/commons-db/query" "github.com/flanksource/commons-db/recordstore/recordresults" - "github.com/flanksource/commons-db/tracing/query" + "github.com/flanksource/telltale/pkg/tracing/query" "github.com/flanksource/telltale/pkg/datadir" "github.com/flanksource/telltale/pkg/tracing" @@ -35,7 +35,7 @@ type QueryParams struct { Provider *query.Provider `json:"provider,omitempty" clicky:"format=json,title=Provider,desc=A provider read in place: its type and connection or query and that provider's typed options"` } -// Query is the params commons-db's handler reads. +// Query is the params the query kind's handler reads. func (p QueryParams) Query() query.Params { params := p.Params params.Provider = p.Provider @@ -44,8 +44,8 @@ func (p QueryParams) Query() query.Params { func (p QueryParams) Validate() error { return p.Query().Validate() } -// queryReads is commons-db's query handler under telltale's runtime: the two -// runtimes' emitters and record readers are the same interfaces. +// queryReads is the query handler over QueryParams, the params carrying the +// host target every telltale capture takes. type queryReads struct{ reads query.Reads } func (q queryReads) Params() QueryParams { @@ -107,12 +107,11 @@ func init() { {Args: "--profile pod-cpu-by-pod --every 30s --duration 10m", Says: "poll a saved profile every 30 seconds for ten minutes"}, }, New: func(deps KindDeps) tracing.TracePlugin { - reads, err := query.NewReads(query.Options{Profiles: configProfiles{}, BeforeExecute: deps.BeforeExecute}) + reads, err := query.NewReads(query.Options{Sampled: true, Profiles: configProfiles{}, BeforeExecute: deps.BeforeExecute}) if err != nil { panic(fmt.Sprintf("query kind: %v", err)) } - capabilities := tracing.Capabilities{Live: query.Capabilities.Live, Historical: query.Capabilities.Historical, Follow: query.Capabilities.Follow} - return tracing.NewHandler[QueryParams, query.Row](queryReads{reads}, capabilities, target.Host) + return tracing.NewHandler[QueryParams, query.Row](queryReads{reads}, query.Capabilities, target.Host).WithTop(tracing.TopSpec{ProjectionColumn: "row"}) }, }) } diff --git a/pkg/traces/builtin_sqltop.go b/pkg/traces/builtin_sqltop.go new file mode 100644 index 0000000..902c6e3 --- /dev/null +++ b/pkg/traces/builtin_sqltop.go @@ -0,0 +1,20 @@ +package traces + +import ( + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/sqltop" + "github.com/flanksource/telltale/pkg/tracing/target" +) + +func init() { + MustRegister(Registration{Name: "sql_top", Title: "SQL top", Summary: "Sample running database requests or active JVM SQL executions", + Description: "Poll pg_stat_activity or sys.dm_exec_requests on a configured connection and record each running request, longest first: " + + "its session, wait, blocker, elapsed time and SQL. Sessions holding an open transaction are always recorded; --idle adds idle sessions. " + + "The agent seam samples active JDBC and R2DBC executions without a database connection; database-only measurements remain absent.", + Seams: []target.Seam{target.Connection, target.Agent}, DefaultSeam: target.Connection, Armable: true, Checkpoint: map[string]any{}, + Examples: []Example{ + {Args: "--connection connection://orders --duration 2m --poll 2s", Says: "watch what a Postgres runs for two minutes"}, + {Args: "--connection connection://warehouse --databases 'sales*' --min-elapsed 500ms --top 20", Says: "record the twenty longest SQL Server requests over half a second"}, + }, + New: func(KindDeps) tracing.TracePlugin { return sqltop.Kind() }}) +} diff --git a/pkg/traces/capture.go b/pkg/traces/capture.go index e555feb..dbde73c 100644 --- a/pkg/traces/capture.go +++ b/pkg/traces/capture.go @@ -9,6 +9,7 @@ import ( "github.com/flanksource/commons-db/query" "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/target" ) // Capture is one running capture: a managed session committing to its own @@ -17,11 +18,14 @@ type Capture struct { host *Host kind string managed *query.ManagedSession + target target.Target } // Kind is the trace kind the capture records. func (c *Capture) Kind() string { return c.kind } +func (c *Capture) Target() target.Target { return c.target } + func (c *Capture) Finished() bool { select { case <-c.managed.Session().Done(): diff --git a/pkg/traces/capture_internal_ginkgo_test.go b/pkg/traces/capture_internal_ginkgo_test.go index ae890af..f2f0f1e 100644 --- a/pkg/traces/capture_internal_ginkgo_test.go +++ b/pkg/traces/capture_internal_ginkgo_test.go @@ -11,6 +11,7 @@ import ( "github.com/flanksource/commons-db/query" "github.com/flanksource/telltale/pkg/tracing/process" + "github.com/flanksource/telltale/pkg/tracing/processcapture" "github.com/flanksource/telltale/pkg/tracing/target" ) @@ -19,7 +20,7 @@ var _ = Describe("marking a capture whose session failed", func() { host, err := Open(OpenOptions{DataDir: GinkgoT().TempDir(), RunID: RunID(time.Now())}) Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { Expect(host.Close(context.Background())).To(Succeed()) }) - capture, err := host.startWith(process.SamplesKind, process.SampleParams{SampleOptions: process.SampleOptions{Options: process.Options{Interval: time.Hour.String()}}, Target: target.Target{PID: os.Getpid()}}) + capture, err := host.startWith(processcapture.Kind, processcapture.Params{Options: processcapture.Options{Options: process.Options{Interval: time.Hour.String()}}, Target: target.Target{PID: os.Getpid()}}) Expect(err).ToNot(HaveOccurred()) cause := errors.New("read disk I/O for process 5675: operation not permitted") _, _ = capture.managed.Abort(cause) diff --git a/pkg/traces/host.go b/pkg/traces/host.go index e4dc480..d504490 100644 --- a/pkg/traces/host.go +++ b/pkg/traces/host.go @@ -29,6 +29,7 @@ import ( "github.com/flanksource/telltale/pkg/tracing/process" "github.com/flanksource/telltale/pkg/tracing/prometheus" "github.com/flanksource/telltale/pkg/tracing/stacks" + "github.com/flanksource/telltale/pkg/tracing/target" ) // Dir is where a data dir keeps its runs' trace stores, one directory each. @@ -59,23 +60,12 @@ type Host struct { releases []func(context.Context) error } -// RunPrefix and RunTimestampLayout name a run: run-, filesystem -// safe (no colons) and chronologically sortable. A run's trace store and its -// per-run snapshot (snapshots.SavePerRun) are both named so. -const ( - RunPrefix = "run-" - RunTimestampLayout = "2006-01-02T15-04-05Z" -) - -// RunID names a run started at started. -func RunID(started time.Time) string { - return RunPrefix + started.UTC().Format(RunTimestampLayout) -} - // OpenOptions name the run a Host is opened for. type OpenOptions struct { DataDir string RunID string + // MaxCaptures bounds simultaneous captures; zero uses the registry default. + MaxCaptures int // Connections answer the connection references its captures name, such // as sql_xevent's connection, before any connections table; nil answers // none. @@ -118,7 +108,7 @@ func runConnections(base *store.Store, dir string, defined []crd.Connection) (*s } // Open creates the run's store under DataDir/traces. A RunID already -// taken by an earlier run in the same second gets a -2, -3, … suffix, so a +// taken by an earlier run in the same second gets a _2, _3, … suffix, so a // run never writes into another's store; RunID reports the name used. func Open(options OpenOptions) (*Host, error) { if options.DataDir == "" || options.RunID == "" { @@ -148,7 +138,7 @@ func Open(options OpenOptions) (*Host, error) { } runtime.PollEvery = pollEvery runtime.Sessions = query.NewSessionRegistry(query.RegistryOptions{ - MaxDuration: maxDuration, BeforeRead: runtime.Results.Registry.BeforeRead, + MaxSessions: options.MaxCaptures, MaxDuration: maxDuration, BeforeRead: runtime.Results.Registry.BeforeRead, }) ctx := dbcontext.New() if connections != nil { @@ -158,7 +148,7 @@ func Open(options OpenOptions) (*Host, error) { if err != nil { return nil, err } - // A run that queries the managed Prometheus holds it as its scraping + // A run that queries VictoriaMetrics holds it as its scraping // captures do, so it is up for every read and stops once no run holds it. managed := prometheus.NewManaged(options.DataDir) releases = append(releases, managed.Release) @@ -237,7 +227,7 @@ func claimRunDir(root, runID string) (string, string, error) { for i := 1; i < 1000; i++ { name := runID if i > 1 { - name = fmt.Sprintf("%s-%d", runID, i) + name = fmt.Sprintf("%s_%d", runID, i) } dir := filepath.Join(root, name) err := os.Mkdir(dir, 0o755) @@ -358,11 +348,19 @@ func (h *Host) Start(options StartOptions) (*Capture, error) { if err := Validate(h.runtime.Kinds, options.Kind, options.Params); err != nil { return nil, err } + var selection struct { + Target target.Target `json:"target"` + } + if len(options.Params) > 0 { + if err := json.Unmarshal(options.Params, &selection); err != nil { + return nil, fmt.Errorf("read %s capture target: %w", options.Kind, err) + } + } managed, err := h.runtime.Start(h.ctx, tracing.StartRequest{Kind: options.Kind, Params: options.Params, Labels: options.Labels}) if err != nil { return nil, fmt.Errorf("start %s trace: %w", options.Kind, err) } - return &Capture{host: h, kind: options.Kind, managed: managed}, nil + return &Capture{host: h, kind: options.Kind, managed: managed, target: selection.Target}, nil } // Summarize is the kind's summary of ref's window; it always has `count`. @@ -371,7 +369,7 @@ func (h *Host) Summarize(ctx context.Context, ref tracing.TraceRef) (map[string] } // Close stops every capture still running, releases what the run's kinds -// hold, such as the managed Prometheus, and closes the store, which stays on +// hold, such as VictoriaMetrics, and closes the store, which stays on // disk for the run's snapshot to reopen. func (h *Host) Close(ctx context.Context) error { errs := []error{h.runtime.Sessions.StopAll(ctx)} diff --git a/pkg/traces/host_ginkgo_test.go b/pkg/traces/host_ginkgo_test.go index d4965ec..cccf257 100644 --- a/pkg/traces/host_ginkgo_test.go +++ b/pkg/traces/host_ginkgo_test.go @@ -9,6 +9,7 @@ import ( "net/url" "os" "path/filepath" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -36,6 +37,14 @@ func getThrough(capture *traces.Capture, target string) { } var _ = Describe("trace host", func() { + It("names a run with at most nine hostname characters and a compact UTC timestamp", func() { + hostname, err := os.Hostname() + Expect(err).NotTo(HaveOccurred()) + prefix := strings.ReplaceAll(strings.Split(hostname, ".")[0], "-", "") + prefix = prefix[:min(9, len(prefix))] + started := time.Date(2026, 10, 15, 11, 30, 0, 0, time.FixedZone("example", 2*60*60)) + Expect(traces.RunID(started)).To(Equal(prefix + "20261015T093000Z")) + }) var server *httptest.Server BeforeEach(func() { server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -54,7 +63,7 @@ var _ = Describe("trace host", func() { Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { Expect(second.Close(context.Background())).To(Succeed()) }) - Expect([]string{first.RunID(), second.RunID()}).To(Equal([]string{"run-2026-10-15T09-30-00Z", "run-2026-10-15T09-30-00Z-2"})) + Expect([]string{first.RunID(), second.RunID()}).To(Equal([]string{runID, runID + "_2"})) Expect(first.Dir()).To(Equal(filepath.Join(dataDir, "traces", first.RunID()))) }) @@ -131,6 +140,22 @@ var _ = Describe("trace host", func() { }) var _ = Describe("Prune", func() { + It("retains the newest start times across hostnames and old IDs", func() { + dataDir := GinkgoT().TempDir() + runs := []string{"zworker20261105T000000Z", "run-2026-11-06T00-00-00Z", "aworker20261107T000000Z"} + for index, run := range runs { + dir := traces.StoreDir(dataDir, run) + Expect(os.MkdirAll(dir, 0o755)).To(Succeed()) + data, err := json.Marshal(map[string]any{"startedAt": time.Date(2026, 11, 5+index, 0, 0, 0, 0, time.UTC)}) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(dir, traces.ManifestFile), data, 0o600)).To(Succeed()) + } + Expect(traces.Prune(dataDir, 2)).To(Succeed()) + Expect(os.Stat(traces.StoreDir(dataDir, runs[0]))).Error().To(MatchError(os.ErrNotExist)) + for _, id := range runs[1:] { + Expect(os.Stat(traces.StoreDir(dataDir, id))).Error().NotTo(HaveOccurred()) + } + }) It("keeps only the newest runs' stores", func() { dataDir := GinkgoT().TempDir() runs := []string{"run-2026-10-13T00-00-00Z", "run-2026-10-14T00-00-00Z", "run-2026-10-15T00-00-00Z"} diff --git a/pkg/traces/params_schema_spec_test.go b/pkg/traces/params_schema_spec_test.go index 1f4c643..b1ff616 100644 --- a/pkg/traces/params_schema_spec_test.go +++ b/pkg/traces/params_schema_spec_test.go @@ -15,10 +15,10 @@ import ( // kinds split their params into Options and a target; added are the // properties each kind gained since, which may differ from the golden. var schemaGoldens = map[string][]string{ - "exec": nil, "goprof": nil, "jvm_jfr": nil, "llm": nil, "process_groups": nil, "process_samples": nil, - "process_tree": nil, "prometheus": nil, "spans": nil, "sql_deadlock": nil, "sql_plancache": nil, - "sql_xevent": nil, "stacks": nil, "terminal": nil, - "http": {"listen", "export"}, + "exec": nil, "goprof": nil, "jvm_jfr": nil, "llm": nil, + "prometheus": nil, "spans": nil, "sql_deadlock": nil, "sql_plancache": nil, + "sql_xevent": nil, "sql_top": nil, "stacks": nil, "terminal": nil, + "http": {"listen", "export", "direction"}, "sql": {"listen", "export"}, "logs": {"target"}, // query's golden is commons-db's own query kind's schema: telltale's diff --git a/pkg/traces/pretty_records_test.go b/pkg/traces/pretty_records_test.go new file mode 100644 index 0000000..9d75397 --- /dev/null +++ b/pkg/traces/pretty_records_test.go @@ -0,0 +1,121 @@ +package traces_test + +import ( + "encoding/json" + "slices" + "time" + + "github.com/flanksource/clicky/api" + clickyexec "github.com/flanksource/clicky/exec" + dbgorm "github.com/flanksource/commons-db/gorm" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons/har" + "github.com/flanksource/commons/har/harpretty" + "github.com/flanksource/telltale/pkg/traces" + "github.com/flanksource/telltale/pkg/tracing/sqlserver/xetrace" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("styled trace records", func() { + type example struct { + kind string + row query.Row + contains []string + } + examples := []example{ + {"http", query.Row{"request": map[string]any{"method": "GET", "url": "https://example.test/orders"}, "response": map[string]any{"status": 200}, "time": 12.0}, []string{"GET", "https://example.test/orders", "200"}}, + {"sql", query.Row{"sql": "SELECT id FROM orders", "durationMs": 1500.0, "rows": 3}, []string{"SLOW SQL", "[1500ms]", "[rows:3]", "SELECT id FROM orders"}}, + {"http_proxy", query.Row{"request": map[string]any{"method": "POST", "url": "https://example.test/orders"}, "response": map[string]any{"status": 201}, "time": 12.0}, []string{"POST", "https://example.test/orders", "201"}}, + {"pg_proxy", query.Row{"sql": "SELECT id FROM orders", "durationMs": 1500.0, "rows": 3}, []string{"SLOW SQL", "[1500ms]", "[rows:3]", "SELECT id FROM orders"}}, + {"sql_xevent", query.Row{"name": "sql_statement_completed", "sql": "SELECT id FROM orders", "durationMs": 12.0, "rowCount": 3}, []string{"SELECT id FROM orders", "rows"}}, + {"sql_plancache", query.Row{"sql": "SELECT id FROM orders", "executions": 4, "avgElapsedMs": 12.0, "database": "checkout"}, []string{"SELECT id FROM orders", "4", "checkout"}}, + {"sql_top", query.Row{"database": "checkout", "sessionId": 41, "state": "active", "waitType": "Lock:transactionid", "elapsedMs": 1500.0, "blocked": true, "blockedBy": 40, "sql": "UPDATE orders SET paid = true"}, []string{"checkout", "#41", "Lock:transactionid", "blocked by #40", "UPDATE orders"}}, + {"sql_deadlock", query.Row{"database": "checkout", "victims": 1, "participants": 2, "victimStatement": "UPDATE orders SET total = 1"}, []string{"deadlock", "checkout", "UPDATE orders"}}, + {"exec", query.Row{"name": "git", "args": []string{"status"}, "exitCode": 0}, []string{"git", "status"}}, + {"entity", query.Row{"entity": "widget", "verb": "update", "targetId": "w-1", "surface": "http", "durationMs": 12.0, "error": "update rejected"}, []string{"widget", "update", "w-1", "http", "update rejected"}}, + {"task", query.Row{"phase": "status", "name": "deploy api", "status": "running", "progress": 2, "maxValue": 5}, []string{"status", "deploy api", "running", "2/5"}}, + {"terminal", query.Row{"kind": "r", "width": 120, "height": 40}, []string{"resize", "120", "40"}}, + {"llm", query.Row{"phase": "response", "model": "example-model", "usage": map[string]any{"inputTokens": 12, "outputTokens": 3}, "cost": 0.004}, []string{"response", "example-model", "12", "3", "$0.004"}}, + {"logs", query.Row{"severity": "error", "logger": "checkout", "message": "payment declined"}, []string{"ERROR", "checkout", "payment declined"}}, + {"spans", query.Row{"service": "checkout", "operation": "orders.submit", "status": "error", "durationMs": 12.0}, []string{"checkout", "orders.submit", "error"}}, + {"goprof", query.Row{"pid": 123, "profile": "heap", "value": 4096, "unit": "bytes", "frames": []map[string]any{{"func": "checkout.allocate"}}}, []string{"123", "heap", "checkout.allocate"}}, + {"stacks", query.Row{"pid": 123, "goroutineId": 7, "state": "IO wait", "frames": []map[string]any{{"func": "checkout.read"}}}, []string{"123", "7", "IO wait", "checkout.read"}}, + {"jvm_jfr", query.Row{"event": "alloc", "thread": "worker", "frame": "Checkout.allocate", "value": 4096, "unit": "bytes"}, []string{"alloc", "worker", "Checkout.allocate"}}, + {"http_top", query.Row{"direction": "outgoing", "method": "GET", "url": "https://example.test/orders/7", "elapsedMs": 125.0}, []string{"outgoing", "GET", "https://example.test/orders/7", "125"}}, + {"jms", query.Row{"operation": "send", "direction": "outgoing", "outcome": "success", "destination": "orders"}, []string{"send", "outgoing", "success", "orders"}}, + {"jvm_notifications", query.Row{"type": "java.management.memory.threshold.exceeded", "sequenceNumber": 12, "message": "heap threshold exceeded", "source": "java.lang:type=Memory"}, []string{"java.management.memory.threshold.exceeded", "heap threshold exceeded", "java.lang:type=Memory", "12"}}, + {"jvm_ognl", query.Row{"expression": "@java.lang.Runtime@getRuntime().freeMemory()", "result": 4096}, []string{"@java.lang.Runtime@getRuntime().freeMemory()"}}, + {"jvm_redefine", query.Row{"class": "example.Checkout", "loader": "app-loader", "applied": true, "digest": "patched-digest"}, []string{"example.Checkout", "app-loader", "patched-digest"}}, + {"jvm_trace", query.Row{"kind": "return", "class": "example.Checkout", "method": "submit", "descriptor": "(J)Z", "value": true}, []string{"return", "example.Checkout", "submit", "(J)Z"}}, + {"sql_top_active", query.Row{"operationId": "sql/7", "api": "jdbc", "elapsedMs": 125.0, "sql": "SELECT id FROM orders"}, []string{"sql/7", "jdbc", "125", "SELECT id FROM orders"}}, + {"prometheus", query.Row{"endpoint": "http://localhost/metrics", "health": "up", "samples": 42, "durationMs": 12.0}, []string{"http://localhost/metrics", "up", "42"}}, + {"query", query.Row{"profile": "request-rate", "provider": "prometheus", "row": map[string]any{"value": 7}}, []string{"request-rate", "prometheus", "7"}}, + {"process", query.Row{"pid": 123, "name": "checkout", "command": "checkout serve"}, []string{"123", "checkout", "serve"}}, + {"process_metrics", query.Row{"entity": "123@started", "cpuPercent": 12.5, "rssBytes": 4096}, []string{"123@started", "12.5", "CPU"}}, + {"process_facets", query.Row{"process": "123@started", "facet": "files", "operation": "open", "entity": "/orders.db"}, []string{"123@started", "files", "open", "/orders.db"}}, + } + + for _, kind := range []string{"goprof", "http_top", "jvm_jfr", "process", "prometheus", "query", "sql_plancache", "sql_top", "stacks"} { + examples = append(examples, example{kind: kind + "_frames", row: query.Row{"source": "worker", "completedAt": "2026-11-11T10:00:00Z", "retained": 1, "observed": 2}, contains: []string{"worker", "1/2 rows"}}) + } + DescribeTableSubtree("presents native fields with a typed formatter", func(e example) { + It("keeps meaningful content and semantic styles", func() { + kinds, err := traces.Kinds() + Expect(err).NotTo(HaveOccurred()) + before, err := json.Marshal(e.row) + Expect(err).NotTo(HaveOccurred()) + p, err := kinds.Present(e.kind, e.row) + Expect(err).NotTo(HaveOccurred()) + for _, content := range e.contains { + Expect(p.Summary.String()).To(ContainSubstring(content)) + } + Expect(p.Summary.HTML()).To(ContainSubstring("color:")) + after, err := json.Marshal(e.row) + Expect(err).NotTo(HaveOccurred()) + Expect(after).To(Equal(before)) + }) + }, func() []TableEntry { + entries := []TableEntry{} + for _, e := range examples { + entries = append(entries, Entry(e.kind, e)) + } + return entries + }()) + + It("covers every registered built-in record and named result", func() { + kinds, err := traces.Kinds() + Expect(err).NotTo(HaveOccurred()) + infos, err := kinds.List() + Expect(err).NotTo(HaveOccurred()) + var actual, expected []string + for _, e := range examples { + actual = append(actual, e.kind) + } + for _, info := range infos { + if slices.Contains(registrationNames(), info.Name) { + expected = append(expected, info.Name) + plugin, ok := kinds.Get(info.Name) + Expect(ok).To(BeTrue()) + for _, table := range plugin.ResultTables() { + expected = append(expected, info.Name+"_"+table.Name) + } + } + } + Expect(actual).To(ConsistOf(expected)) + }) + + DescribeTable("reuses established formatting without changing its styles", func(kind string, row query.Row, expected api.Text) { + kinds, err := traces.Kinds() + Expect(err).NotTo(HaveOccurred()) + p, err := kinds.Present(kind, row) + Expect(err).NotTo(HaveOccurred()) + Expect(p.Summary).To(Equal(expected)) + }, + Entry("HTTP HAR", "http", query.Row{"request": map[string]any{"method": "GET", "url": "https://example.test"}, "response": map[string]any{"status": 200}, "time": 12.0}, harpretty.Entry(har.Entry{Request: har.Request{Method: "GET", URL: "https://example.test"}, Response: har.Response{Status: 200}, Time: 12}).Pretty()), + Entry("SQL slow log", "sql", query.Row{"sql": "SELECT 1", "durationMs": 1500.0, "rows": 3}, dbgorm.SQLTrace{SQL: "SELECT 1", Duration: 1500 * time.Millisecond, Rows: 3, Slow: true}.Pretty()), + Entry("execution", "exec", query.Row{"name": "git", "args": []string{"status"}, "exitCode": 0}, clickyexec.ExecResult{Command: "git", Args: []string{"status"}, ExitCode: 0}.Pretty()), + Entry("SQL extended event", "sql_xevent", query.Row{"name": "sql_statement_completed", "sql": "SELECT 1", "durationMs": 12.0}, xetrace.StreamLine(xetrace.Event{Name: "sql_statement_completed", SQL: "SELECT 1", Duration: 12 * time.Millisecond}, xetrace.StreamLineOptions{ShowDatabase: true})), + ) +}) diff --git a/pkg/traces/pretty_variants_test.go b/pkg/traces/pretty_variants_test.go new file mode 100644 index 0000000..ad8b3a7 --- /dev/null +++ b/pkg/traces/pretty_variants_test.go @@ -0,0 +1,50 @@ +package traces_test + +import ( + "strings" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/telltale/pkg/traces" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("styled trace variants", func() { + DescribeTable("preserves status and failure semantics", func(kind string, row query.Row, content, color string) { + kinds, err := traces.Kinds() + Expect(err).NotTo(HaveOccurred()) + p, err := kinds.Present(kind, row) + Expect(err).NotTo(HaveOccurred()) + Expect(strings.Join(strings.Fields(p.Summary.String()), " ")).To(ContainSubstring(content)) + Expect(strings.ToLower(p.Summary.HTML())).To(ContainSubstring(color)) + }, + Entry("HTTP success", "http", query.Row{"request": map[string]any{"method": "GET"}, "response": map[string]any{"status": 200}}, "200", "#22c55e"), + Entry("HTTP redirect", "http", query.Row{"request": map[string]any{"method": "GET"}, "response": map[string]any{"status": 302}}, "302", "#3b82f6"), + Entry("HTTP client error", "http", query.Row{"request": map[string]any{"method": "GET"}, "response": map[string]any{"status": 404}}, "404", "#eab308"), + Entry("HTTP server error", "http", query.Row{"request": map[string]any{"method": "GET"}, "response": map[string]any{"status": 503}}, "503", "#ef4444"), + Entry("HTTP transport failure", "http", query.Row{"error": "connection refused"}, "connection refused", "#ef4444"), + Entry("SQL normal", "sql", query.Row{"sql": "SELECT 1", "rows": -1, "durationMs": 2.0}, "[rows:-1]", "#3b82f6"), + Entry("SQL error", "sql", query.Row{"sql": "SELECT missing", "error": "missing table"}, "missing table", "#ef4444"), + Entry("exec error", "exec", query.Row{"name": "git", "exitCode": 1, "error": "permission denied"}, "permission denied", "#ef4444"), + Entry("terminal output", "terminal", query.Row{"kind": "o", "data": "git version\n"}, "git version", "#3b82f6"), + Entry("terminal snapshot", "terminal", query.Row{"kind": "snapshot", "data": "ready"}, "snapshot", "#3b82f6"), + Entry("terminal failed exit", "terminal", query.Row{"kind": "exit", "exitCode": 1}, "exit 1", "#ef4444"), + Entry("LLM request", "llm", query.Row{"phase": "request", "model": "example-model", "prompt": "private prompt"}, "request", "#3b82f6"), + ) + + DescribeTable("keeps expanded payloads out of the compact row", func(kind string, row query.Row, detail string) { + kinds, err := traces.Kinds() + Expect(err).NotTo(HaveOccurred()) + p, err := kinds.Present(kind, row) + Expect(err).NotTo(HaveOccurred()) + Expect(p.Summary.String()).NotTo(ContainSubstring(detail)) + Expect(p.Detail.String()).To(ContainSubstring(detail)) + }, + Entry("HTTP response body", "http", query.Row{"request": map[string]any{"method": "GET"}, "response": map[string]any{"status": 200, "content": map[string]any{"mimeType": "application/json", "text": map[string]any{"result": "recorded-response"}}}}, "recorded-response"), + Entry("execution stdout", "exec", query.Row{"name": "git", "exitCode": 0, "stdoutTail": "recorded-stdout"}, "recorded-stdout"), + Entry("LLM prompt", "llm", query.Row{"phase": "request", "model": "example-model", "prompt": "recorded-prompt"}, "recorded-prompt"), + Entry("OGNL result", "jvm_ognl", query.Row{"expression": "@java.lang.Runtime@getRuntime().freeMemory()", "result": 4096}, "4096"), + Entry("JVM method value", "jvm_trace", query.Row{"kind": "return", "class": "example.Checkout", "method": "findOrder", "descriptor": "(J)Ljava/util/Map;", "value": map[string]any{"orderId": "recorded-method-value"}}, "recorded-method-value"), + ) +}) diff --git a/pkg/traces/process.go b/pkg/traces/process.go index fa8bbe0..1821737 100644 --- a/pkg/traces/process.go +++ b/pkg/traces/process.go @@ -10,23 +10,23 @@ import ( "github.com/flanksource/commons-db/query" "github.com/flanksource/telltale/pkg/tracing" "github.com/flanksource/telltale/pkg/tracing/process" + "github.com/flanksource/telltale/pkg/tracing/processcapture" "github.com/flanksource/telltale/pkg/tracing/stacks" "github.com/flanksource/telltale/pkg/tracing/target" ) -// ProcessTrace is the run-scope capture of the process kinds over one root: -// its tree, its samples and its stacks, the streams the Processes tab reads -// through the query mount. +// ProcessTrace captures a root's identities, metrics and resource changes, +// with independent stack checkpoints. type ProcessTrace struct { host *Host interval time.Duration tree *Capture - samples *Capture stacks *Capture } // ProcessTraceOptions choose what a process trace reads. type ProcessTraceOptions struct { + Capture *Capture // RootPID is the process whose tree is traced. RootPID int // Interval is how often the tree is sampled; zero is a second. @@ -35,10 +35,9 @@ type ProcessTraceOptions struct { Tree TreeOptions } -// TreeOptions choose what the run-wide process tree reads of each process: -// the same environment params a process_tree capture takes, and the facets. +// TreeOptions choose the environment and resource facets read by the process capture. type TreeOptions struct { - // Facets arms every process_samples facet and, unless Environment narrows + // Facets arms every process facet and, unless Environment narrows // it, reads each process's whole environment onto its node, as a run // watched in a UI does. Facets bool `json:"facets,omitempty" clicky:"title=Process facets,desc=Read every process's open files sockets and environment into the process tree"` @@ -52,6 +51,7 @@ type TreeOptions struct { type ProcessRefs struct { Tree tracing.TraceRef `json:"tree"` Samples tracing.TraceRef `json:"samples"` + Facets tracing.TraceRef `json:"facets"` Stacks tracing.TraceRef `json:"stacks"` } @@ -70,26 +70,32 @@ func (h *Host) StartProcessTrace(options ProcessTraceOptions) (*ProcessTrace, er if options.Interval == 0 { options.Interval = time.Second } - root, interval := target.Target{PID: options.RootPID}, process.Options{Interval: options.Interval.String()} - tree := process.TreeParams{TreeOptions: process.TreeOptions{Options: interval}, Target: root} - samples := process.SampleParams{SampleOptions: process.SampleOptions{Options: interval}, Target: root} + params := processcapture.Params{Options: processcapture.Options{Options: process.Options{Interval: options.Interval.String()}}, Target: target.Target{PID: options.RootPID}} if options.Tree.Facets { - tree.Environment, samples.Facets = &process.EnvironmentParams{}, process.Facets + params.Environment, params.Facets = &process.EnvironmentParams{}, []string{processcapture.Files, processcapture.Sockets, "connections"} } if options.Tree.Environment != nil { - tree.Environment = options.Tree.Environment + params.Environment = options.Tree.Environment } trace := &ProcessTrace{host: h, interval: options.Interval} + if options.Capture != nil { + if options.Capture.host != h || options.Capture.Kind() != processcapture.Kind { + return nil, fmt.Errorf("process trace requires a process capture from the same host") + } + trace.tree = options.Capture + } var err error for _, start := range []struct { capture **Capture kind string params any }{ - {&trace.tree, process.TreeKind, tree}, - {&trace.samples, process.SamplesKind, samples}, + {&trace.tree, processcapture.Kind, params}, {&trace.stacks, stacks.Kind, stacks.Params{Target: target.Target{PID: options.RootPID}}}, } { + if *start.capture != nil { + continue + } if *start.capture, err = h.startWith(start.kind, start.params); err != nil { return nil, errors.Join(err, trace.Stop(context.Background())) } @@ -110,13 +116,13 @@ func (p *ProcessTrace) Interval() time.Duration { return p.interval } // Refs are the whole tree, samples and stacks streams. func (p *ProcessTrace) Refs() ProcessRefs { - return ProcessRefs{Tree: p.tree.Ref(0, 0), Samples: p.samples.Ref(0, 0), Stacks: p.stacks.Ref(0, 0)} + return ProcessRefs{Tree: p.tree.Ref(0, 0), Samples: tracing.TraceRef{Kind: processcapture.MetricsTable, Stream: p.tree.Stream() + "-metrics"}, Facets: tracing.TraceRef{Kind: processcapture.FacetsTable, Stream: p.tree.Stream() + "-facets"}, Stacks: p.stacks.Ref(0, 0)} } // Sessions are the captures' sessions. func (p *ProcessTrace) Sessions() ProcessSessions { return ProcessSessions{ - Tree: p.tree.managed.Session().ID(), Samples: p.samples.managed.Session().ID(), Stacks: p.stacks.managed.Session().ID(), + Tree: p.tree.managed.Session().ID(), Samples: p.tree.managed.Session().ID(), Stacks: p.stacks.managed.Session().ID(), } } @@ -146,7 +152,7 @@ func (p *ProcessTrace) CheckpointStacks(ctx context.Context, params stacks.Check // Stop ends the captures; their streams stay in the run's store. func (p *ProcessTrace) Stop(ctx context.Context) error { var errs []error - for _, capture := range []*Capture{p.tree, p.samples, p.stacks} { + for _, capture := range []*Capture{p.tree, p.stacks} { if capture != nil { _, err := capture.Stop(ctx) errs = append(errs, err) diff --git a/pkg/traces/process_ginkgo_test.go b/pkg/traces/process_ginkgo_test.go index 237ac3c..3fe36f4 100644 --- a/pkg/traces/process_ginkgo_test.go +++ b/pkg/traces/process_ginkgo_test.go @@ -18,6 +18,7 @@ import ( "github.com/flanksource/telltale/pkg/traceserve" "github.com/flanksource/telltale/pkg/tracing" "github.com/flanksource/telltale/pkg/tracing/process" + "github.com/flanksource/telltale/pkg/tracing/processcapture" "github.com/flanksource/telltale/pkg/tracing/stacks" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -117,7 +118,7 @@ var _ = Describe("process trace", func() { host := openHost(GinkgoT().TempDir()) trace := startTrace(host, traces.ProcessTraceOptions{RootPID: root}) refs := trace.Refs() - Expect([]string{refs.Tree.Kind, refs.Samples.Kind, refs.Stacks.Kind}).To(Equal([]string{process.TreeKind, process.SamplesKind, stacks.Kind})) + Expect([]string{refs.Tree.Kind, refs.Samples.Kind, refs.Facets.Kind, refs.Stacks.Kind}).To(Equal([]string{processcapture.Kind, processcapture.MetricsTable, processcapture.FacetsTable, stacks.Kind})) store := serveHost(host) Eventually(func() map[string]map[string]any { return byCommand(store.records(refs.Tree)) }).WithTimeout(5 * time.Second). @@ -125,7 +126,7 @@ var _ = Describe("process trace", func() { sleep := byCommand(store.records(refs.Tree))["sleep 30"] Expect(sleep).To(HaveKeyWithValue("environment", BeNil())) Eventually(func() []map[string]any { return store.records(refs.Samples) }).WithTimeout(5 * time.Second). - Should(ContainElement(HaveKeyWithValue("key", sleep["key"]))) + Should(ContainElement(HaveKeyWithValue("entity", sleep["key"]))) Expect(store.records(refs.Samples)).To(HaveEach(HaveKeyWithValue("openFiles", BeNil()))) }) @@ -138,10 +139,9 @@ var _ = Describe("process trace", func() { Eventually(func() map[string]map[string]any { return byCommand(store.records(trace.Refs().Tree)) }).WithTimeout(5 * time.Second). Should(HaveKeyWithValue("sleep 30", HaveKeyWithValue("environment", HaveKeyWithValue("TELLTALE_TRACE_PROBE", "1")))) Eventually(func() []map[string]any { return store.records(trace.Refs().Samples) }).WithTimeout(5 * time.Second). - Should(ContainElement(And( - HaveKeyWithValue("openFiles", BeNumerically(">", 0)), HaveKeyWithValue("openFileList", Not(BeEmpty())), - HaveKeyWithValue("sockets", Not(BeNil())), HaveKeyWithValue("ports", Not(BeNil())), HaveKeyWithValue("connections", Not(BeNil())), - ))) + Should(ContainElement(And(HaveKeyWithValue("openFiles", BeNumerically(">", 0)), HaveKeyWithValue("receivingSockets", Not(BeNil())), HaveKeyWithValue("connectionsUnknown", Not(BeNil()))))) + Expect(store.records(trace.Refs().Samples)).To(HaveEach(Not(HaveKey("openFileList")))) + Expect(store.records(trace.Refs().Facets)).To(ContainElement(And(HaveKeyWithValue("facet", "files"), HaveKeyWithValue("operation", "added"), HaveKey("resource")))) }) It("reads only the environment its tree options name onto each process, arming no facet", func() { @@ -163,7 +163,7 @@ var _ = Describe("process trace", func() { trace := startTrace(host, traces.ProcessTraceOptions{RootPID: os.Getpid()}) store := serveHost(host) - Expect(store.post("/api/v1/sessions/"+trace.Sessions().Stacks+"/checkpoint", `{"transport": "inprocess", "reason": "manual"}`)).To(Equal(http.StatusNoContent)) + Expect(store.post("/api/v1/sessions/"+trace.Sessions().Stacks+"/checkpoint", `{"transport": "inprocess", "reason": "manual"}`)).To(Equal(http.StatusOK)) Expect(store.records(trace.Refs().Stacks)).To(ContainElement(And( HaveKeyWithValue("pid", BeEquivalentTo(os.Getpid())), HaveKeyWithValue("transport", stacks.TransportInProcess), diff --git a/pkg/traces/process_readings.go b/pkg/traces/process_readings.go index aceff82..ce7d86f 100644 --- a/pkg/traces/process_readings.go +++ b/pkg/traces/process_readings.go @@ -6,6 +6,7 @@ import ( "github.com/flanksource/telltale/pkg/tracing" "github.com/flanksource/telltale/pkg/tracing/process" + "github.com/flanksource/telltale/pkg/tracing/processcapture" ) // ProcessMark is where a span of the run begins in its samples stream. @@ -14,7 +15,10 @@ type ProcessMark struct{ Samples int64 } // Mark commits the samples taken so far and returns where the next span // begins. func (p *ProcessTrace) Mark(ctx context.Context) (ProcessMark, error) { - seq, err := p.samples.Mark(ctx) + if err := p.CheckpointTree(ctx); err != nil { + return ProcessMark{}, err + } + seq, err := p.host.highSeq(ctx, p.Refs().Samples.Stream) return ProcessMark{Samples: seq}, err } @@ -27,27 +31,44 @@ type ProcessReadings struct { // Readings are the samples taken after mark through now, and the tree. func (p *ProcessTrace) Readings(ctx context.Context, after ProcessMark) (ProcessReadings, error) { - to, err := p.samples.Mark(ctx) + mark, err := p.Mark(ctx) if err != nil { return ProcessReadings{}, err } var readings ProcessReadings - if to > after.Samples { - if readings.Samples, err = readAll[process.Sample](ctx, p.host, p.samples.Ref(after.Samples, to)); err != nil { - return ProcessReadings{}, err - } + refs := p.Refs() + identities, err := readAll[processcapture.Identity](ctx, p.host, refs.Tree) + if err != nil { + return readings, err + } + refs.Samples.AfterSeq, refs.Samples.ToSeq = after.Samples, mark.Samples + if mark.Samples <= after.Samples { + readings.Nodes = processcapture.Nodes(identities, nil) + return readings, nil + } + metrics, err := readAll[processcapture.MetricRow](ctx, p.host, refs.Samples) + if err != nil { + return readings, err } - readings.Nodes, err = p.Nodes(ctx) - return readings, err + readings.Nodes, readings.Samples = processcapture.Nodes(identities, metrics), processcapture.Samples(identities, metrics) + return readings, nil } // Nodes are every process of the tree as last seen, after re-observing it. func (p *ProcessTrace) Nodes(ctx context.Context) ([]process.Node, error) { - to, err := p.tree.Mark(ctx) - if err != nil || to == 0 { + if err := p.CheckpointTree(ctx); err != nil { + return nil, err + } + refs := p.Refs() + identities, err := readAll[processcapture.Identity](ctx, p.host, refs.Tree) + if err != nil { + return nil, err + } + metrics, err := readAll[processcapture.MetricRow](ctx, p.host, refs.Samples) + if err != nil { return nil, err } - return readAll[process.Node](ctx, p.host, p.tree.Ref(0, to)) + return processcapture.Nodes(identities, metrics), nil } func readAll[R any](ctx context.Context, host *Host, ref tracing.TraceRef) ([]R, error) { diff --git a/pkg/traces/prune.go b/pkg/traces/prune.go index c616f79..2a72456 100644 --- a/pkg/traces/prune.go +++ b/pkg/traces/prune.go @@ -1,11 +1,14 @@ package traces import ( + "cmp" + "encoding/json" "errors" "fmt" "os" "path/filepath" "slices" + "time" "github.com/flanksource/telltale/internal/safefs" ) @@ -14,7 +17,8 @@ import ( const KeepRuns = 20 // Prune removes every run store under dataDir/traces but the newest -// keep, newest by run id (a run id is its start time). A data dir that has +// keep, newest by recorded start time. A store without a manifest uses its +// directory modification time. A data dir that has // never traced a run has nothing to prune. func Prune(dataDir string, keep int) error { if keep < 1 { @@ -28,19 +32,56 @@ func Prune(dataDir string, keep int) error { if err != nil { return fmt.Errorf("prune trace stores: %w", err) } - var runs []string + type storedRun struct { + name string + started time.Time + } + var runs []storedRun for _, entry := range entries { if entry.IsDir() { - runs = append(runs, entry.Name()) + started, err := storeStartedAt(filepath.Join(root, entry.Name()), entry) + if err != nil { + return err + } + runs = append(runs, storedRun{entry.Name(), started}) } } if len(runs) <= keep { return nil } - slices.Sort(runs) + slices.SortFunc(runs, func(a, b storedRun) int { + if compared := a.started.Compare(b.started); compared != 0 { + return compared + } + return cmp.Compare(a.name, b.name) + }) var errs []error for _, run := range runs[:len(runs)-keep] { - errs = append(errs, safefs.SafeRemove(filepath.Join(root, run), safefs.RemoveContext{AllowedRoot: root})) + errs = append(errs, safefs.SafeRemove(filepath.Join(root, run.name), safefs.RemoveContext{AllowedRoot: root})) } return errors.Join(errs...) } + +func storeStartedAt(dir string, entry os.DirEntry) (time.Time, error) { + data, err := os.ReadFile(filepath.Join(dir, ManifestFile)) + if errors.Is(err, os.ErrNotExist) { + info, err := entry.Info() + if err != nil { + return time.Time{}, err + } + return info.ModTime(), nil + } + if err != nil { + return time.Time{}, err + } + var manifest struct { + StartedAt time.Time `json:"startedAt"` + } + if err := json.Unmarshal(data, &manifest); err != nil { + return time.Time{}, fmt.Errorf("prune trace store %s: %w", dir, err) + } + if manifest.StartedAt.IsZero() { + return time.Time{}, fmt.Errorf("prune trace store %s: manifest has no start time", dir) + } + return manifest.StartedAt, nil +} diff --git a/pkg/traces/query_ginkgo_test.go b/pkg/traces/query_ginkgo_test.go index a27b84f..c8d2154 100644 --- a/pkg/traces/query_ginkgo_test.go +++ b/pkg/traces/query_ginkgo_test.go @@ -20,7 +20,7 @@ import ( const sqliteRead = `{"provider":{"type":"sql","sql":{"type":"sqlite","url":":memory:"},"query":%q}}` // readTimeout bounds a read, long enough for one whose connection starts the -// managed Prometheus. +// managed VictoriaMetrics. const readTimeout = 30 * time.Second // readOnce runs a query capture with params in dataDir to its end and summarizes it. diff --git a/pkg/traces/query_managed_ginkgo_test.go b/pkg/traces/query_managed_ginkgo_test.go index 9b919bf..e8f1f02 100644 --- a/pkg/traces/query_managed_ginkgo_test.go +++ b/pkg/traces/query_managed_ginkgo_test.go @@ -11,16 +11,14 @@ import ( "github.com/flanksource/telltale/pkg/tracing/prometheus" ) -// Set TELLTALE_PROMETHEUS_LIVE=1 to run it: the first run downloads -// Prometheus prometheus.Version into ~/.deps. -var _ = Describe("the query kind over the managed Prometheus (live)", func() { +var _ = Describe("the query kind over VictoriaMetrics (live)", func() { BeforeEach(func() { if os.Getenv("TELLTALE_PROMETHEUS_LIVE") == "" { - Skip("TELLTALE_PROMETHEUS_LIVE is not set; this spec downloads and starts Prometheus " + prometheus.Version) + Skip("set TELLTALE_PROMETHEUS_LIVE to install and run VictoriaMetrics") } }) - It("starts and holds the data dir's managed Prometheus for a run that only queries it", func() { + It("starts and holds VictoriaMetrics for a run that only queries it", func() { params := json.RawMessage(fmt.Sprintf(`{"provider":{"type":"prometheus","connection":"connection://%s","query":"vector(1)"}}`, prometheus.ConnectionName)) Expect(readOnce(GinkgoT().TempDir(), params)).To(HaveKeyWithValue("count", BeNumerically("==", 1))) }) diff --git a/pkg/traces/read/events.go b/pkg/traces/read/events.go new file mode 100644 index 0000000..bee9e42 --- /dev/null +++ b/pkg/traces/read/events.go @@ -0,0 +1,20 @@ +package read + +import "github.com/flanksource/telltale/pkg/tracing" + +// EventRefs selects captures for the combined event view. +func EventRefs(refs []Ref) []Ref { + events := make([]Ref, 0, len(refs)) + for _, ref := range refs { + owner, _ := ref.Runtime.Kinds.ResultOf(ref.Kind) + plugin, _ := ref.Runtime.Kinds.Get(owner) + if tracing.TopOf(plugin) != nil { + continue + } + plugin, _ = ref.Runtime.Kinds.Get(ref.Kind) + if tracing.TopOf(plugin) == nil { + events = append(events, ref) + } + } + return events +} diff --git a/pkg/traces/read/feed.go b/pkg/traces/read/feed.go new file mode 100644 index 0000000..4617564 --- /dev/null +++ b/pkg/traces/read/feed.go @@ -0,0 +1,230 @@ +package read + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "maps" + + "github.com/flanksource/telltale/pkg/tracing" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/query/profilestore" + sqlpages "github.com/flanksource/commons-db/query/providers/sqlite" +) + +type Record struct { + presentation tracing.RecordPresentation + Summary string `json:"summary"` + RunID string `json:"runId"` + Kind string `json:"kind"` + Result string `json:"result,omitempty"` + Stream string `json:"stream"` + Seq int64 `json:"seq"` + Timestamp any `json:"timestamp"` + Record query.Row `json:"record"` +} + +type FeedQuery struct { + Query + Sort string `json:"sort"` +} + +type FeedResult struct { + Rows []Record `json:"rows"` + NextCursor string `json:"nextCursor"` +} + +type feedWindow struct { + Generation string `json:"g"` + High int64 `json:"h"` + Low int64 `json:"l"` + Compactions int64 `json:"c"` +} + +type feedPosition struct { + Scope string `json:"scope"` + Windows []feedWindow `json:"windows"` + Keys json.RawMessage `json:"keys"` + State map[string][]byte `json:"state"` +} + +func recordOf(ref Ref, row query.Row) (Record, error) { + seq, ok := row["seq"].(int64) + if !ok || seq < 1 { + return Record{}, fmt.Errorf("%s/%s: invalid sequence %v", ref.Kind, ref.Stream, row["seq"]) + } + plugin, ok := ref.Runtime.Kinds.Get(ref.Kind) + if !ok { + return Record{}, fmt.Errorf("unknown kind %s", ref.Kind) + } + schema, err := plugin.Schema(ref.Kind) + if err != nil { + return Record{}, err + } + presentation, err := ref.Runtime.Kinds.Present(ref.Kind, row) + if err != nil { + return Record{}, err + } + kind, result := ref.Runtime.Kinds.ResultOf(ref.Kind) + return Record{presentation: presentation, Summary: presentation.Summary.String(), RunID: ref.RunID, Kind: kind, Result: result, Stream: ref.Stream, Seq: seq, Timestamp: row[schema.Options.TimeColumn], Record: maps.Clone(row)}, nil +} + +// FeedPage reads one frozen window over every selected capture, through each +// kind's native profile and SQLite's common ordered-page merger. +func FeedPage(ctx context.Context, refs []Ref, q FeedQuery) (FeedResult, error) { + result := FeedResult{Rows: []Record{}} + if q.Sort == "" { + q.Sort = "seq" + } + if q.Sort != "seq" && q.Sort != "timestamp" { + return result, fmt.Errorf("unknown record sort %q", q.Sort) + } + if err := q.Validate(); err != nil { + return result, err + } + if q.Where != "" || q.All { + return result, fmt.Errorf("feed pages do not support CEL or unbounded reads") + } + if len(q.Cursor) > query.MaxCursorBytes { + return result, fmt.Errorf("feed cursor exceeds maximum size") + } + if q.Limit == 0 { + q.Limit = 100 + } + if q.Limit > 500 { + return result, fmt.Errorf("record page limit exceeds 500") + } + scope, err := cursorScope(refs, q.Query) + if err != nil { + return result, err + } + scope += ":" + q.Sort + // Scope binds the exact ordered refs, so windows need no repeated stream names. + saved := feedPosition{Scope: scope, Windows: make([]feedWindow, len(refs))} + if q.Cursor != "" { + data, err := base64.RawURLEncoding.DecodeString(q.Cursor) + if err != nil { + return result, fmt.Errorf("invalid feed cursor: %w", err) + } + if err := json.Unmarshal(data, &saved); err != nil { + return result, err + } + if saved.Scope != scope || len(saved.Windows) != len(refs) { + return result, fmt.Errorf("feed cursor no longer matches selection or sort") + } + } + order := query.Order{{Column: "seq", Unique: true}} + if q.Sort == "timestamp" { + order = append(query.Order{{Column: "timestamp"}}, order...) + } + members := []sqlpages.SQLMember{} + for index, ref := range refs { + key := ref.RunID + "/" + ref.Kind + "/" + ref.Stream + meta, err := ref.Runtime.Results.Backend.Meta(ctx, ref.Stream) + if err != nil { + return result, err + } + if q.Cursor == "" { + saved.Windows[index] = feedWindow{Generation: meta.Generation, High: meta.HighSeq, Low: meta.LowSeq, Compactions: meta.Compactions} + } + window := saved.Windows[index] + if window.Generation != meta.Generation { + return result, fmt.Errorf("feed stream %s changed generation", key) + } + if window.High > 0 && (window.Low != meta.LowSeq || window.Compactions != meta.Compactions) { + return result, fmt.Errorf("feed stream %s changed retained records", key) + } + if window.High == 0 { + continue + } + ref.ToSeq = window.High + member, err := feedMember(ctx, ref, q, order) + if err != nil { + return result, err + } + members = append(members, member) + } + if len(members) == 0 { + return result, nil + } + position := query.CursorPosition{State: saved.State} + if len(saved.Keys) > 0 { + position.Keys, err = query.UnmarshalCursorKeys(saved.Keys) + if err != nil { + return result, err + } + } + request := sqlpages.SQLMergeRequest{Members: members, Read: sqlpages.SQLPageRequest{Order: order, Position: position, Page: query.PageRequest{Limit: q.Limit, Strategy: query.PagingCursor, SkipTotal: true}}} + for page, err := range sqlpages.MergeSQLPages(ctx, request) { + if err != nil { + return result, err + } + for _, row := range page.Rows { + result.Rows = append(result.Rows, row["envelope"].(Record)) + } + if page.HasMore { + saved.Keys, err = query.MarshalCursorKeys(page.NextKeys) + if err != nil { + return result, err + } + saved.State = page.State + encoded, err := json.Marshal(saved) + if err != nil { + return result, err + } + result.NextCursor = base64.RawURLEncoding.EncodeToString(encoded) + if len(result.NextCursor) > query.MaxCursorBytes { + return result, fmt.Errorf("feed cursor exceeds maximum size") + } + } + break + } + return result, nil +} + +func feedMember(ctx context.Context, ref Ref, q FeedQuery, order query.Order) (sqlpages.SQLMember, error) { + dbctx, profile, err := ref.profile(ctx, "") + if err != nil { + return sqlpages.SQLMember{}, err + } + plugin, _ := ref.Runtime.Kinds.Get(ref.Kind) + schema, err := plugin.Schema(ref.Kind) + if err != nil { + return sqlpages.SQLMember{}, err + } + profile.Order = query.Order{{Column: "seq", Unique: true}} + if q.Sort == "timestamp" { + if schema.Options.TimeColumn == "" { + return sqlpages.SQLMember{}, fmt.Errorf("kind %s has no timestamp column", ref.Kind) + } + profile.Order = append(query.Order{{Column: schema.Options.TimeColumn}}, profile.Order...) + } + params, err := q.params(ref, profile) + if err != nil { + return sqlpages.SQLMember{}, err + } + return sqlpages.SQLMember{Name: ref.RunID + "/" + ref.Kind + "/" + ref.Stream, ReadPage: func(ctx context.Context, request sqlpages.SQLPageRequest) (sqlpages.SQLPageResult, error) { + release, err := ref.Runtime.Results.Registry.BeforeExecute(ctx, []profilestore.ReadRequest{{Profile: profile, Params: params}}) + if err != nil { + return sqlpages.SQLPageResult{}, err + } + defer release() + for page, err := range query.ExecutePages(dbctx.Wrap(ctx), profile, query.PageRequest{Limit: request.Page.Limit, Strategy: query.PagingCursor, After: request.Position.Keys}, params) { + if err != nil { + return sqlpages.SQLPageResult{}, err + } + rows := make([]query.Row, 0, len(page.Rows)) + for _, row := range page.Rows { + record, err := recordOf(ref, row) + if err != nil { + return sqlpages.SQLPageResult{}, err + } + rows = append(rows, query.Row{"seq": record.Seq, "timestamp": record.Timestamp, "envelope": record}) + } + return sqlpages.SQLPageResult{Rows: rows, HasMore: page.HasMore, Total: page.Total}, nil + } + return sqlpages.SQLPageResult{}, fmt.Errorf("profile %s returned no page", profile.Name) + }}, nil +} diff --git a/pkg/traces/read/feed_follow.go b/pkg/traces/read/feed_follow.go new file mode 100644 index 0000000..68e9036 --- /dev/null +++ b/pkg/traces/read/feed_follow.go @@ -0,0 +1,42 @@ +package read + +import ( + "context" + "errors" + "github.com/flanksource/commons-db/query" + "sync" +) + +// Feed follows native sequences independently. A callback is never concurrent +// with another callback, and cancellation waits for every reader to release. +func Feed(ctx context.Context, refs []Ref, q Query, fn func(Record) error) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + var group sync.WaitGroup + var output sync.Mutex + errs := make(chan error, len(refs)) + for _, ref := range refs { + group.Go(func() { + err := Follow(ctx, ref, q, func(row query.Row) error { + record, err := recordOf(ref, row) + if err != nil { + return err + } + output.Lock() + defer output.Unlock() + return fn(record) + }) + if err != nil && !errors.Is(err, context.Canceled) { + errs <- err + cancel() + } + }) + } + group.Wait() + close(errs) + var err error + for failure := range errs { + err = errors.Join(err, failure) + } + return err +} diff --git a/pkg/traces/read/feed_test.go b/pkg/traces/read/feed_test.go new file mode 100644 index 0000000..46f8646 --- /dev/null +++ b/pkg/traces/read/feed_test.go @@ -0,0 +1,97 @@ +package read_test + +import ( + "context" + "fmt" + "time" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/telltale/pkg/traces" + "github.com/flanksource/telltale/pkg/traces/read" + "github.com/flanksource/telltale/pkg/tracing" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("unified record feed", func() { + It("excludes Top samples and their named results from event selections", func() { + kinds, err := traces.Kinds() + Expect(err).NotTo(HaveOccurred()) + runtime := &tracing.Runtime{Kinds: kinds} + var refs []read.Ref + for _, kind := range []string{"http", "sql", "jms", "http_top", "http_top_frames", "sql_top", "sql_top_active", "sql_top_frames", "jms_top", "jms_top_frames"} { + refs = append(refs, read.Ref{Runtime: runtime, TraceRef: tracing.TraceRef{Kind: kind, Stream: kind}}) + } + Expect(read.EventRefs(refs)).To(ConsistOf(HaveField("Kind", "http"), HaveField("Kind", "sql"), HaveField("Kind", "jms"))) + Expect(refs).To(HaveLen(10)) + }) + BeforeEach(func() { feeding = &feed{rows: make(chan reading), ack: make(chan error, 1)} }) + It("keeps identical stream names from different runs separate across pages", func() { + host, capture := fixture(GinkgoT().TempDir(), "run-shared-stream") + appendRow(capture, reading{At: time.Now().UTC(), Label: "recorded"}) + refs := []read.Ref{ + {RunID: "first", Runtime: host.Runtime(), TraceRef: capture.Ref(0, 0)}, + {RunID: "second", Runtime: host.Runtime(), TraceRef: capture.Ref(0, 0)}, + } + first, err := read.FeedPage(context.Background(), refs, read.FeedQuery{Limit: 1}) + Expect(err).NotTo(HaveOccurred()) + Expect(first.Rows).To(HaveLen(1)) + Expect(first.Rows[0].RunID).To(Equal("first")) + second, err := read.FeedPage(context.Background(), refs, read.FeedQuery{Limit: 1, Cursor: first.NextCursor}) + Expect(err).NotTo(HaveOccurred()) + Expect(second.Rows).To(HaveLen(1)) + Expect(second.Rows[0].RunID).To(Equal("second")) + Expect(second.NextCursor).To(BeEmpty()) + }) + It("pages eighteen captures without exceeding the cursor size limit", func() { + host, capture := fixture(GinkgoT().TempDir(), "run-feed-capacity") + base := time.Now().UTC() + for range 3 { + appendRow(capture, reading{At: base, Label: "recorded"}) + } + refs := make([]read.Ref, 18) + for i := range refs { + refs[i] = read.Ref{RunID: fmt.Sprintf("demo-host20261107T205920Z_%02d", i), Runtime: host.Runtime(), TraceRef: capture.Ref(0, 0)} + } + cursor, count := "", 0 + for { + page, err := read.FeedPage(context.Background(), refs, read.FeedQuery{Sort: "timestamp", Limit: 1, Cursor: cursor}) + Expect(err).NotTo(HaveOccurred()) + Expect(len(page.NextCursor)).To(BeNumerically("<=", query.MaxCursorBytes)) + count += len(page.Rows) + cursor = page.NextCursor + if cursor == "" { + break + } + } + Expect(count).To(Equal(54)) + }) + It("pages overlapping sequences from distinct captures and freezes the read window", func() { + host, first := fixture(GinkgoT().TempDir(), "run-feed") + base := time.Now().UTC().Truncate(time.Second) + appendRow(first, reading{At: base.Add(time.Second), Label: "first", Kind: "stdout"}) + _, err := first.Stop(context.Background()) + Expect(err).NotTo(HaveOccurred()) + second, err := host.Start(traces.StartOptions{Kind: "read_static"}) + Expect(err).NotTo(HaveOccurred()) + appendRow(second, reading{At: base, Label: "second"}) + refs := []read.Ref{{RunID: host.RunID(), Runtime: host.Runtime(), TraceRef: first.Ref(0, 0)}, {RunID: host.RunID(), Runtime: host.Runtime(), TraceRef: second.Ref(0, 0)}} + page, err := read.FeedPage(context.Background(), refs, read.FeedQuery{Limit: 1}) + Expect(err).NotTo(HaveOccurred()) + Expect(page.Rows).To(HaveLen(1)) + Expect(page.Rows[0].Kind).To(Equal("read_fixture")) + Expect(page.Rows[0].Record).To(HaveKeyWithValue("kind", "stdout")) + Expect(page.NextCursor).NotTo(BeEmpty()) + appendRow(second, reading{At: base, Label: "later"}) + next, err := read.FeedPage(context.Background(), refs, read.FeedQuery{Limit: 1, Cursor: page.NextCursor}) + Expect(err).NotTo(HaveOccurred()) + Expect(next.Rows).To(HaveLen(1)) + Expect(next.Rows[0].Record).To(HaveKeyWithValue("label", "second")) + Expect(next.NextCursor).To(BeEmpty()) + sorted, err := read.FeedPage(context.Background(), refs, read.FeedQuery{Sort: "timestamp"}) + Expect(err).NotTo(HaveOccurred()) + Expect(sorted.Rows[0].Kind).To(Equal("read_static")) + _, err = read.FeedPage(context.Background(), refs, read.FeedQuery{Sort: "timestamp", Cursor: page.NextCursor}) + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/pkg/traces/read/follow.go b/pkg/traces/read/follow.go index 2da66fd..bcac7e2 100644 --- a/pkg/traces/read/follow.go +++ b/pkg/traces/read/follow.go @@ -7,7 +7,6 @@ import ( "sync" "github.com/flanksource/commons-db/query" - "github.com/flanksource/commons-db/recordstore" "github.com/flanksource/telltale" "github.com/flanksource/telltale/pkg/traces" ) @@ -21,7 +20,7 @@ func Follow(ctx context.Context, ref Ref, q Query, fn func(query.Row) error) err if q.Cursor != "" { return fmt.Errorf("--follow cannot use a page cursor; use --after-seq") } - dbctx, p, err := ref.profile(ctx, "") + _, p, err := ref.profile(ctx, "") if err != nil { return err } @@ -35,92 +34,52 @@ func Follow(ctx context.Context, ref Ref, q Query, fn func(query.Row) error) err if through > 0 && max(ref.AfterSeq, q.AfterSeq) >= through { return nil } - count := 0 - // Query every notified seq through the original profile, preserving closing - // time/filter edges that query.Follow deliberately opens for rolling views. - deliver := func(seq int64) error { - if through > 0 && seq > through { - return errFollowLimit - } + count, last := 0, max(ref.AfterSeq, q.AfterSeq) + // Finish each native profile read before rendering; output holds no SQLite reader. + deliver := func(high int64) error { window := q - window.AfterSeq = seq - 1 - window.ToSeq = seq - window.All = true + window.AfterSeq, window.ToSeq = last, high + window.All, window.seqOrder = false, true + window.Limit = 500 window.Cursor = "" window.NextCursor = nil - for row, err := range Rows(ctx, []Ref{ref}, window) { - if err != nil { - return err - } + rows, err := query.CollectRows(Rows(ctx, []Ref{ref}, window)) + if err != nil { + return err + } + for _, row := range rows { if err := fn(row); err != nil { return err } + last = row["seq"].(int64) count++ if q.Limit > 0 && count >= q.Limit { return errFollowLimit } } - if through > 0 && seq >= through { - return errFollowLimit + if len(rows) < window.Limit { + last = high } return nil } - if !query.SupportsStreaming(p.Provider.Type) { - err = ref.Runtime.Results.Backend.Tail(ctx, ref.Stream, max(ref.AfterSeq, q.AfterSeq), func(seq int64, _ recordstore.Row) error { return deliver(seq) }) - } else { - followCtx, cancel := dbctx.WithCancel() - defer cancel() - // The notification stream carries every seq; filtering stays in deliver. - window := Query{AfterSeq: q.AfterSeq, ToSeq: q.ToSeq} - params, paramsErr := window.params(ref, p) - if paramsErr != nil { - return paramsErr - } - p = query.Follow(p) - p.Trace.Buffer = nil - session, startErr := query.ExecuteStream(followCtx, ref.Runtime.Sessions, p, params) - if startErr != nil { - return startErr - } - defer func() { cancel(); <-session.Done() }() - replay, live, unsubscribe := session.Subscribe() - defer unsubscribe() - var previous int64 - consume := func(event query.Event) error { - if event.Sequence != previous+1 { - return fmt.Errorf("follow subscriber lost events after %d (received %d)", previous, event.Sequence) - } - previous = event.Sequence - if event.Error != "" { - return errors.New(event.Error) - } - if event.Row == nil { - return nil - } - seq, ok := event.Row["seq"].(int64) - if !ok { - return fmt.Errorf("follow %s: expected int64 seq, got %T", ref.Kind, event.Row["seq"]) - } - return deliver(seq) - } - for _, event := range replay { - if err = consume(event); err != nil { - break - } - } - if err == nil { - for event := range live { - if err = consume(event); err != nil { - break - } - } + backend := ref.Runtime.Results.Backend + meta, err := backend.Meta(ctx, ref.Stream) + if err != nil { + return err + } + for err == nil { + high := meta.HighSeq + if through > 0 { + high = min(high, through) } - if err == nil && session.Snapshot().Error != "" { - err = errors.New(session.Snapshot().Error) + if last < high { + err = deliver(high) + continue } - if err == nil && session.Snapshot().EventCount != previous { - err = fmt.Errorf("follow subscriber lost trailing events after %d (produced %d)", previous, session.Snapshot().EventCount) + if meta.Sealed || through > 0 && last >= through { + return nil } + meta, err = backend.Wait(ctx, ref.Stream, last, meta.Generation) } if errors.Is(err, errFollowLimit) { return nil @@ -133,7 +92,7 @@ func Follow(ctx context.Context, ref Ref, q Query, fn func(query.Row) error) err // Wait follows every armed capture while Wait performs the run's usual stop, // summary and manifest write. Callbacks are serialized across captures. -func Wait(ctx context.Context, run *telltale.Run, follow bool, fn func(query.Row) error) (_ *telltale.Result, err error) { +func Wait(ctx context.Context, run *telltale.Run, follow bool, fn func(Record) error) (_ *telltale.Result, err error) { if !follow { return run.Wait() } @@ -144,7 +103,7 @@ func Wait(ctx context.Context, run *telltale.Run, follow bool, fn func(query.Row } defer func() { err = errors.Join(err, closeStore()) }() runContext := ctx - ctx, cancel := context.WithCancel(ctx) + ctx, cancel := context.WithCancel(context.WithoutCancel(ctx)) defer cancel() var group sync.WaitGroup var output sync.Mutex @@ -155,10 +114,11 @@ func Wait(ctx context.Context, run *telltale.Run, follow bool, fn func(query.Row followErr := Follow(ctx, ref, Query{}, func(row query.Row) error { output.Lock() defer output.Unlock() - if len(info.Captures) > 1 { - row["kind"] = ref.Kind + record, err := recordOf(ref, row) + if err != nil { + return err } - return fn(row) + return fn(record) }) if runContext.Err() != nil && errors.Is(followErr, runContext.Err()) { return diff --git a/pkg/traces/read/follow_backpressure_test.go b/pkg/traces/read/follow_backpressure_test.go new file mode 100644 index 0000000..a02023c --- /dev/null +++ b/pkg/traces/read/follow_backpressure_test.go @@ -0,0 +1,58 @@ +package read_test + +import ( + "context" + "fmt" + "time" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/telltale/pkg/traces/read" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("following with a slow output", func() { + BeforeEach(func() { feeding = &feed{rows: make(chan reading), ack: make(chan error, 1)} }) + It("drains a sealed capture without dropping records while rendering is slower than reading", func() { + const count = 600 + host, capture := fixture(GinkgoT().TempDir(), "run-slow-output") + for range count { + appendRow(capture, reading{At: time.Now().UTC(), Label: "recorded"}) + } + end, err := capture.Stop(context.Background()) + Expect(err).NotTo(HaveOccurred()) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + got := 0 + err = read.Follow(ctx, read.Ref{RunID: host.RunID(), Runtime: host.Runtime(), TraceRef: capture.Ref(0, end)}, read.Query{}, func(query.Row) error { + time.Sleep(2 * time.Millisecond) + got++ + return nil + }) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(Equal(count)) + }) + It("follows eighteen captures sharing a SQLite store without exhausting reader connections", func() { + host, capture := fixture(GinkgoT().TempDir(), "run-many-followers") + for range 3 { + appendRow(capture, reading{At: time.Now().UTC(), Label: "recorded"}) + } + end, err := capture.Stop(context.Background()) + Expect(err).NotTo(HaveOccurred()) + refs := make([]read.Ref, 18) + for i := range refs { + refs[i] = read.Ref{RunID: fmt.Sprint(i), Runtime: host.Runtime(), TraceRef: capture.Ref(0, end)} + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + got := 0 + err = read.Feed(ctx, refs, read.Query{}, func(read.Record) error { + time.Sleep(time.Millisecond) + got++ + return nil + }) + Expect(err).NotTo(HaveOccurred()) + Expect(ctx.Err()).NotTo(HaveOccurred()) + Expect(got).To(Equal(54)) + }) +}) diff --git a/pkg/traces/read/lifecycle_test.go b/pkg/traces/read/lifecycle_test.go index 0424442..ea34813 100644 --- a/pkg/traces/read/lifecycle_test.go +++ b/pkg/traces/read/lifecycle_test.go @@ -7,7 +7,6 @@ import ( "path/filepath" "time" - "github.com/flanksource/commons-db/query" "github.com/flanksource/telltale" "github.com/flanksource/telltale/pkg/traces/read" . "github.com/onsi/ginkgo/v2" @@ -18,16 +17,16 @@ var _ = Describe("live run manifest", func() { It("still returns the final result after the run context is cancelled", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - run, err := telltale.Start(ctx, telltale.StartOptions{DataDir: GinkgoT().TempDir(), PID: os.Getpid(), Duration: 5 * time.Second, Traces: []telltale.TraceSpec{{Kind: "process_samples"}}}) + run, err := telltale.Start(ctx, telltale.StartOptions{DataDir: GinkgoT().TempDir(), PID: os.Getpid(), Duration: 5 * time.Second, Traces: []telltale.TraceSpec{{Kind: "process"}}}) Expect(err).NotTo(HaveOccurred()) DeferCleanup(func() { Expect(run.Stop()).To(Succeed()); _, err := run.Wait(); Expect(err).NotTo(HaveOccurred()) }) - result, err := read.Wait(ctx, run, true, func(query.Row) error { cancel(); return nil }) + result, err := read.Wait(ctx, run, true, func(read.Record) error { cancel(); return nil }) Expect(err).NotTo(HaveOccurred()) Expect(result).NotTo(BeNil()) - Expect(result.Captures).To(ConsistOf(HaveField("Kind", "process_samples"))) + Expect(result.Captures).To(ConsistOf(HaveField("Kind", "process"))) }) It("publishes seq-zero refs at start and replaces them with the compatible final result", func() { - run, err := telltale.Start(context.Background(), telltale.StartOptions{DataDir: GinkgoT().TempDir(), PID: os.Getpid(), Duration: time.Second, Traces: []telltale.TraceSpec{{Kind: "process_samples"}}}) + run, err := telltale.Start(context.Background(), telltale.StartOptions{DataDir: GinkgoT().TempDir(), PID: os.Getpid(), Duration: time.Second, Traces: []telltale.TraceSpec{{Kind: "process"}}}) Expect(err).NotTo(HaveOccurred()) DeferCleanup(func() { Expect(run.Stop()).To(Succeed()); _, err := run.Wait(); Expect(err).NotTo(HaveOccurred()) }) data, err := os.ReadFile(filepath.Join(run.Dir(), telltale.ManifestFile)) @@ -37,7 +36,8 @@ var _ = Describe("live run manifest", func() { Expect(live.Status).To(Equal(telltale.StatusRunning)) Expect(live.EndedAt.IsZero()).To(BeTrue()) Expect(live.ExitCode).To(BeNil()) - Expect(live.Captures).To(ConsistOf(And(HaveField("Kind", "process_samples"), HaveField("Ref.AfterSeq", int64(0)), HaveField("Ref.ToSeq", int64(0)), HaveField("Ref.Stream", Not(BeEmpty()))))) + Expect(live.Captures).To(ConsistOf(And(HaveField("Kind", "process"), HaveField("Ref.AfterSeq", int64(0)), HaveField("Ref.ToSeq", int64(0)), HaveField("Ref.Stream", Not(BeEmpty()))))) + Expect(live.Process.Tree.Stream).To(Equal(live.Captures[0].Ref.Stream), "reuse the configured process capture") result, err := run.Wait() Expect(err).NotTo(HaveOccurred()) final, err := telltale.ReadManifest(run.Dir()) diff --git a/pkg/traces/read/order.go b/pkg/traces/read/order.go new file mode 100644 index 0000000..e2df26d --- /dev/null +++ b/pkg/traces/read/order.go @@ -0,0 +1,47 @@ +package read + +import ( + "cmp" + "fmt" + sqlpages "github.com/flanksource/commons-db/query/providers/sqlite" + "slices" +) + +func OrderRecords(records []Record, order string) ([]Record, error) { + if order != "seq" && order != "timestamp" { + return nil, fmt.Errorf("unknown record sort %q", order) + } + rows := slices.Clone(records) + var err error + slices.SortStableFunc(rows, func(a, b Record) int { + if order == "timestamp" && a.Timestamp != nil && b.Timestamp != nil { + compared, compareErr := sqlpages.CompareSQLite(a.Timestamp, b.Timestamp) + if compareErr != nil { + err = compareErr + return 0 + } + if compared != 0 { + return compared + } + } + if order == "timestamp" { + if a.Timestamp == nil && b.Timestamp != nil { + return 1 + } + if b.Timestamp == nil && a.Timestamp != nil { + return -1 + } + } + if compared := cmp.Compare(a.Seq, b.Seq); compared != 0 { + return compared + } + if compared := cmp.Compare(a.Kind, b.Kind); compared != 0 { + return compared + } + if compared := cmp.Compare(a.RunID, b.RunID); compared != 0 { + return compared + } + return cmp.Compare(a.Stream, b.Stream) + }) + return rows, err +} diff --git a/pkg/traces/read/presentation.go b/pkg/traces/read/presentation.go new file mode 100644 index 0000000..b088045 --- /dev/null +++ b/pkg/traces/read/presentation.go @@ -0,0 +1,23 @@ +package read + +import ( + "fmt" + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" +) + +func (r Record) Pretty() api.Text { + return api.Text{}.Add(r.prefix()).Space().Add(r.presentation.Summary) +} + +func (r Record) PrettyFull() api.Textable { + return api.Text{}.Add(r.prefix()).Space().Add(r.presentation.Detail) +} + +func (r Record) prefix() api.Textable { + kind := r.Kind + if r.Result != "" { + kind += "/" + r.Result + } + return clicky.Text(fmt.Sprintf("%s #%d", kind, r.Seq), "text-muted") +} diff --git a/pkg/traces/read/processes.go b/pkg/traces/read/processes.go new file mode 100644 index 0000000..52f5017 --- /dev/null +++ b/pkg/traces/read/processes.go @@ -0,0 +1,36 @@ +package read + +import ( + "context" + + "github.com/flanksource/telltale/pkg/tracing/processcapture" +) + +type ProcessRow struct { + processcapture.ProcessRow + RunID string `json:"runId"` + Stream string `json:"stream"` +} + +type ProcessSnapshot struct { + Armed bool `json:"armed"` + Rows []ProcessRow `json:"rows"` +} + +func Processes(ctx context.Context, refs []Ref) (ProcessSnapshot, error) { + result := ProcessSnapshot{Rows: []ProcessRow{}} + for _, ref := range refs { + if ref.Kind != processcapture.Kind { + continue + } + result.Armed = true + rows, err := processcapture.ReadLatest(ctx, ref.Runtime, ref.TraceRef) + if err != nil { + return result, err + } + for _, row := range rows { + result.Rows = append(result.Rows, ProcessRow{ProcessRow: row, RunID: ref.RunID, Stream: ref.Stream}) + } + } + return result, nil +} diff --git a/pkg/traces/read/processes_test.go b/pkg/traces/read/processes_test.go new file mode 100644 index 0000000..d5cf79d --- /dev/null +++ b/pkg/traces/read/processes_test.go @@ -0,0 +1,36 @@ +package read_test + +import ( + "context" + "time" + + "github.com/flanksource/commons-db/recordstore/metrics" + "github.com/flanksource/telltale/pkg/traces" + "github.com/flanksource/telltale/pkg/traces/read" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/processcapture" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("process header reader", func() { + It("returns latest measurements for every process without the history page limit", func() { + host, err := traces.Open(traces.OpenOptions{DataDir: GinkgoT().TempDir(), RunID: "header"}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { Expect(host.Close(context.Background())).To(Succeed()) }) + store, err := processcapture.NewStore(processcapture.StoreOptions{Backend: host.Runtime().Results.Backend, Stream: "process-header"}) + Expect(err).NotTo(HaveOccurred()) + at := time.Unix(100, 0).UTC() + for tick := int64(1); tick <= 501; tick++ { + Expect(store.Apply(context.Background(), processcapture.Observation{At: at.Add(time.Duration(tick) * time.Second), Tick: tick, + Processes: []processcapture.Identity{{Key: "7@start", PID: 7, Name: "worker", FirstObservedAt: at}}, + Metrics: []metrics.Sample{{Entity: "7@start", Values: map[string]any{"rssBytes": uint64(tick * 1024), "memoryPercent": 2.0, "cpuPercent": 12.5}}}, + })).To(Succeed()) + } + refs := []read.Ref{{RunID: "header", Runtime: host.Runtime(), TraceRef: tracing.TraceRef{Kind: "process", Stream: "process-header"}}} + snapshot, err := read.Processes(context.Background(), refs) + Expect(err).NotTo(HaveOccurred()) + Expect(snapshot.Armed).To(BeTrue()) + Expect(snapshot.Rows).To(HaveExactElements(And(HaveField("PID", 7), HaveField("RSSBytes", HaveValue(Equal(uint64(501*1024)))), HaveField("Stream", "process-header")))) + }) +}) diff --git a/pkg/traces/read/read.go b/pkg/traces/read/read.go index 07dd81d..2b435c1 100644 --- a/pkg/traces/read/read.go +++ b/pkg/traces/read/read.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "github.com/flanksource/commons-db/query" "strings" "time" @@ -18,6 +19,7 @@ type Scope struct { Run string Runs []string Since time.Time + Result string } type Ref struct { @@ -27,10 +29,14 @@ type Ref struct { } type Selection struct { + Runs []telltale.RunInfo Refs []Ref closers []func() error } +// AddCloser transfers the lifetime of a constituent selection to this one. +func (s *Selection) AddCloser(close func() error) { s.closers = append(s.closers, close) } + func (s *Selection) Close() error { var errs []error for _, closeStore := range s.closers { @@ -40,6 +46,8 @@ func (s *Selection) Close() error { } type Query struct { + Sort string + Desc bool From, Until time.Time AfterSeq, ToSeq int64 Filters []string @@ -48,6 +56,9 @@ type Query struct { Cursor string All bool NextCursor func(string) `json:"-"` + seqOrder bool + order query.Order + profile *query.Profile } func Resolve(kind string, scope Scope) (_ *Selection, err error) { @@ -58,6 +69,14 @@ func Resolve(kind string, scope Scope) (_ *Selection, err error) { if _, err := traces.Lookup(kinds, kind); err != nil { return nil, err } + if scope.Result != "" { + storage := kind + "_" + scope.Result + owner, result := kinds.ResultOf(storage) + if owner != kind || result == "" { + return nil, fmt.Errorf("kind %s has no result %q", kind, scope.Result) + } + kind = storage + } if scope.Run != "" && len(scope.Runs) > 0 { return nil, fmt.Errorf("--run and --runs are mutually exclusive") } @@ -74,7 +93,8 @@ func Resolve(kind string, scope Scope) (_ *Selection, err error) { if !scope.Since.IsZero() { limit = 0 } - runs, err = telltale.List(telltale.ListOptions{DataDir: scope.DataDir, Kind: kind, Since: scope.Since, Limit: limit}) + owner, _ := kinds.ResultOf(kind) + runs, err = telltale.List(telltale.ListOptions{DataDir: scope.DataDir, Kind: owner, Since: scope.Since, Limit: limit}) if err != nil { return nil, err } @@ -105,25 +125,27 @@ func Resolve(kind string, scope Scope) (_ *Selection, err error) { } }() for _, run := range runs { - var refs []tracing.TraceRef - for _, capture := range run.Captures { - if capture.Kind == kind { - refs = append(refs, capture.Ref) - } - } - if len(refs) == 0 { - return nil, fmt.Errorf("run %s has no %s capture", run.RunID, kind) + if run.Unavailable != "" { + return nil, fmt.Errorf("read %s: %s", run.RunID, run.Unavailable) } runtime, closeStore, err := telltale.Reopen(scope.DataDir, run.RunID) if err != nil { return nil, err } selected.closers = append(selected.closers, closeStore) + refs, err := RunRefs(run, runtime) + if err != nil { + return nil, err + } + found := false for _, ref := range refs { - if err := ref.Validate(); err != nil { - return nil, err + if ref.Kind == kind { + selected.Refs = append(selected.Refs, ref) + found = true } - selected.Refs = append(selected.Refs, Ref{RunID: run.RunID, Runtime: runtime, TraceRef: ref}) + } + if !found { + return nil, fmt.Errorf("run %s has no %s capture", run.RunID, kind) } } return selected, nil diff --git a/pkg/traces/read/read_test.go b/pkg/traces/read/read_test.go index 88a1e94..6cde7c3 100644 --- a/pkg/traces/read/read_test.go +++ b/pkg/traces/read/read_test.go @@ -23,6 +23,7 @@ import ( func TestRead(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Trace read") } type reading struct { + Kind string `json:"kind,omitempty"` At time.Time `json:"at"` Value float64 `json:"value" filter:"range"` Label string `json:"label" filter:"text"` diff --git a/pkg/traces/read/refs.go b/pkg/traces/read/refs.go new file mode 100644 index 0000000..438a9de --- /dev/null +++ b/pkg/traces/read/refs.go @@ -0,0 +1,36 @@ +package read + +import ( + "fmt" + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/pkg/tracing" +) + +// RunRefs enumerates a run's captures and named tables, including its process trace. +func RunRefs(info telltale.RunInfo, runtime *tracing.Runtime) ([]Ref, error) { + refs := []Ref{} + seen := map[string]bool{} + primary := []tracing.TraceRef{} + for _, capture := range info.Captures { + primary = append(primary, capture.Ref) + } + if info.Process != nil { + primary = append(primary, info.Process.Tree, info.Process.Stacks) + } + for _, ref := range primary { + if _, exists := runtime.Kinds.Get(ref.Kind); !exists { + return nil, fmt.Errorf("run %s uses unsupported trace kind %q; recordings with process_tree, process_samples or process_groups require the build that recorded them", info.RunID, ref.Kind) + } + for _, result := range runtime.Kinds.CaptureRefs(ref) { + if err := result.Validate(); err != nil { + return nil, err + } + if seen[result.Stream] { + continue + } + seen[result.Stream] = true + refs = append(refs, Ref{RunID: info.RunID, Runtime: runtime, TraceRef: result}) + } + } + return refs, nil +} diff --git a/pkg/traces/read/request.go b/pkg/traces/read/request.go index a09e2cf..1028adc 100644 --- a/pkg/traces/read/request.go +++ b/pkg/traces/read/request.go @@ -25,8 +25,8 @@ func ParseRequest(dataDir string, values url.Values) (scope Scope, q Query, aggr if err != nil { return } - scope = Scope{DataDir: dataDir, Run: values.Get("run"), Runs: requestList(values["runs"])} - q = Query{Filters: values["filter"], Search: values.Get("q"), Where: values.Get("where"), Cursor: values.Get("cursor")} + scope = Scope{DataDir: dataDir, Run: values.Get("run"), Runs: requestList(values["runs"]), Result: values.Get("result")} + q = Query{Filters: values["filter"], Search: values.Get("q"), Where: values.Get("where"), Cursor: values.Get("cursor"), Sort: values.Get("sort")} aggregate = Aggregate{Columns: requestList(values["stats"]), GroupBy: values.Get("groupBy"), View: values.Get("view")} if err = parseRequestTimes(values, &scope, &q); err != nil { return @@ -47,7 +47,7 @@ func ParseRequest(dataDir string, values url.Values) (scope Scope, q Query, aggr return } } - for name, target := range map[string]*bool{"all": &q.All, "follow": &aggregate.Follow} { + for name, target := range map[string]*bool{"all": &q.All, "follow": &aggregate.Follow, "desc": &q.Desc} { if values.Has(name) { *target, err = strconv.ParseBool(values.Get(name)) if err != nil { @@ -132,7 +132,10 @@ func validateRequest(scope Scope, q Query, aggregate Aggregate, limitSet bool) e if aggregate.Follow && (aggregate.Enabled() || aggregate.View != "" || q.Cursor != "") { return fmt.Errorf("--follow cannot combine with --stats, --group-by, --view or --cursor") } - if aggregate.View != "" && (aggregate.Enabled() || !q.From.IsZero() || !q.Until.IsZero() || len(q.Filters) > 0 || q.Search != "" || q.Where != "" || q.Cursor != "" || limitSet) { + if aggregate.View == "top" && aggregate.Enabled() { + return fmt.Errorf("--view top cannot combine with --stats or --group-by") + } + if aggregate.View != "" && aggregate.View != "top" && (aggregate.Enabled() || !q.From.IsZero() || !q.Until.IsZero() || len(q.Filters) > 0 || q.Search != "" || q.Where != "" || q.Cursor != "" || limitSet) { return fmt.Errorf("--view accepts a seq window, without row filters, time bounds, paging or --stats/--group-by") } return nil diff --git a/pkg/traces/read/rows.go b/pkg/traces/read/rows.go index f72efb6..df39da2 100644 --- a/pkg/traces/read/rows.go +++ b/pkg/traces/read/rows.go @@ -154,6 +154,15 @@ func Rows(ctx context.Context, refs []Ref, q Query) iter.Seq2[query.Row, error] yield(nil, err) return } + if q.profile != nil { + p = *q.profile + } + if q.seqOrder { + p.Order = query.Order{{Column: "seq", Unique: true}} + } + if len(q.order) > 0 { + p.Order = q.order + } params, err := q.params(ref, p) if err != nil { yield(nil, err) @@ -168,7 +177,7 @@ func Rows(ctx context.Context, refs []Ref, q Query) iter.Seq2[query.Row, error] defer release() cursor := pos.Cursor for { - page := query.PageRequest{Limit: 500, Strategy: query.PagingCursor, Cursor: cursor} + page := query.PageRequest{Limit: 500, Strategy: query.PagingCursor, Cursor: cursor, Sort: q.Sort, Desc: q.Desc} if !q.All { page.Limit = min(remaining, 500) } diff --git a/pkg/traces/read/runs.go b/pkg/traces/read/runs.go new file mode 100644 index 0000000..e720c8c --- /dev/null +++ b/pkg/traces/read/runs.go @@ -0,0 +1,67 @@ +package read + +import ( + "errors" + "fmt" + + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/pkg/tracing" +) + +// ResolveRuns opens every armed capture of explicitly selected runs. +func ResolveRuns(scope Scope) (_ *Selection, err error) { + if scope.DataDir == "" || scope.Run == "" && len(scope.Runs) == 0 { + return nil, fmt.Errorf("replay requires a data dir and at least one run ID") + } + if scope.Run != "" && len(scope.Runs) > 0 { + return nil, fmt.Errorf("--run and --runs are mutually exclusive") + } + ids := scope.Runs + if scope.Run != "" { + ids = []string{scope.Run} + } + selected := &Selection{} + defer func() { + if err != nil { + err = errors.Join(err, selected.Close()) + } + }() + seen := map[string]bool{} + for _, id := range ids { + if seen[id] { + return nil, fmt.Errorf("run %q selected twice", id) + } + seen[id] = true + info, err := telltale.Get(telltale.GetOptions{DataDir: scope.DataDir, RunID: id}) + if err != nil { + return nil, err + } + if info.Unavailable != "" { + return nil, fmt.Errorf("replay %s: %s", id, info.Unavailable) + } + selected.Runs = append(selected.Runs, info) + } + for _, info := range selected.Runs { + run, live, err := telltale.Live(scope.DataDir, info.RunID) + if err != nil { + return nil, err + } + var runtime *tracing.Runtime + if live { + runtime = run.Runtime() + } else { + var closeStore func() error + runtime, closeStore, err = telltale.Reopen(scope.DataDir, info.RunID) + if err != nil { + return nil, err + } + selected.AddCloser(closeStore) + } + refs, err := RunRefs(info, runtime) + if err != nil { + return nil, err + } + selected.Refs = append(selected.Refs, refs...) + } + return selected, nil +} diff --git a/pkg/traces/read/top.go b/pkg/traces/read/top.go new file mode 100644 index 0000000..98c11e3 --- /dev/null +++ b/pkg/traces/read/top.go @@ -0,0 +1,266 @@ +package read + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "github.com/flanksource/clicky/api" + "slices" + "strings" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/flanksource/telltale/pkg/tracing" +) + +var ErrNoCompleteSamples = errors.New("complete samples unavailable; raw history is still available") + +type TopQuery struct { + Frame string `json:"frame,omitempty"` + Source string `json:"source,omitempty"` + Metric string `json:"metric,omitempty"` + Query +} + +type TopResult struct { + Selection string `json:"selection"` + Metrics []tracing.TopMetric `json:"metrics,omitempty"` + Frames []sampleframe.Frame `json:"frames"` + Display api.TextTable `json:"-"` + Hierarchy *sampleframe.Hierarchy `json:"hierarchy,omitempty"` + Metric string `json:"metric,omitempty"` + Frame sampleframe.Frame `json:"frame"` + Rows []query.Row `json:"rows"` + Table json.RawMessage `json:"table"` + Count int `json:"count"` + NextCursor string `json:"nextCursor,omitempty"` +} + +func Frames(ctx context.Context, ref Ref) ([]sampleframe.Frame, error) { + frames := []sampleframe.Frame{} + err := ref.Runtime.Results.Backend.Scan(ctx, ref.Stream+"-frames", 0, func(_ int64, row recordstore.Row) error { + data, err := json.Marshal(row) + if err != nil { + return err + } + var frame sampleframe.Frame + if err := json.Unmarshal(data, &frame); err != nil { + return err + } + if frame.Error == "" { + frames = append(frames, frame) + } + return nil + }) + if errors.Is(err, recordstore.ErrNotFound) || (err == nil && len(frames) == 0) { + return nil, ErrNoCompleteSamples + } + return frames, err +} + +func Top(ctx context.Context, ref Ref, q TopQuery) (TopResult, error) { + if err := q.Validate(); err != nil { + return TopResult{}, err + } + plugin, ok := ref.Runtime.Kinds.Get(ref.Kind) + if !ok || tracing.TopOf(plugin) == nil { + return TopResult{}, fmt.Errorf("kind %s does not support Top", ref.Kind) + } + spec := tracing.TopOf(plugin) + frames, err := Frames(ctx, ref) + if err != nil { + return TopResult{}, err + } + if spec.Result == "" { + frames = slices.DeleteFunc(frames, func(frame sampleframe.Frame) bool { + return frame.AfterSeq < ref.AfterSeq || ref.ToSeq > 0 && frame.ToSeq > ref.ToSeq + }) + } + requested := strings.Split(q.Frame, ",") + if len(requested) > 1 && !spec.PerSource { + return TopResult{}, fmt.Errorf("kind requires a single completed sample") + } + index := slices.IndexFunc(frames, func(f sampleframe.Frame) bool { + return f.ID == requested[0] && (q.Source == "" || f.Source == q.Source) + }) + if q.Frame == "" { + index = latestFrameIndex(frames, q.Source) + } + if index < 0 { + return TopResult{}, fmt.Errorf("sample %q for source %q not found", q.Frame, q.Source) + } + frame := frames[index] + if spec.Stack != nil { + data, err := json.Marshal(frame.Summary["metrics"]) + if err != nil { + return TopResult{}, err + } + copy := *spec + if err := json.Unmarshal(data, ©.Metrics); err != nil { + return TopResult{}, err + } + spec = © + if q.Metric == "" && len(spec.Metrics) > 0 { + q.Metric = spec.Metrics[0].Key + } + } + selectedFrames := []sampleframe.Frame{frame} + if len(requested) > 1 { + seen := map[string]bool{frame.Source: true} + for _, id := range requested[1:] { + i := slices.IndexFunc(frames, func(f sampleframe.Frame) bool { return f.ID == id && (q.Source == "" || q.Source == f.Source) }) + if i < 0 || seen[frames[i].Source] { + return TopResult{}, fmt.Errorf("invalid sample selection %q", q.Frame) + } + selectedFrames = append(selectedFrames, frames[i]) + seen[frames[i].Source] = true + } + } + if spec.PerSource && q.Source == "" && q.Frame == "" { + selectedFrames = nil + seen := map[string]bool{} + for i := len(frames) - 1; i >= 0; i-- { + if !seen[frames[i].Source] { + if latest := latestFrameIndex(frames, frames[i].Source); latest >= 0 { + selectedFrames = append(selectedFrames, frames[latest]) + } + seen[frames[i].Source] = true + } + } + } + window := ref + window.AfterSeq, window.ToSeq = frame.AfterSeq, frame.ToSeq + if spec.Result != "" { + window.Kind = ref.Kind + "_" + spec.Result + window.Stream = ref.Stream + "-" + spec.Result + } + selection := q.Query + selection.order = append(slices.Clone(spec.Order), query.OrderBy{Column: "seq", Unique: true}) + if spec.Hierarchy != nil && q.Sort == "" { + metric, err := selectMetric(nil, spec, q.Metric) + if err != nil { + return TopResult{}, err + } + selection.order = append(query.Order{{Column: metric, Desc: true}}, slices.DeleteFunc(selection.order, func(by query.OrderBy) bool { return by.Column == metric })...) + } + selection.All, selection.Cursor, selection.NextCursor = true, "", nil + rows := []query.Row{} + profile, err := sampleProfile(ctx, window, spec, frame) + if err != nil { + return TopResult{}, err + } + selection.profile = &profile + if spec.ProjectionColumn != "" { + selection.order = profile.Order + } + if _, err := q.params(window, profile); err != nil { + return TopResult{}, err + } + if _, err := profile.RequestedOrder(q.Sort, q.Desc); err != nil { + return TopResult{}, err + } + for _, sample := range selectedFrames { + if sample.Retained == 0 { + continue + } + member := window + member.AfterSeq, member.ToSeq = sample.AfterSeq, sample.ToSeq + part, err := query.CollectRows(Rows(ctx, []Ref{member}, selection)) + if err != nil { + return TopResult{}, err + } + rows = append(rows, part...) + } + if len(selectedFrames) > 1 { + order := selection.order + if q.Sort != "" { + order = append(query.Order{{Column: q.Sort, Desc: q.Desc}}, order...) + } + rows = query.SortRows(rows, order) + } + metric, err := selectMetric(rows, spec, q.Metric) + if err != nil { + return TopResult{}, err + } + if spec.Stack != nil { + rows = slices.DeleteFunc(rows, func(row query.Row) bool { return row[spec.Stack.MetricColumn] != metric }) + } + hierarchy, err := topHierarchy(rows, spec, metric) + if err != nil { + return TopResult{}, err + } + scope, err := cursorScope([]Ref{window}, selection) + if err != nil { + return TopResult{}, err + } + for _, sample := range selectedFrames { + scope += "/" + sample.ID + } + scope += "/" + metric + offset, err := topOffset(q.Cursor, scope) + if err != nil || offset > len(rows) { + return TopResult{}, fmt.Errorf("invalid sample cursor") + } + limit := q.Limit + if limit == 0 { + limit = 100 + } + end := offset + min(len(rows)-offset, limit) + if q.All { + end = len(rows) + } + ids := make([]string, len(selectedFrames)) + for i, sample := range selectedFrames { + ids[i] = sample.ID + } + result := TopResult{Selection: strings.Join(ids, ","), Metrics: spec.Metrics, Frame: frame, Frames: selectedFrames, Rows: rows[offset:end], Count: len(rows), Hierarchy: hierarchy, Metric: metric} + if end < len(rows) { + data, err := json.Marshal(topPosition{Scope: scope, Offset: end}) + if err != nil { + return TopResult{}, err + } + result.NextCursor = base64.RawURLEncoding.EncodeToString(data) + } + filterKeys, err := profile.ColumnFilterKeys() + if err != nil { + return TopResult{}, err + } + sortKeys, err := profile.ColumnSortKeys() + if err != nil { + return TopResult{}, err + } + display := &query.Result{Rows: result.Rows, Presenter: profile.Presenter, ColumnFilterKeys: filterKeys, ColumnSortKeys: sortKeys} + result.Display, err = display.DisplayTable(profile.Columns) + if err != nil { + return TopResult{}, err + } + document, err := display.Render(profile.Columns, "clicky-json") + result.Table = json.RawMessage(document) + return result, err +} + +type topPosition struct { + Scope string + Offset int +} + +func topOffset(cursor, scope string) (int, error) { + if cursor == "" { + return 0, nil + } + data, err := base64.RawURLEncoding.DecodeString(cursor) + if err != nil { + return 0, err + } + var pos topPosition + if err := json.Unmarshal(data, &pos); err != nil { + return 0, err + } + if pos.Scope != scope || pos.Offset < 0 { + return 0, fmt.Errorf("cursor belongs to another sample or filter") + } + return pos.Offset, nil +} diff --git a/pkg/traces/read/top_hierarchy.go b/pkg/traces/read/top_hierarchy.go new file mode 100644 index 0000000..80f9848 --- /dev/null +++ b/pkg/traces/read/top_hierarchy.go @@ -0,0 +1,107 @@ +package read + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/flanksource/telltale/pkg/tracing" +) + +func topHierarchy(rows []query.Row, spec *tracing.TopSpec, metric string) (*sampleframe.Hierarchy, error) { + if spec.Hierarchy != nil { + return spec.Hierarchy(rows, metric) + } + if spec.Stack == nil { + return nil, nil + } + if len(spec.Metrics) == 0 && metric == "" { + return nil, nil + } + index := slices.IndexFunc(spec.Metrics, func(m tracing.TopMetric) bool { return m.Key == metric }) + if index < 0 { + return nil, fmt.Errorf("unknown profile metric %q", metric) + } + stacks := make([]sampleframe.WeightedStack, 0, len(rows)) + for _, row := range rows { + data, err := json.Marshal(row["value"]) + if err != nil { + return nil, err + } + var value float64 + if err := json.Unmarshal(data, &value); err != nil { + return nil, fmt.Errorf("profile weight: %w", err) + } + frames, err := stackFrames(row[spec.Stack.Column], *spec.Stack) + if err != nil { + return nil, err + } + if spec.Stack.LeafFirst { + slices.Reverse(frames) + } + stacks = append(stacks, sampleframe.WeightedStack{Frames: frames, Value: value}) + } + root, err := sampleframe.BuildHierarchy(stacks) + if err != nil { + return nil, err + } + return &sampleframe.Hierarchy{Metric: metric, Unit: spec.Metrics[index].Unit, Root: root}, nil +} + +func stackFrames(raw any, spec tracing.TopStack) ([]string, error) { + if spec.Separator != "" { + stack, ok := raw.(string) + if !ok { + return nil, fmt.Errorf("profile stack must be a string, got %T", raw) + } + if stack == "" { + return nil, nil + } + return strings.Split(stack, spec.Separator), nil + } + data, err := json.Marshal(raw) + if err != nil { + return nil, err + } + var frames []map[string]any + if err := json.Unmarshal(data, &frames); err != nil { + return nil, err + } + names := make([]string, len(frames)) + for i, frame := range frames { + name, ok := frame[spec.FrameKey].(string) + if !ok || name == "" { + return nil, fmt.Errorf("profile frame has no %q", spec.FrameKey) + } + file, _ := frame["file"].(string) + if file != "" { + name += fmt.Sprintf(" (%s:%v)", file, frame["line"]) + } + names[i] = name + } + return names, nil +} + +func selectMetric(rows []query.Row, spec *tracing.TopSpec, requested string) (string, error) { + if spec.Stack == nil && spec.Hierarchy == nil { + if requested != "" { + return "", fmt.Errorf("kind does not offer profile metrics") + } + return "", nil + } + if len(spec.Metrics) == 0 && requested == "" { + return "", nil + } + for _, metric := range spec.Metrics { + if requested != "" && metric.Key != requested { + continue + } + if spec.Hierarchy != nil || requested != "" || len(rows) == 0 || slices.ContainsFunc(rows, func(row query.Row) bool { return row[spec.Stack.MetricColumn] == metric.Key }) { + return metric.Key, nil + } + } + return "", fmt.Errorf("profile metric %q unavailable in selected sample", requested) +} diff --git a/pkg/traces/read/top_profile.go b/pkg/traces/read/top_profile.go new file mode 100644 index 0000000..6952db3 --- /dev/null +++ b/pkg/traces/read/top_profile.go @@ -0,0 +1,73 @@ +package read + +import ( + "context" + "encoding/json" + "slices" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/flanksource/telltale/pkg/tracing" +) + +func sampleProfile(ctx context.Context, ref Ref, spec *tracing.TopSpec, frame sampleframe.Frame) (query.Profile, error) { + if spec.Profile != nil { + return spec.Profile(ctx, ref.Runtime, ref.TraceRef, frame) + } + _, profile, err := ref.profile(ctx, "") + if err != nil || spec.ProjectionColumn == "" { + return profile, err + } + data, err := json.Marshal(frame.Summary) + if err != nil { + return query.Profile{}, err + } + var metadata struct { + Columns []query.ColumnDef + Order query.Order + } + if err := json.Unmarshal(data, &metadata); err != nil { + return query.Profile{}, err + } + return recordresults.ProjectJSON(profile, spec.ProjectionColumn, metadata.Columns, metadata.Order) +} + +// TopProfile exposes the same resolved sample columns used by the Top reader. +func TopProfile(ctx context.Context, ref Ref, q TopQuery) (query.Profile, error) { + result, err := Top(ctx, ref, TopQuery{Frame: q.Frame, Source: q.Source, Metric: q.Metric, Query: Query{Limit: 1}}) + if err != nil { + return query.Profile{}, err + } + plugin, _ := ref.Runtime.Kinds.Get(ref.Kind) + spec := tracing.TopOf(plugin) + if spec.Result != "" { + ref.Kind += "_" + spec.Result + ref.Stream += "-" + spec.Result + } + profile, err := sampleProfile(ctx, ref, spec, result.Frame) + if err != nil { + return query.Profile{}, err + } + if profile.Presenter != nil { + columns := slices.Clone(profile.Columns) + profile.Columns = nil + for _, visible := range profile.Presenter.Columns() { + for i, column := range columns { + if column.Name == visible.Name { + column.Label = visible.Label + profile.Columns = append(profile.Columns, column) + columns = slices.Delete(columns, i, i+1) + break + } + } + } + profile.Columns = append(profile.Columns, columns...) + } + for i := range profile.Columns { + if profile.Columns[i].Name == "seq" { + profile.Columns[i].Hidden = true + } + } + return profile, nil +} diff --git a/pkg/traces/read/top_projection_test.go b/pkg/traces/read/top_projection_test.go new file mode 100644 index 0000000..fd88f5e --- /dev/null +++ b/pkg/traces/read/top_projection_test.go @@ -0,0 +1,50 @@ +package read_test + +import ( + "context" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/flanksource/telltale/pkg/traces/read" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/target" + "github.com/flanksource/telltale/pkg/tracing/tracingtest" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type capturedProfileRow struct { + Values map[string]any `json:"row"` +} +type capturedProfileFixture struct{} + +func (capturedProfileFixture) Params() struct{} { return struct{}{} } +func (capturedProfileFixture) Schema() recordresults.ResultType[capturedProfileRow] { + return recordresults.ResultType[capturedProfileRow]{Title: "Captured profile"} +} +func (capturedProfileFixture) Handle(ctx dbcontext.Context, _ struct{}, out tracing.Emitter[capturedProfileRow], _ tracing.Records[capturedProfileRow]) error { + at := time.Date(2026, 11, 11, 10, 0, 0, 0, time.UTC) + return tracing.PublishFrame(ctx, out, sampleframe.Frame{Source: "saved-profile", StartedAt: at, CompletedAt: at, Observed: 3, + Summary: map[string]any{"columns": []query.ColumnDef{{Name: "job", Type: query.ColumnTypeString}, {Name: "elapsed", Type: query.ColumnTypeDuration, Unit: "ms"}, {Name: "enabled", Type: query.ColumnTypeBoolean}}, "order": query.Order{{Column: "elapsed", Desc: true}}}}, + []capturedProfileRow{{Values: map[string]any{"job": "fast", "elapsed": 0, "enabled": false}}, {Values: map[string]any{"job": "slow", "elapsed": 1200, "enabled": true}}, {Values: map[string]any{"job": "medium", "elapsed": 500, "enabled": false}}}) +} + +var _ = Describe("Top captured profile columns", func() { + It("uses original duration and boolean filters before profile ordering and pagination", func() { + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{"sampled": tracing.NewHandler[struct{}, capturedProfileRow](capturedProfileFixture{}, tracing.Capabilities{Live: true}, target.Self).WithTop(tracing.TopSpec{ProjectionColumn: "row"})}) + info := tracingtest.Ended(env.Start("sampled", `{}`)) + Expect(info.Error).To(BeEmpty()) + ref := read.Ref{Runtime: env.Runtime, TraceRef: tracing.TraceRef{Kind: "sampled", Stream: info.Events.Stream}} + result, err := read.Top(context.Background(), ref, read.TopQuery{Query: read.Query{Filters: []string{"elapsed=<1s", "enabled=false"}, Limit: 1}}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Count).To(Equal(2)) + Expect(result.Rows[0]["job"]).To(Equal("medium")) + Expect(result.Rows[0]["enabled"]).To(Equal(false)) + Expect(result.Display.Columns[0].Name).To(Equal("job")) + Expect(result.Display.RowDetail).To(HaveLen(1)) + Expect(result.Display.RowDetail[0]).NotTo(BeNil()) + }) +}) diff --git a/pkg/traces/read/top_sequence.go b/pkg/traces/read/top_sequence.go new file mode 100644 index 0000000..86d6f64 --- /dev/null +++ b/pkg/traces/read/top_sequence.go @@ -0,0 +1,22 @@ +package read + +import "github.com/flanksource/commons-db/recordstore/sampleframe" + +func latestFrameIndex(frames []sampleframe.Frame, source string) int { + latest := -1 + for i := len(frames) - 1; i >= 0; i-- { + if complete, exists := frames[i].Summary["complete"].(bool); exists && !complete { + continue + } + if source != "" && frames[i].Source != source || latest >= 0 && frames[i].Source != frames[latest].Source { + continue + } + if latest < 0 { + latest = i + source = frames[i].Source + } else if frames[i].Tick > frames[latest].Tick { + latest = i + } + } + return latest +} diff --git a/pkg/traces/read/top_sequence_internal_test.go b/pkg/traces/read/top_sequence_internal_test.go new file mode 100644 index 0000000..484e3a7 --- /dev/null +++ b/pkg/traces/read/top_sequence_internal_test.go @@ -0,0 +1,14 @@ +package read + +import ( + "github.com/flanksource/commons-db/recordstore/sampleframe" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("latest producer frame", func() { + It("keeps an appended legacy frame with an empty source separate from other producers", func() { + frames := []sampleframe.Frame{{Source: "producer", Tick: 20}, {Source: "", Tick: 0}} + Expect(latestFrameIndex(frames, "")).To(Equal(1)) + }) +}) diff --git a/pkg/traces/read/top_sequence_test.go b/pkg/traces/read/top_sequence_test.go new file mode 100644 index 0000000..f6c2dfa --- /dev/null +++ b/pkg/traces/read/top_sequence_test.go @@ -0,0 +1,64 @@ +package read_test + +import ( + "context" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/flanksource/telltale/pkg/traces/read" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/target" + "github.com/flanksource/telltale/pkg/tracing/tracingtest" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type sequencedTopFixture struct{ topFixture } + +func (sequencedTopFixture) Handle(ctx dbcontext.Context, _ struct{}, out tracing.Emitter[topRow], _ tracing.Records[topRow]) error { + at := time.Date(2026, 11, 14, 10, 0, 0, 0, time.UTC) + for _, snapshot := range []struct { + source string + tick int64 + rows []topRow + complete bool + }{ + {"first", 2, nil, true}, + {"first", 1, []topRow{{ID: "obsolete-first", CPU: 10}}, true}, + {"second", 7, []topRow{{ID: "fresh-second", CPU: 20}}, true}, + {"second", 4, []topRow{{ID: "obsolete-second", CPU: 30}}, true}, + {"second", 9, nil, false}, + } { + frame := sampleframe.Frame{Source: snapshot.source, Tick: snapshot.tick, StartedAt: at, CompletedAt: at, Observed: len(snapshot.rows), Summary: map[string]any{"complete": snapshot.complete}} + if err := tracing.PublishFrame(ctx, out, frame, snapshot.rows); err != nil { + return err + } + } + return nil +} + +var _ = Describe("producer-sequenced Top reads", func() { + It("uses the newest complete producer snapshot per source despite reordered delivery", func() { + plugin := tracing.NewHandler[struct{}, topRow](sequencedTopFixture{}, tracing.Capabilities{Live: true}, target.Self).WithTop(tracing.TopSpec{PerSource: true, Identity: []string{"id"}, Order: query.Order{{Column: "cpu", Desc: true}}}) + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{"sampled": plugin}) + info := tracingtest.Ended(env.Start("sampled", `{}`)) + Expect(info.Error).To(BeEmpty()) + ref := read.Ref{Runtime: env.Runtime, TraceRef: tracing.TraceRef{Kind: "sampled", Stream: info.Events.Stream}} + result, err := read.Top(context.Background(), ref, read.TopQuery{}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Rows).To(ConsistOf(HaveKeyWithValue("id", "fresh-second"))) + Expect(result.Frames).To(ContainElements(And(HaveField("Source", "first"), HaveField("Tick", int64(2))), And(HaveField("Source", "second"), HaveField("Tick", int64(7))))) + first, err := read.Top(context.Background(), ref, read.TopQuery{Source: "first"}) + Expect(err).NotTo(HaveOccurred()) + Expect(first.Rows).To(BeEmpty()) + Expect(first.Frame.Tick).To(Equal(int64(2))) + frames, err := read.Frames(context.Background(), ref) + Expect(err).NotTo(HaveOccurred()) + Expect(frames).To(HaveLen(5)) + history, err := read.Top(context.Background(), ref, read.TopQuery{Frame: frames[1].ID}) + Expect(err).NotTo(HaveOccurred()) + Expect(history.Rows).To(ConsistOf(HaveKeyWithValue("id", "obsolete-first"))) + }) +}) diff --git a/pkg/traces/read/top_test.go b/pkg/traces/read/top_test.go new file mode 100644 index 0000000..7616ec0 --- /dev/null +++ b/pkg/traces/read/top_test.go @@ -0,0 +1,124 @@ +package read_test + +import ( + "context" + "fmt" + "strings" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/flanksource/telltale/pkg/traces/read" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/target" + "github.com/flanksource/telltale/pkg/tracing/tracingtest" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type topRow struct { + ID string `json:"id" filter:"terms" sort:"id"` + CPU float64 `json:"cpu" filter:"range" sort:"cpu"` +} + +type topFixture struct{ empty bool } + +type multiTopFixture struct{ topFixture } + +func (multiTopFixture) Handle(ctx dbcontext.Context, _ struct{}, out tracing.Emitter[topRow], _ tracing.Records[topRow]) error { + at := time.Date(2026, 11, 11, 10, 0, 0, 0, time.UTC) + for i, source := range []string{"first", "second", "first", "third"} { + rows := []topRow{{ID: source, CPU: float64(i + 1)}} + if i == 2 { + rows = nil + } + if err := tracing.PublishFrame(ctx, out, sampleframe.Frame{Source: source, StartedAt: at, CompletedAt: at, Observed: len(rows)}, rows); err != nil { + return err + } + } + return nil +} + +func (topFixture) Params() struct{} { return struct{}{} } +func (topFixture) Schema() recordresults.ResultType[topRow] { + return recordresults.ResultType[topRow]{Title: "Sample fixture"} +} +func (f topFixture) Handle(ctx dbcontext.Context, _ struct{}, out tracing.Emitter[topRow], _ tracing.Records[topRow]) error { + at := time.Date(2026, 11, 11, 10, 0, 0, 0, time.UTC) + for sample := range 2 { + rows := []topRow{} + if sample == 0 || !f.empty { + for i := range 150 { + rows = append(rows, topRow{ID: fmt.Sprintf("worker-%03d", i), CPU: float64(i + sample*150)}) + } + } + if err := tracing.PublishFrame(ctx, out, sampleframe.Frame{Source: "test", StartedAt: at, CompletedAt: at, Observed: len(rows)}, rows); err != nil { + return err + } + } + return tracing.PublishFrame(ctx, out, sampleframe.Frame{Source: "test", StartedAt: at, CompletedAt: at, Observed: 1, Error: "capture interrupted"}, []topRow{{ID: "failed", CPU: 1000}}) +} + +func topRef(empty bool) read.Ref { + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{"sampled": tracing.NewHandler[struct{}, topRow](topFixture{empty: empty}, tracing.Capabilities{Live: true}, target.Self).WithTop(tracing.TopSpec{Identity: []string{"id"}, Order: query.Order{{Column: "cpu", Desc: true}}})}) + info := tracingtest.Ended(env.Start("sampled", `{}`)) + Expect(info.Error).To(BeEmpty()) + return read.Ref{Runtime: env.Runtime, TraceRef: tracing.TraceRef{Kind: "sampled", Stream: info.Events.Stream}} +} + +var _ = Describe("Top reads", func() { + It("freezes all endpoint samples and respects the latest empty scrape for each source", func() { + plugin := tracing.NewHandler[struct{}, topRow](multiTopFixture{}, tracing.Capabilities{Live: true}, target.Self).WithTop(tracing.TopSpec{PerSource: true, Identity: []string{"id"}, Order: query.Order{{Column: "cpu", Desc: true}}}) + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{"sampled": plugin}) + info := tracingtest.Ended(env.Start("sampled", `{}`)) + Expect(info.Error).To(BeEmpty()) + ref := read.Ref{Runtime: env.Runtime, TraceRef: tracing.TraceRef{Kind: "sampled", Stream: info.Events.Stream}} + frames, err := read.Frames(context.Background(), ref) + Expect(err).NotTo(HaveOccurred()) + latest, err := read.Top(context.Background(), ref, read.TopQuery{}) + Expect(err).NotTo(HaveOccurred()) + Expect(latest.Rows).To(HaveLen(2)) + Expect(latest.Rows).NotTo(ContainElement(HaveKeyWithValue("id", "first"))) + frozen, err := read.Top(context.Background(), ref, read.TopQuery{Frame: strings.Join([]string{frames[0].ID, frames[1].ID}, ",")}) + Expect(err).NotTo(HaveOccurred()) + Expect(frozen.Rows).To(ConsistOf(HaveKeyWithValue("id", "first"), HaveKeyWithValue("id", "second"))) + }) + It("pins pagination to one sample and rejects a cursor reused with different filters", func() { + ref := topRef(false) + first, err := read.Top(context.Background(), ref, read.TopQuery{Query: read.Query{Limit: 100}}) + Expect(err).NotTo(HaveOccurred()) + second, err := read.Top(context.Background(), ref, read.TopQuery{Frame: first.Frame.ID, Query: read.Query{Limit: 100, Cursor: first.NextCursor}}) + Expect(err).NotTo(HaveOccurred()) + Expect(second.Rows).To(HaveLen(50)) + Expect(second.Rows[0]["cpu"]).To(BeNumerically("==", 199)) + _, err = read.Top(context.Background(), ref, read.TopQuery{Frame: first.Frame.ID, Query: read.Query{Cursor: first.NextCursor, Filters: []string{"cpu=>200"}}}) + Expect(err).To(HaveOccurred()) + }) + It("rejects unknown filters and ordering even for a successful empty sample", func() { + ref := topRef(true) + _, err := read.Top(context.Background(), ref, read.TopQuery{Query: read.Query{Filters: []string{"unknown=1"}}}) + Expect(err).To(HaveOccurred()) + _, err = read.Top(context.Background(), ref, read.TopQuery{Query: read.Query{Sort: "unknown"}}) + Expect(err).To(HaveOccurred()) + }) + It("selects the latest complete sample before filtering, ordering and pagination", func() { + ref := topRef(false) + result, err := read.Top(context.Background(), ref, read.TopQuery{Query: read.Query{Limit: 2}}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Rows).To(HaveLen(2)) + Expect(result.Rows[0]["cpu"]).To(BeNumerically("==", 299)) + Expect(result.Count).To(Equal(150)) + result, err = read.Top(context.Background(), ref, read.TopQuery{Query: read.Query{Filters: []string{"cpu=<100"}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Rows).To(BeEmpty()) + Expect(result.Frame.Retained).To(Equal(150)) + }) + It("returns a successful empty latest sample instead of resurrecting old rows", func() { + result, err := read.Top(context.Background(), topRef(true), read.TopQuery{}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Rows).To(BeEmpty()) + Expect(result.Frame.Retained).To(BeZero()) + }) +}) diff --git a/pkg/traces/registry.go b/pkg/traces/registry.go index d016751..3f74047 100644 --- a/pkg/traces/registry.go +++ b/pkg/traces/registry.go @@ -55,6 +55,9 @@ type Registration struct { Description string Seams []target.Seam DefaultSeam target.Seam + // Base names the kind a seam preset (tracing.Handler.Preset) arms on its + // one seam; empty for a kind of its own. + Base string // Exports are the environment variables a capture hands the command. Exports []string Examples []Example @@ -69,6 +72,20 @@ type Registration struct { // process whose gops agent a gops reading needs; nil reads any. Readable func(pid int, checkpoint json.RawMessage) error New func(KindDeps) tracing.TracePlugin + // Commands are the task commands the kind is also served as beside + // `run `, each with its own param defaults. + Commands []KindCommand +} + +// KindCommand is a kind preset served as its own CLI command and API path, +// such as `jvm watch` for jvm_trace with values recorded and calls not. +type KindCommand struct { + // Path is the command's words from the root, such as [jvm watch]. + Path []string + Summary string + // Defaults replace the kind's param defaults, nested by JSON path; flags + // still override them. + Defaults map[string]any } func (r Registration) validate() error { @@ -87,9 +104,17 @@ func (r Registration) validate() error { if !slices.Contains(r.Seams, r.DefaultSeam) { errs = append(errs, fmt.Errorf("default seam %q must be among supported seams %v", r.DefaultSeam, r.Seams)) } + if r.Base != "" && len(r.Seams) != 1 { + errs = append(errs, fmt.Errorf("a preset of %q arms exactly one seam, not %v", r.Base, r.Seams)) + } if strings.TrimSpace(r.Title) == "" || strings.TrimSpace(r.Summary) == "" { errs = append(errs, errors.New("a title and a summary are required")) } + for _, command := range r.Commands { + if len(command.Path) < 2 || slices.Contains(command.Path, "") || strings.TrimSpace(command.Summary) == "" { + errs = append(errs, fmt.Errorf("command %v needs a path below the root and a summary", command.Path)) + } + } if err := errors.Join(errs...); err != nil { return fmt.Errorf("register trace kind %q: %w", r.Name, err) } diff --git a/pkg/traces/registry_test.go b/pkg/traces/registry_test.go index 68f71dd..e163ca1 100644 --- a/pkg/traces/registry_test.go +++ b/pkg/traces/registry_test.go @@ -21,7 +21,7 @@ func registrationNames() []string { } var _ = Describe("the trace kind registry", func() { - builtins := []string{"goprof", "http", "process_samples", "process_tree", "sql", "stacks", "terminal"} + builtins := []string{"goprof", "http", "http_proxy", "pg_proxy", "process", "sql", "stacks", "terminal"} valid := func(name string) traces.Registration { return traces.Registration{ Name: name, Title: "Registered for a test", Summary: "Records nothing", Seams: []target.Seam{target.Self}, DefaultSeam: target.Self, @@ -43,13 +43,16 @@ var _ = Describe("the trace kind registry", func() { It("registers the supported seam and default for each builtin", func() { for kind, seam := range map[string]target.Seam{ - "process_tree": target.Process, "process_samples": target.Process, - "stacks": target.Process, "goprof": target.Process, - "process_groups": target.Host, "http": target.Self, "sql": target.Self, "terminal": target.Self, + "process": target.Process, + "stacks": target.Process, "goprof": target.Process, + "http": target.Self, "sql": target.Self, "terminal": target.Self, } { registration, err := traces.Registered(kind) Expect(err).NotTo(HaveOccurred()) seams := []target.Seam{seam} + if kind == "process" { + seams = append(seams, target.Host) + } if kind == "stacks" || kind == "goprof" { seams = append(seams, target.Agent, target.Self) } @@ -60,10 +63,32 @@ var _ = Describe("the trace kind registry", func() { Expect(registration.DefaultSeam).To(Equal(seam), kind) } }) + It("serves http_proxy and pg_proxy as their base kinds armed only on the proxy seam", func() { + kinds, err := traces.Kinds() + Expect(err).NotTo(HaveOccurred()) + infos, err := kinds.List() + Expect(err).NotTo(HaveOccurred()) + columns := map[string]any{} + for _, info := range infos { + columns[info.Name] = info.Columns + } + for preset, base := range map[string]string{"http_proxy": "http", "pg_proxy": "sql"} { + registration, err := traces.Registered(preset) + Expect(err).NotTo(HaveOccurred()) + Expect(registration.Base).To(Equal(base), preset) + Expect(registration.Seams).To(Equal([]target.Seam{target.Proxy}), preset) + Expect(registration.DefaultSeam).To(Equal(target.Proxy), preset) + Expect(registration.Armable).To(BeTrue(), preset) + Expect(columns[preset]).To(Equal(columns[base]), preset) + Expect(traces.Validate(kinds, preset, []byte(`{}`))).To(Succeed(), preset) + Expect(traces.Validate(kinds, preset, []byte(`{"target":{"seam":"self"}}`))).To(MatchError(ContainSubstring(`arms seam "proxy"`)), preset) + } + }) + It("rejects an unsupported seam before starting a capture", func() { kinds, err := traces.Kinds() Expect(err).NotTo(HaveOccurred()) - err = traces.Validate(kinds, "process_tree", []byte(`{"target":{"seam":"proxy"}}`)) + err = traces.Validate(kinds, "process", []byte(`{"target":{"seam":"proxy"}}`)) Expect(err).To(MatchError(ContainSubstring("unsupported seam"))) Expect(err).To(MatchError(ContainSubstring("process"))) }) @@ -94,6 +119,9 @@ var _ = Describe("the trace kind registry", func() { Entry("no plugin", func(r *traces.Registration) { r.New = nil }, "a New func is required"), Entry("an unknown seam", func(r *traces.Registration) { r.Seams = []target.Seam{"elsewhere"} }, `unknown seam "elsewhere"`), Entry("no summary", func(r *traces.Registration) { r.Summary = "" }, "a title and a summary are required"), + Entry("a preset arming more than one seam", func(r *traces.Registration) { + r.Base, r.Seams = "http", []target.Seam{target.Self, target.Proxy} + }, `a preset of "http" arms exactly one seam`), Entry("a builtin's name", func(r *traces.Registration) { r.Name = "http" }, `"http": it is registered twice`), ) diff --git a/pkg/traces/runid.go b/pkg/traces/runid.go new file mode 100644 index 0000000..88901d5 --- /dev/null +++ b/pkg/traces/runid.go @@ -0,0 +1,44 @@ +package traces + +import ( + "fmt" + "os" + "regexp" + "strings" + "time" +) + +const RunTimestampLayout = "20060102T150405Z" + +const ManifestFile = "run.json" + +var runIDPattern = regexp.MustCompile(`^[A-Za-z0-9]{1,9}[0-9]{8}T[0-9]{6}Z(?:_[2-9][0-9]*|_1[0-9]+)?$`) + +// IsRunID identifies compact generated IDs, including collision suffixes. +func IsRunID(id string) bool { return runIDPattern.MatchString(id) } + +// RunID names a run by its short hostname and UTC start time. +func RunID(started time.Time) string { + hostname, err := os.Hostname() + if err != nil { + panic(fmt.Errorf("name trace run: hostname: %w", err)) + } + prefix, err := hostnamePrefix(hostname) + if err != nil { + panic(err) + } + return prefix + started.UTC().Format(RunTimestampLayout) +} + +func hostnamePrefix(hostname string) (string, error) { + prefix := strings.ReplaceAll(strings.Split(hostname, ".")[0], "-", "") + if prefix == "" { + return "", fmt.Errorf("name trace run: hostname %q has no short name", hostname) + } + for _, char := range prefix { + if (char < 'a' || char > 'z') && (char < 'A' || char > 'Z') && (char < '0' || char > '9') { + return "", fmt.Errorf("name trace run: hostname %q contains an invalid character", hostname) + } + } + return prefix[:min(9, len(prefix))], nil +} diff --git a/pkg/traces/runid_test.go b/pkg/traces/runid_test.go new file mode 100644 index 0000000..1bf4536 --- /dev/null +++ b/pkg/traces/runid_test.go @@ -0,0 +1,23 @@ +package traces + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("run hostname prefix", func() { + DescribeTable("uses up to nine dash-free characters of the short hostname", func(hostname, expected string) { + prefix, err := hostnamePrefix(hostname) + Expect(err).NotTo(HaveOccurred()) + Expect(prefix).To(Equal(expected)) + }, + Entry("short", "dev", "dev"), + Entry("long", "buildworker12", "buildwork"), + Entry("qualified", "worker.example.com", "worker"), + Entry("dashes", "build-worker-12.local", "buildwork"), + ) + DescribeTable("refuses unusable hostnames", func(hostname string) { + _, err := hostnamePrefix(hostname) + Expect(err).To(HaveOccurred()) + }, Entry("empty", ""), Entry("dashes alone", "---"), Entry("path", "../escape")) +}) diff --git a/pkg/traces/testdata/params/logs.json b/pkg/traces/testdata/params/logs.json index 3cfd3f0..c75b182 100644 --- a/pkg/traces/testdata/params/logs.json +++ b/pkg/traces/testdata/params/logs.json @@ -5,7 +5,12 @@ "type": "string" }, "connection": { - "type": "string" + "type": "string", + "x-clicky-lookup": { + "filter": "connection", + "searchParam": "__lookup_q", + "url": "/api/v1/connection" + } }, "container": { "type": "string" @@ -62,11 +67,11 @@ "seam": { "type": "string", "title": "Seam", - "description": "How the capture is armed (supported seams: [self agent process])", + "description": "How the capture is armed (supported seams: [self agent host])", "enum": [ "self", "agent", - "process" + "host" ], "default": "self" }, @@ -88,7 +93,7 @@ "x-telltale-seams": [ "self", "agent", - "process" + "host" ] }, "timeField": { diff --git a/pkg/traces/testdata/params/spans.json b/pkg/traces/testdata/params/spans.json index 327bd78..007de8d 100644 --- a/pkg/traces/testdata/params/spans.json +++ b/pkg/traces/testdata/params/spans.json @@ -45,10 +45,7 @@ "properties": { "aggregations": { "type": "object", - "additionalProperties": { - "type": "string", - "format": "byte" - } + "additionalProperties": {} }, "fields": { "type": "array", diff --git a/pkg/traces/testdata/params/sql.json b/pkg/traces/testdata/params/sql.json index d6ef7be..413bc78 100644 --- a/pkg/traces/testdata/params/sql.json +++ b/pkg/traces/testdata/params/sql.json @@ -2,10 +2,15 @@ "type": "object", "properties": { "connection": { - "type": "string", - "title": "Connection label", + "default": "proxy", "description": "Label the statements are recorded under", - "default": "proxy" + "title": "Connection label", + "type": "string", + "x-clicky-lookup": { + "filter": "connection", + "searchParam": "__lookup_q", + "url": "/api/v1/connection" + } }, "connections": { "type": "array", diff --git a/pkg/traces/testdata/params/sql_deadlock.json b/pkg/traces/testdata/params/sql_deadlock.json index 038609d..ae31a11 100644 --- a/pkg/traces/testdata/params/sql_deadlock.json +++ b/pkg/traces/testdata/params/sql_deadlock.json @@ -2,8 +2,13 @@ "type": "object", "properties": { "connection": { + "title": "Connection", "type": "string", - "title": "Connection" + "x-clicky-lookup": { + "filter": "connection", + "searchParam": "__lookup_q", + "url": "/api/v1/connection" + } }, "databases": { "type": "array", diff --git a/pkg/traces/testdata/params/sql_plancache.json b/pkg/traces/testdata/params/sql_plancache.json index 0f2d956..7605a27 100644 --- a/pkg/traces/testdata/params/sql_plancache.json +++ b/pkg/traces/testdata/params/sql_plancache.json @@ -2,9 +2,14 @@ "type": "object", "properties": { "connection": { - "type": "string", + "description": "SQL Server connection reference", "title": "Connection", - "description": "SQL Server connection reference" + "type": "string", + "x-clicky-lookup": { + "filter": "connection", + "searchParam": "__lookup_q", + "url": "/api/v1/connection?types=sql_server" + } }, "databases": { "type": "array", diff --git a/pkg/traces/testdata/params/sql_top.json b/pkg/traces/testdata/params/sql_top.json new file mode 100644 index 0000000..6ba4a6e --- /dev/null +++ b/pkg/traces/testdata/params/sql_top.json @@ -0,0 +1,102 @@ +{ + "type": "object", + "properties": { + "connection": { + "description": "Postgres or SQL Server connection reference for the database seam", + "title": "Connection", + "type": "string", + "x-clicky-lookup": { + "filter": "connection", + "searchParam": "__lookup_q", + "url": "/api/v1/connection?types=postgres,sql_server" + } + }, + "databases": { + "type": "array", + "title": "Databases", + "description": "Database patterns with * wildcards and ! exclusions; empty keeps every database", + "items": { + "type": "string" + } + }, + "idle": { + "type": "boolean", + "title": "Idle sessions", + "description": "Also record idle sessions; sessions holding an open transaction are always recorded" + }, + "minElapsed": { + "type": "string", + "title": "Minimum elapsed", + "description": "Only record requests running at least this long such as 500ms" + }, + "poll": { + "type": "string", + "title": "Poll", + "description": "Poll interval such as 5s", + "default": "5s" + }, + "target": { + "properties": { + "agent": { + "type": "string", + "title": "Agent", + "description": "Agent ID to observe" + }, + "allowMutation": { + "type": "boolean", + "title": "Allow mutation", + "description": "Allow probes to change runtime settings" + }, + "pid": { + "type": "integer", + "title": "PID", + "format": "int32", + "description": "Process to observe" + }, + "seam": { + "type": "string", + "title": "Seam", + "description": "How the capture is armed (supported seams: [connection agent])", + "enum": [ + "connection", + "agent" + ], + "default": "connection" + }, + "selector": { + "type": "object", + "title": "Selector", + "description": "Agent resource attributes to match", + "additionalProperties": { + "type": "string" + } + }, + "wait": { + "type": "boolean", + "title": "Wait for agents", + "description": "Wait for a matching agent to connect" + } + }, + "type": "object", + "x-telltale-seams": [ + "connection", + "agent" + ] + }, + "text": { + "type": "array", + "title": "Statement text", + "description": "SQL text patterns with * wildcards and ! exclusions; bare values match substrings", + "items": { + "type": "string" + } + }, + "top": { + "type": "integer", + "title": "Top", + "format": "int32", + "description": "Maximum requests per poll ranked by elapsed time; defaults to 50", + "default": 50 + } + } +} diff --git a/pkg/traces/testdata/params/sql_xevent.json b/pkg/traces/testdata/params/sql_xevent.json index c24996e..078129e 100644 --- a/pkg/traces/testdata/params/sql_xevent.json +++ b/pkg/traces/testdata/params/sql_xevent.json @@ -8,8 +8,13 @@ } }, "connection": { + "title": "Connection", "type": "string", - "title": "Connection" + "x-clicky-lookup": { + "filter": "connection", + "searchParam": "__lookup_q", + "url": "/api/v1/connection" + } }, "databases": { "type": "array", diff --git a/pkg/traces/top_contract_test.go b/pkg/traces/top_contract_test.go new file mode 100644 index 0000000..77f4c4e --- /dev/null +++ b/pkg/traces/top_contract_test.go @@ -0,0 +1,42 @@ +package traces_test + +import ( + "context" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/commons-db/recordstore/recordresults/recordresultstest" + "github.com/flanksource/telltale/pkg/traces" + "github.com/flanksource/telltale/pkg/tracing" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("catalog table contracts", func() { + It("advertises Top for all eight sampled kinds and validates every table's native bindings", func() { + kinds, err := traces.Kinds() + Expect(err).NotTo(HaveOccurred()) + for _, name := range []string{"sql_top", "sql_plancache", "process", "goprof", "jvm_jfr", "prometheus", "stacks", "query"} { + plugin, ok := kinds.Get(name) + Expect(ok).To(BeTrue(), name) + Expect(tracing.TopOf(plugin)).NotTo(BeNil(), name) + } + results := recordresultstest.OpenResults(recordresults.OpenOptions{Prefix: "traces", ConnectionName: "traces", Settings: recordresultstest.LocalSettings(recordstore.BackendSQLite), Register: kinds.RegisterResultTypes}) + for _, registered := range results.Registry.ResultTypes() { + profile, err := results.Registry.Get(context.Background(), registered.Profile) + Expect(err).NotTo(HaveOccurred(), registered.Kind) + columns, err := query.ClickyColumns(profile) + Expect(err).NotTo(HaveOccurred(), registered.Kind) + for _, column := range columns { + if column.FilterKey != "" { + _, err := query.ResolveColumnFilters(profile, map[string]any{}) + Expect(err).NotTo(HaveOccurred(), registered.Kind+"/"+column.Name) + } + if column.SortKey != "" { + _, err := profile.RequestedOrder(column.SortKey, true) + Expect(err).NotTo(HaveOccurred(), registered.Kind+"/"+column.Name) + } + } + } + }) +}) diff --git a/pkg/traceserve/handler.go b/pkg/traceserve/handler.go index e29931c..4b5a70b 100644 --- a/pkg/traceserve/handler.go +++ b/pkg/traceserve/handler.go @@ -1,203 +1,35 @@ package traceserve import ( - "encoding/json" "errors" - "fmt" - "io" "net/http" - "net/url" - "strconv" - "github.com/flanksource/commons-db/recordstore" + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query/profilestore" + "github.com/flanksource/telltale/pkg/sessions" "github.com/flanksource/telltale/pkg/tracing" ) -// maxCheckpointBody bounds a checkpoint's params, which can carry a stack -// dumped elsewhere. -const maxCheckpointBody = 8 << 20 - -// Handler serves runtime's store, rooted at the path the host mounts it at -// (strip that prefix before it): -// -// GET /api/v1/traces/kinds the kinds served, with their export formats -// GET /api/v1/traces/{kind}/seq the seq a stream committed through (?stream) -// GET /api/v1/traces/{kind}/export a window downloaded (?stream, ?afterSeq, ?toSeq, ?format) -// POST /api/v1/sessions/{id}/checkpoint a running capture's reading, committed (JSON body: the kind's checkpoint params) -// -// Every other path is 404. +// Handler serves runtime's store as the sessions API in its read-only mode +// (sessions.ReadOnly), rooted at the path the host mounts it at (strip that +// prefix before it): the trace kinds, a stream's seq, window exports and the +// sessions of runtime's registry, each listed, read, followed and +// checkpointed. The host starts and stops the captures itself, so every start, +// stop, extend and restart is refused 403. Every other path is 404. func Handler(runtime *tracing.Runtime) (http.Handler, error) { - if runtime == nil || runtime.Results == nil { - return nil, errors.New("serve trace store: an open runtime is required") - } - h := handler{runtime: runtime} - mux := http.NewServeMux() - mux.HandleFunc("GET "+APIPrefix+"/traces/kinds", h.kinds) - mux.HandleFunc("GET "+APIPrefix+"/traces/{kind}/seq", h.seq) - mux.HandleFunc("GET "+APIPrefix+"/traces/{kind}/export", h.export) - mux.HandleFunc("POST "+APIPrefix+"/sessions/{id}/checkpoint", h.checkpoint) - return mux, nil -} - -type handler struct { - runtime *tracing.Runtime -} - -func (h handler) kinds(w http.ResponseWriter, _ *http.Request) { - kinds, err := h.runtime.Kinds.List() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - writeJSON(w, kinds) -} - -// seq answers {"seq": n}, the seq ?stream of kind has committed through: 0 for -// one that never committed a row. -func (h handler) seq(w http.ResponseWriter, r *http.Request) { - kind := r.PathValue("kind") - if _, ok := h.runtime.Kinds.Get(kind); !ok { - http.Error(w, fmt.Sprintf("%v: %s", tracing.ErrUnknownKind, kind), http.StatusNotFound) - return - } - stream := r.URL.Query().Get("stream") - if stream == "" { - http.Error(w, "stream is required", http.StatusBadRequest) - return - } - var seq int64 - meta, err := h.runtime.Results.Backend.Meta(r.Context(), stream) - switch { - case errors.Is(err, recordstore.ErrNotFound): - case err != nil: - http.Error(w, err.Error(), http.StatusInternalServerError) - return - default: - seq = meta.HighSeq - } - writeJSON(w, map[string]int64{"seq": seq}) -} - -// export downloads the window ?stream, ?afterSeq, ?toSeq of kind in ?format. -// A refusal found before the first byte is answered with its status; a failure -// once the download has begun aborts it, so a client never keeps a file cut -// short. -func (h handler) export(w http.ResponseWriter, r *http.Request) { - kind := r.PathValue("kind") - query := r.URL.Query() - format, err := h.runtime.Kinds.Format(kind, query.Get("format")) + if runtime == nil || runtime.Results == nil || runtime.Sessions == nil { + return nil, errors.New("serve trace store: an open runtime with a session registry is required") + } + service, err := sessions.New(sessions.Options{ + Profiles: func() (profilestore.Store, error) { return runtime.Results.Registry, nil }, + Context: func() dbcontext.Context { return dbcontext.New() }, + Registry: runtime.Sessions, + Records: runtime.Results.Backend, + Traces: runtime, + Authorize: sessions.ReadOnly, + }) if err != nil { - http.Error(w, err.Error(), exportErrorStatus(err)) - return - } - ref, err := traceRef(kind, query) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - download := &attachment{w: w, format: format, filename: fmt.Sprintf("%s-%s.%s", kind, ref.Stream, format.Ext)} - err = h.runtime.Export(r.Context(), ref, format.Name, download) - switch { - case err != nil && download.started: - panic(http.ErrAbortHandler) - case err != nil: - http.Error(w, err.Error(), exportErrorStatus(err)) - default: - // An empty export is still answered as a download; an empty write cannot fail. - _, _ = download.Write(nil) - } -} - -// checkpoint has the running capture of session id take a reading, with the -// params its JSON body gives the kind, and commit what it emitted. A session -// that is no running capture of this runtime is a conflict; params its kind -// refuses or does not read are a bad request. -func (h handler) checkpoint(w http.ResponseWriter, r *http.Request) { - id := r.PathValue("id") - params, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxCheckpointBody)) - if err != nil { - http.Error(w, fmt.Sprintf("checkpoint session %s: read params: %v", id, err), http.StatusBadRequest) - return - } - if err := h.runtime.CheckpointSession(tracing.WithCheckpointParams(r.Context(), params), id); err != nil { - status := http.StatusInternalServerError - switch { - case errors.Is(err, tracing.ErrNoCapture): - status = http.StatusConflict - case errors.Is(err, tracing.ErrCheckpointParams): - status = http.StatusBadRequest - } - http.Error(w, err.Error(), status) - return - } - w.WriteHeader(http.StatusNoContent) -} - -// traceRef is the window query names of kind's stream. -func traceRef(kind string, query url.Values) (tracing.TraceRef, error) { - ref := tracing.TraceRef{Kind: kind, Stream: query.Get("stream")} - if ref.Stream == "" { - return ref, errors.New("stream is required") - } - for name, seq := range map[string]*int64{"afterSeq": &ref.AfterSeq, "toSeq": &ref.ToSeq} { - if raw := query.Get(name); raw != "" { - parsed, err := strconv.ParseInt(raw, 10, 64) - if err != nil { - return ref, fmt.Errorf("%s %q is not a seq", name, raw) - } - *seq = parsed - } - } - return ref, ref.Validate() -} - -func exportErrorStatus(err error) int { - switch { - case errors.Is(err, tracing.ErrUnknownKind), errors.Is(err, recordstore.ErrNotFound): - return http.StatusNotFound - case errors.Is(err, tracing.ErrUnknownFormat), errors.Is(err, tracing.ErrInvalidRef): - return http.StatusBadRequest - } - return http.StatusInternalServerError -} - -func writeJSON(w http.ResponseWriter, value any) { - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(value); err != nil { - panic(http.ErrAbortHandler) - } -} - -// attachment writes a download's headers with its first byte, so an export -// refused before it writes anything is still answered with its error. -type attachment struct { - w http.ResponseWriter - format tracing.ExportFormat - filename string - started bool -} - -// Write sets the download headers on the same writer it writes the body to, so -// the non-HTML content type is visibly bound to the untrusted bytes it carries. -func (a *attachment) Write(p []byte) (int, error) { - w := a.w - if !a.started { - a.started = true - header := w.Header() - // A plugin's format must not turn captured bytes into an active browser document. - switch a.format.MIME { - case "application/x-ndjson", "application/json", "application/x-asciicast": - header.Set("Content-Type", a.format.MIME) - default: - header.Set("Content-Type", "application/octet-stream") - } - header.Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", a.filename)) - header.Set("X-Content-Type-Options", "nosniff") - header.Set("Content-Security-Policy", "sandbox") - w.WriteHeader(http.StatusOK) - } - if len(p) == 0 { - return 0, nil + return nil, err } - return w.Write(p) + return service.Handler(APIPrefix, http.NotFoundHandler()) } diff --git a/pkg/traceserve/records/feed.go b/pkg/traceserve/records/feed.go new file mode 100644 index 0000000..0760788 --- /dev/null +++ b/pkg/traceserve/records/feed.go @@ -0,0 +1,214 @@ +package records + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "strconv" + "time" + + "github.com/flanksource/telltale/pkg/traces/read" +) + +// FeedHandler serves the combined records of a single run, never arbitrary +// streams selected by the client. +func FeedHandler(dataDir string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + selected, err := selectFeed(dataDir, r.PathValue("run")) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + defer func() { + if err := selected.Close(); err != nil { + panic(http.ErrAbortHandler) + } + }() + refs, err := filterFeed(selected.Refs, r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + values := r.URL.Query() + for key := range values { + switch key { + case "kind", "excludeKind", "presentation", "stream", "limit", "sort", "cursor", "follow", "format", "resume", "from", "until": + default: + http.Error(w, "unknown feed parameter: "+key, http.StatusBadRequest) + return + } + } + if presentation := values.Get("presentation"); presentation != "" && presentation != "ansi" { + http.Error(w, "unknown feed presentation", http.StatusBadRequest) + return + } + q := read.FeedQuery{Sort: values.Get("sort"), Query: read.Query{Cursor: values.Get("cursor")}} + if q.Sort != "" && q.Sort != "seq" && q.Sort != "timestamp" { + http.Error(w, "invalid record sort", http.StatusBadRequest) + return + } + for name, destination := range map[string]*time.Time{"from": &q.From, "until": &q.Until} { + if raw := values.Get(name); raw != "" { + *destination, err = time.Parse(time.RFC3339Nano, raw) + if err != nil { + http.Error(w, err.Error(), 400) + return + } + } + } + if err := q.Validate(); err != nil { + http.Error(w, err.Error(), 400) + return + } + if raw := values.Get("follow"); raw != "" { + follow, parseErr := strconv.ParseBool(raw) + if parseErr != nil { + http.Error(w, parseErr.Error(), 400) + return + } + if follow { + if q.Cursor != "" || (values.Get("format") != "" && values.Get("format") != "ndjson") { + http.Error(w, "invalid follow cursor or format", 400) + return + } + streamFeed(w, r, refs, q.Query) + return + } + } + if raw := values.Get("limit"); raw != "" { + q.Limit, err = strconv.Atoi(raw) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + } + page, err := read.FeedPage(r.Context(), refs, q) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(presentFeedPage(page, values.Get("presentation"))); err != nil { + panic(http.ErrAbortHandler) + } + }) +} + +func selectFeed(dataDir, id string) (*read.Selection, error) { + return read.ResolveRuns(read.Scope{DataDir: dataDir, Run: id}) +} + +func filterFeed(refs []read.Ref, r *http.Request) ([]read.Ref, error) { + selected := []read.Ref{} + if excluded := r.URL.Query().Get("excludeKind"); excluded != "" && len(refs) > 0 { + if _, ok := refs[0].Runtime.Kinds.Get(excluded); !ok { + return nil, fmt.Errorf("unknown excluded capture kind %q", excluded) + } + } + for _, ref := range refs { + owner, result := ref.Runtime.Kinds.ResultOf(ref.Kind) + if result == "frames" && r.URL.Query().Get("stream") != ref.Stream && r.URL.Query().Get("kind") != ref.Kind { + continue + } + if excluded := r.URL.Query().Get("excludeKind"); excluded != "" && (ref.Kind == excluded || owner == excluded) { + continue + } + if kind := r.URL.Query().Get("kind"); kind != "" && ref.Kind != kind && owner != kind { + continue + } + if stream := r.URL.Query().Get("stream"); stream != "" && ref.Stream != stream { + continue + } + selected = append(selected, ref) + } + if len(selected) == 0 && len(refs) != 0 && (r.URL.Query().Get("kind") != "" || r.URL.Query().Get("stream") != "") { + return nil, fmt.Errorf("capture selection is not armed by this run") + } + return read.EventRefs(selected), nil +} + +type feedResume struct { + From time.Time `json:"from"` + Until time.Time `json:"until"` + Run string `json:"run"` + Positions map[string]int64 `json:"positions"` + Generations map[string]string `json:"generations"` +} + +func streamFeed(w http.ResponseWriter, r *http.Request, refs []read.Ref, q read.Query) { + if r.Header.Get("Last-Event-ID") == "" && r.URL.Query().Get("resume") != "" { + r.Header.Set("Last-Event-ID", r.URL.Query().Get("resume")) + } + resume := feedResume{From: q.From, Until: q.Until, Run: r.PathValue("run"), Positions: map[string]int64{}, Generations: map[string]string{}} + if last := r.Header.Get("Last-Event-ID"); last != "" { + data, err := base64.RawURLEncoding.DecodeString(last) + if err != nil || json.Unmarshal(data, &resume) != nil || resume.Run != r.PathValue("run") || len(resume.Positions) != len(refs) || !resume.From.Equal(q.From) || !resume.Until.Equal(q.Until) { + http.Error(w, "invalid feed resume token", http.StatusBadRequest) + return + } + } + for i, ref := range refs { + key := ref.Stream + meta, err := ref.Runtime.Results.Backend.Meta(r.Context(), ref.Stream) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + _, hasPosition := resume.Positions[key] + if last := r.Header.Get("Last-Event-ID"); last != "" && (!hasPosition || resume.Generations[key] != meta.Generation || resume.Positions[key] < 0 || resume.Positions[key] > meta.HighSeq) { + http.Error(w, "feed resume stream changed", http.StatusBadRequest) + return + } + refs[i].AfterSeq = max(ref.AfterSeq, resume.Positions[key]) + resume.Positions[key] = refs[i].AfterSeq + resume.Generations[key] = meta.Generation + } + ndjson := r.URL.Query().Get("format") == "ndjson" + w.Header().Set("Content-Type", "text/event-stream") + if ndjson { + w.Header().Set("Content-Type", "application/x-ndjson") + } + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("X-Accel-Buffering", "no") + controller := http.NewResponseController(w) + if err := controller.Flush(); err != nil { + return + } + err := read.Feed(r.Context(), refs, q, func(record read.Record) error { + data, err := json.Marshal(presentFeedRecord(record, r.URL.Query().Get("presentation"))) + if err != nil { + return err + } + resume.Positions[record.Stream] = record.Seq + if ndjson { + _, err = fmt.Fprintf(w, "%s\n", data) + } else { + token, marshalErr := json.Marshal(resume) + if marshalErr != nil { + return marshalErr + } + _, err = fmt.Fprintf(w, "id: %s\ndata: %s\n\n", base64.RawURLEncoding.EncodeToString(token), data) + } + if err != nil { + return err + } + return controller.Flush() + }) + if r.Context().Err() != nil { + return + } + if err != nil { + if ndjson { + panic(http.ErrAbortHandler) + } + payload, marshalErr := json.Marshal(map[string]string{"error": err.Error()}) + if marshalErr != nil { + panic(http.ErrAbortHandler) + } + _, _ = fmt.Fprintf(w, "event: error\ndata: %s\n\n", payload) + } else if !ndjson { + _, _ = fmt.Fprint(w, "event: end\ndata: {}\n\n") + } + _ = controller.Flush() +} diff --git a/pkg/traceserve/records/handler.go b/pkg/traceserve/records/handler.go index dabb4bf..57fef12 100644 --- a/pkg/traceserve/records/handler.go +++ b/pkg/traceserve/records/handler.go @@ -51,7 +51,7 @@ func Handler(dataDir string) http.Handler { serveSchema(w, r, refs) return } - if err == nil { + if err == nil && aggregate.View != "top" && aggregate.View != "frames" { err = read.ValidateSelection(r.Context(), refs, q) } if err != nil { @@ -77,7 +77,7 @@ func recordValues(r *http.Request) (url.Values, error) { } for name := range values { switch name { - case "stream", "from", "until", "afterSeq", "toSeq", "filter", "where", "q", "limit", "cursor", "stats", "groupBy", "view", "follow", "format", "all": + case "stream", "result", "from", "until", "afterSeq", "toSeq", "filter", "where", "q", "limit", "cursor", "stats", "groupBy", "view", "follow", "format", "all", "frame", "source", "metric", "sort", "desc": default: return nil, fmt.Errorf("unknown records parameter %q", name) } @@ -114,13 +114,25 @@ func selectRun(kind string, scope read.Scope) (*read.Selection, error) { return read.Resolve(kind, scope) } runtime := run.Runtime() + if scope.Result != "" { + storage := kind + "_" + scope.Result + owner, result := runtime.Kinds.ResultOf(storage) + if owner != kind || result == "" { + return nil, fmt.Errorf("kind %s has no result %q", kind, scope.Result) + } + kind = storage + } if _, ok := runtime.Kinds.Get(kind); !ok { return nil, fmt.Errorf("unknown trace kind %q", kind) } selected := &read.Selection{} - for _, capture := range run.Info().Captures { - if capture.Kind == kind { - selected.Refs = append(selected.Refs, read.Ref{RunID: scope.Run, Runtime: runtime, TraceRef: capture.Ref}) + refs, err := read.RunRefs(run.Info(), runtime) + if err != nil { + return nil, err + } + for _, ref := range refs { + if ref.Kind == kind { + selected.Refs = append(selected.Refs, ref) } } if len(selected.Refs) == 0 { @@ -151,6 +163,10 @@ type page struct { } func servePage(w http.ResponseWriter, r *http.Request, refs []read.Ref, q read.Query, aggregate read.Aggregate) { + if aggregate.View == "top" || aggregate.View == "frames" { + serveTop(w, r, refs, q, aggregate.View) + return + } result := page{Rows: []query.Row{}} q.NextCursor = func(cursor string) { result.NextCursor = cursor } var err error diff --git a/pkg/traceserve/records/handler_test.go b/pkg/traceserve/records/handler_test.go index 7ca1c0c..65b69d3 100644 --- a/pkg/traceserve/records/handler_test.go +++ b/pkg/traceserve/records/handler_test.go @@ -90,6 +90,7 @@ var _ = Describe("run records route", func() { Expect(os.WriteFile(filepath.Join(host.Dir(), telltale.ManifestFile), data, 0o600)).To(Succeed()) mux := http.NewServeMux() mux.Handle("GET /api/traces/{run}/traces/{kind}/records", records.Handler(dir)) + mux.Handle("GET /api/traces/{run}/records", records.FeedHandler(dir)) server = httptest.NewServer(mux) DeferCleanup(server.Close) path = server.URL + "/api/traces/run-http/traces/" + fixtureKind + "/records" @@ -114,6 +115,84 @@ var _ = Describe("run records route", func() { Expect(json.Unmarshal(body, &result)).To(Succeed(), string(body)) return result } + It("adds styled presentation only when requested and permits excluding an unarmed kind", func() { + appendRows(1) + endpoint := server.URL + "/api/traces/run-http/records?excludeKind=process&presentation=ansi&limit=1" + response, err := server.Client().Get(endpoint) + Expect(err).NotTo(HaveOccurred()) + body, err := io.ReadAll(response.Body) + Expect(err).NotTo(HaveOccurred()) + Expect(response.Body.Close()).To(Succeed()) + Expect(response.StatusCode).To(Equal(http.StatusOK), string(body)) + var result struct { + Rows []struct { + ANSI string `json:"ansi"` + } `json:"rows"` + } + Expect(json.Unmarshal(body, &result)).To(Succeed()) + Expect(result.Rows).To(HaveLen(1)) + Expect(result.Rows[0].ANSI).To(ContainSubstring("worker")) + }) + It("resumes a unified SSE feed after the last record without replaying it", func() { + appendRows(1, 2) + _, err := capture.Stop(context.Background()) + Expect(err).NotTo(HaveOccurred()) + endpoint := server.URL + "/api/traces/run-http/records?follow=1" + response, err := server.Client().Get(endpoint) + Expect(err).NotTo(HaveOccurred()) + scanner := bufio.NewScanner(response.Body) + last := "" + count := 0 + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "id: ") { + last = strings.TrimPrefix(line, "id: ") + } + if strings.HasPrefix(line, "data: ") && line != "data: {}" { + count++ + } + } + Expect(scanner.Err()).NotTo(HaveOccurred()) + Expect(response.Body.Close()).To(Succeed()) + Expect(count).To(Equal(2)) + Expect(last).NotTo(BeEmpty()) + request, err := http.NewRequest(http.MethodGet, endpoint, nil) + Expect(err).NotTo(HaveOccurred()) + request.Header.Set("Last-Event-ID", last) + response, err = server.Client().Do(request) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(response.Body.Close) + body, err := io.ReadAll(response.Body) + Expect(err).NotTo(HaveOccurred()) + Expect(string(body)).To(Equal("event: end\ndata: {}\n\n")) + }) + + It("serves the unified feed with native payloads and resumable sequence pages", func() { + appendRows(2, 1) + response, err := server.Client().Get(server.URL + "/api/traces/run-http/records?limit=1") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(response.Body.Close) + Expect(response.StatusCode).To(Equal(http.StatusOK)) + var result struct { + Rows []struct { + Kind string `json:"kind"` + Seq int64 `json:"seq"` + Record query.Row `json:"record"` + } + NextCursor string `json:"nextCursor"` + } + Expect(json.NewDecoder(response.Body).Decode(&result)).To(Succeed()) + Expect(result.Rows).To(HaveLen(1)) + Expect(result.Rows[0].Seq).To(Equal(int64(1))) + Expect(result.Rows[0].Kind).To(Equal(fixtureKind)) + Expect(result.Rows[0].Record).To(HaveKeyWithValue("value", float64(2))) + Expect(result.NextCursor).NotTo(BeEmpty()) + response, err = server.Client().Get(server.URL + "/api/traces/run-http/records?sort=timestamp&cursor=" + url.QueryEscape(result.NextCursor)) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(response.Body.Close) + Expect(response.StatusCode).To(Equal(http.StatusBadRequest)) + }) + It("pages in profile order and applies repeated filters, time, seq, search and CEL before limit", func() { appendRows(0, 1, 2, 3) first := get(url.Values{"limit": {"2"}}) @@ -269,7 +348,7 @@ var _ = Describe("run records route", func() { }) It("reads and follows an in-process live run until it finishes", func() { dir := GinkgoT().TempDir() - run, err := telltale.Start(context.Background(), telltale.StartOptions{DataDir: dir, PID: os.Getpid(), Duration: time.Second, Traces: []telltale.TraceSpec{{Kind: "process_samples"}}}) + run, err := telltale.Start(context.Background(), telltale.StartOptions{DataDir: dir, PID: os.Getpid(), Duration: time.Second, Traces: []telltale.TraceSpec{{Kind: "process"}}}) Expect(err).NotTo(HaveOccurred()) DeferCleanup(func() { Expect(run.Stop()).To(Succeed()); _, err := run.Wait(); Expect(err).NotTo(HaveOccurred()) }) mux := http.NewServeMux() @@ -278,7 +357,7 @@ var _ = Describe("run records route", func() { DeferCleanup(liveServer.Close) ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) defer cancel() - request, err := http.NewRequestWithContext(ctx, http.MethodGet, liveServer.URL+"/api/traces/"+run.ID()+"/traces/process_samples/records?follow=1", nil) + request, err := http.NewRequestWithContext(ctx, http.MethodGet, liveServer.URL+"/api/traces/"+run.ID()+"/traces/process/records?result=metrics&follow=1", nil) Expect(err).NotTo(HaveOccurred()) response, err := liveServer.Client().Do(request) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/traceserve/records/presentation.go b/pkg/traceserve/records/presentation.go new file mode 100644 index 0000000..06460db --- /dev/null +++ b/pkg/traceserve/records/presentation.go @@ -0,0 +1,29 @@ +package records + +import "github.com/flanksource/telltale/pkg/traces/read" + +type styledRecord struct { + read.Record + ANSI string `json:"ansi"` +} + +func presentFeedRecord(record read.Record, presentation string) any { + if presentation == "ansi" { + return styledRecord{Record: record, ANSI: record.Pretty().ANSI()} + } + return record +} + +func presentFeedPage(page read.FeedResult, presentation string) any { + if presentation != "ansi" { + return page + } + rows := make([]any, len(page.Rows)) + for i, row := range page.Rows { + rows[i] = presentFeedRecord(row, presentation) + } + return struct { + Rows []any `json:"rows"` + NextCursor string `json:"nextCursor"` + }{Rows: rows, NextCursor: page.NextCursor} +} diff --git a/pkg/traceserve/records/process.go b/pkg/traceserve/records/process.go new file mode 100644 index 0000000..2823a20 --- /dev/null +++ b/pkg/traceserve/records/process.go @@ -0,0 +1,55 @@ +package records + +import ( + "encoding/json" + "fmt" + "github.com/flanksource/telltale/pkg/traces/read" + "github.com/flanksource/telltale/pkg/tracing/processcapture" + "net/http" + "strconv" +) + +func ProcessGroupsHandler(dataDir string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for key := range r.URL.Query() { + if key != "stream" && key != "afterSeq" { + http.Error(w, "unknown process groups parameter: "+key, 400) + return + } + } + selected, err := selectRun(processcapture.Kind, read.Scope{DataDir: dataDir, Run: r.PathValue("run")}) + if err != nil { + http.Error(w, err.Error(), 400) + return + } + defer func() { + if selected.Close() != nil { + panic(http.ErrAbortHandler) + } + }() + refs, err := selectStreams(selected.Refs, r.URL.Query().Get("stream")) + if err != nil || len(refs) != 1 { + http.Error(w, fmt.Sprintf("select one process capture: %v", err), 400) + return + } + after := int64(0) + if value := r.URL.Query().Get("afterSeq"); value != "" { + after, err = strconv.ParseInt(value, 10, 64) + } + if err != nil || after < 0 { + http.Error(w, "invalid afterSeq", 400) + return + } + ref := refs[0].TraceRef + ref.AfterSeq = after + frames, err := processcapture.ReadFrames(r.Context(), refs[0].Runtime, ref) + if err != nil { + http.Error(w, err.Error(), 400) + return + } + w.Header().Set("Content-Type", "application/json") + if json.NewEncoder(w).Encode(frames) != nil { + panic(http.ErrAbortHandler) + } + }) +} diff --git a/pkg/traceserve/records/processes.go b/pkg/traceserve/records/processes.go new file mode 100644 index 0000000..4f4e4c8 --- /dev/null +++ b/pkg/traceserve/records/processes.go @@ -0,0 +1,36 @@ +package records + +import ( + "encoding/json" + "net/http" + + "github.com/flanksource/telltale/pkg/traces/read" +) + +func ProcessesHandler(dataDir string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.RawQuery != "" { + http.Error(w, "process snapshot takes no query parameters", http.StatusBadRequest) + return + } + selected, err := selectFeed(dataDir, r.PathValue("run")) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + defer func() { + if selected.Close() != nil { + panic(http.ErrAbortHandler) + } + }() + snapshot, err := read.Processes(r.Context(), selected.Refs) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + if json.NewEncoder(w).Encode(snapshot) != nil { + panic(http.ErrAbortHandler) + } + }) +} diff --git a/pkg/traceserve/records/schema.go b/pkg/traceserve/records/schema.go index f7236c8..e596090 100644 --- a/pkg/traceserve/records/schema.go +++ b/pkg/traceserve/records/schema.go @@ -11,9 +11,14 @@ import ( ) type queryOptions struct { - Where string `json:"where,omitempty" clicky:"title=Row predicate,desc=CEL expression evaluated with a row binding"` - From string `json:"from,omitempty" clicky:"title=From,desc=Inclusive row timestamp in RFC3339 format"` - Until string `json:"until,omitempty" clicky:"title=Until,desc=Exclusive row timestamp in RFC3339 format"` + Frame string `json:"frame,omitempty" clicky:"title=Sample,desc=Completed sample id; empty selects the latest complete sample"` + Source string `json:"source,omitempty" clicky:"title=Source,desc=Sample source identity"` + Metric string `json:"metric,omitempty" clicky:"title=Metric,desc=Profile measurement to display"` + Sort string `json:"sort,omitempty" clicky:"title=Sort,desc=Native column sort key"` + Desc bool `json:"desc,omitempty" clicky:"title=Descending,desc=Sort descending"` + Where string `json:"where,omitempty" clicky:"title=Row predicate,desc=CEL expression evaluated with a row binding"` + From string `json:"from,omitempty" clicky:"title=From,desc=Inclusive row timestamp in RFC3339 format"` + Until string `json:"until,omitempty" clicky:"title=Until,desc=Exclusive row timestamp in RFC3339 format"` } func serveSchema(w http.ResponseWriter, r *http.Request, refs []read.Ref) { @@ -22,6 +27,9 @@ func serveSchema(w http.ResponseWriter, r *http.Request, refs []read.Ref) { return } profile, err := refs[0].Runtime.Results.Registry.Get(r.Context(), "traces/"+refs[0].Kind) + if r.URL.Query().Get("view") == "top" { + profile, err = read.TopProfile(r.Context(), refs[0], read.TopQuery{Frame: r.URL.Query().Get("frame"), Source: r.URL.Query().Get("source"), Metric: r.URL.Query().Get("metric")}) + } if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/pkg/traceserve/records/top.go b/pkg/traceserve/records/top.go new file mode 100644 index 0000000..efb5b9b --- /dev/null +++ b/pkg/traceserve/records/top.go @@ -0,0 +1,36 @@ +package records + +import ( + "encoding/json" + "errors" + "net/http" + + "github.com/flanksource/telltale/pkg/traces/read" +) + +func serveTop(w http.ResponseWriter, r *http.Request, refs []read.Ref, q read.Query, view string) { + if len(refs) != 1 { + http.Error(w, "Top requires one capture stream; specify stream", http.StatusBadRequest) + return + } + var result any + var err error + if view == "frames" { + result, err = read.Frames(r.Context(), refs[0]) + } else { + values := r.URL.Query() + result, err = read.Top(r.Context(), refs[0], read.TopQuery{Frame: values.Get("frame"), Source: values.Get("source"), Metric: values.Get("metric"), Query: q}) + } + if err != nil { + status := http.StatusBadRequest + if errors.Is(err, read.ErrNoCompleteSamples) { + status = http.StatusConflict + } + http.Error(w, err.Error(), status) + return + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(result); err != nil { + panic(http.ErrAbortHandler) + } +} diff --git a/pkg/traceserve/traceserve.go b/pkg/traceserve/traceserve.go index 493b635..acffcf0 100644 --- a/pkg/traceserve/traceserve.go +++ b/pkg/traceserve/traceserve.go @@ -69,20 +69,12 @@ func Open(options OpenOptions) (*tracing.Runtime, func() error, error) { // A shared SQLite backend retains the first opener's resolver. Fill it before // opening the backend so concurrent registries cannot see half its catalog. func storeSchemas(kinds *tracing.Kinds) (*recordstore.Schemas, error) { - infos, err := kinds.List() + definitions, err := kinds.Schemas() if err != nil { return nil, err } schemas := recordstore.NewSchemas() - for _, info := range infos { - plugin, ok := kinds.Get(info.Name) - if !ok { - return nil, fmt.Errorf("trace kind %q disappeared from catalog", info.Name) - } - schema, err := plugin.Schema(info.Name) - if err != nil { - return nil, err - } + for _, schema := range definitions { if err := schemas.Register(schema.Kind, schema.Columns, schema.Options); err != nil { return nil, err } @@ -105,7 +97,7 @@ type ReopenOptions struct { const reopenedRetention = time.Hour // Reopen serves the store a closed host left in options.Dir, with the close -// that stops using it. It runs no capture, so a checkpoint is a conflict. A +// that stops using it. It runs no capture, so its registry is empty and a checkpoint is not found. A // store that is gone is os.ErrNotExist; one that is there but does not open is // an error. func Reopen(options ReopenOptions) (http.Handler, func() error, error) { diff --git a/pkg/traceserve/traceserve_test.go b/pkg/traceserve/traceserve_test.go index 418166c..e6632c5 100644 --- a/pkg/traceserve/traceserve_test.go +++ b/pkg/traceserve/traceserve_test.go @@ -172,27 +172,34 @@ var _ = Describe("a served trace store", func() { Expect(column(kinds, "name")).To(ConsistOf(terminal.Kind, stacks.Kind)) }) - It("serves nothing but its trace routes", func() { - for _, path := range []string{"/", "/api/openapi.json", "/api/v1/profiles", "/api/v1/sessions"} { + It("serves nothing but its trace and session routes", func() { + for _, path := range []string{"/", "/api/openapi.json", "/api/v1/profiles"} { status, body, _ := store.do(http.MethodGet, path) Expect(status).To(Equal(http.StatusNotFound), path) Expect(body).ToNot(ContainSubstring(" 0 { + key := record.SampleType + "/" + record.Unit + if functions[key] == nil { + functions[key] = map[string]int64{} + } + functions[key][record.Frames[0].Func] += record.Value + } + } + ranked := map[string]any{} + for key, values := range functions { + ranked[key] = tracing.RankedValues(values) } - return map[string]any{"checkpoints": len(checkpoints), "totals": totals}, nil + return map[string]any{"checkpoints": len(checkpoints), "totals": totals, "functions": ranked}, nil } // Export rebuilds the window's records into one pprof profile: its sample diff --git a/pkg/tracing/goprof/goprof.go b/pkg/tracing/goprof/goprof.go index 0d9821c..e9a506c 100644 --- a/pkg/tracing/goprof/goprof.go +++ b/pkg/tracing/goprof/goprof.go @@ -17,6 +17,7 @@ import ( dbcontext "github.com/flanksource/commons-db/context" "github.com/flanksource/commons-db/recordstore" "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/commons-db/recordstore/sampleframe" "github.com/flanksource/telltale/pkg/agentclient" "github.com/flanksource/telltale/pkg/tracing" "github.com/flanksource/telltale/pkg/tracing/seam" @@ -214,25 +215,27 @@ func (p profiler) Handle(ctx dbcontext.Context, params Params, out tracing.Emitt func NewKind(addresses Addresses) *tracing.Handler[Params, Sample] { agent := seam.AgentSeam[Params, Sample]{Probe: Kind, Registry: addresses.Registry, Target: func(p Params) target.Target { return p.Target }, Mutates: func(p Params) bool { return p.Target.AllowMutation }, ArmParams: func(p Params) (json.RawMessage, error) { return json.Marshal(map[string]any{"profile": "heap", "mutex": p.Target.AllowMutation, "block": p.Target.AllowMutation}) - }, DecodeBlob: func(blob agentclient.Blob, resource map[string]string) ([]Sample, error) { + }, DecodeBlob: func(blob agentclient.Blob, resource map[string]string) (seam.DecodedBlob[Sample], error) { + started := time.Now().UTC() if blob.Info.ContentType != protocol.ContentTypePprof { - return nil, fmt.Errorf("unexpected profile content type %s", blob.Info.ContentType) + return seam.DecodedBlob[Sample]{}, fmt.Errorf("unexpected profile content type %s", blob.Info.ContentType) } pid, err := strconv.Atoi(resource["process.pid"]) if err != nil { - return nil, fmt.Errorf("agent process pid: %w", err) + return seam.DecodedBlob[Sample]{}, fmt.Errorf("agent process pid: %w", err) } parsed, err := profile.ParseData(blob.Data) if err != nil { - return nil, fmt.Errorf("parse agent profile: %w", err) + return seam.DecodedBlob[Sample]{}, fmt.Errorf("parse agent profile: %w", err) } var request CheckpointParams if len(blob.CheckpointParams) > 0 { if err := json.Unmarshal(blob.CheckpointParams, &request); err != nil { - return nil, fmt.Errorf("profile checkpoint params: %w", err) + return seam.DecodedBlob[Sample]{}, fmt.Errorf("profile checkpoint params: %w", err) } } - return Aggregate(Sample{CheckpointID: uuid.NewString(), TakenAt: time.Now().UTC(), Reason: request.Reason, Source: "agent", PID: pid, Profile: blob.Info.Name}, parsed), nil + rows := Aggregate(Sample{CheckpointID: uuid.NewString(), TakenAt: time.Now().UTC(), Reason: request.Reason, Source: "agent", PID: pid, Profile: blob.Info.Name}, parsed) + return seam.DecodedBlob[Sample]{Rows: rows, Frame: &sampleframe.Frame{Source: resource["telltale.agent.id"] + "/" + resource["process.pid"] + "/" + blob.Info.Name, StartedAt: started, CompletedAt: time.Now().UTC(), Observed: len(rows)}}, nil }} - return tracing.NewHandler[Params, Sample](profiler{addresses: addresses, agent: agent}, tracing.Capabilities{Live: true}, target.Process, target.Agent, target.Self) + return tracing.NewHandler[Params, Sample](profiler{addresses: addresses, agent: agent}, tracing.Capabilities{Live: true}, target.Process, target.Agent, target.Self).WithTop(topSpec()) } diff --git a/pkg/tracing/goprof/pretty.go b/pkg/tracing/goprof/pretty.go new file mode 100644 index 0000000..c256491 --- /dev/null +++ b/pkg/tracing/goprof/pretty.go @@ -0,0 +1,26 @@ +package goprof + +import ( + "fmt" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" + "github.com/flanksource/telltale/pkg/tracing" +) + +func (r Sample) Pretty() api.Text { + t := clicky.Text(fmt.Sprintf("pid %d", r.PID), "text-muted").Space().Append(r.Profile, "text-blue-500 font-bold"). + Space().Append(r.SampleType, "text-muted").Space().Add(tracing.PrettySampleValue(r.Value, r.Unit)) + if len(r.Frames) > 0 { + t = t.Space().Append(r.Frames[0].Func, "font-mono") + } + return t +} + +func (r Sample) PrettyFull() api.Textable { + t := r.Pretty() + for _, frame := range r.Frames { + t = t.NewLine().Add(frame.Pretty()) + } + return t +} diff --git a/pkg/tracing/goprof/table.go b/pkg/tracing/goprof/table.go new file mode 100644 index 0000000..7250ee7 --- /dev/null +++ b/pkg/tracing/goprof/table.go @@ -0,0 +1,34 @@ +package goprof + +import ( + "github.com/flanksource/clicky/api" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/telltale/pkg/tracing" +) + +func (Sample) Columns() []api.ColumnDef { + return query.TableColumnsFor[Sample]("pid", "profile", "sampleType", "value", "frames") +} +func (s Sample) Row() map[string]any { + row := query.TableRowFor(s) + row["value"] = api.TableCell{Value: tracing.PrettySampleValue(s.Value, s.Unit), FilterValue: s.Value} + if len(s.Frames) > 0 { + row["frames"] = api.Text{}.Append(s.Frames[0].Func, "font-mono") + } + return row +} +func (s Sample) RowDetail() api.Textable { return s.PrettyFull() } + +func topSpec() tracing.TopSpec { + return tracing.TopSpec{Stack: &tracing.TopStack{Column: "frames", MetricColumn: "sampleType", FrameKey: "func", LeafFirst: true}, Identity: []string{"stackHash", "sampleType"}, GroupBy: []string{"key", "profile"}, Order: query.Order{{Column: "value", Desc: true}}, Metrics: []tracing.TopMetric{ + {Key: "cpu", Label: "CPU time", Unit: "nanoseconds", View: "flamegraph"}, + {Key: "inuse_space", Label: "In-use bytes", Unit: "bytes", View: "treemap"}, + {Key: "alloc_space", Label: "Allocated bytes", Unit: "bytes", View: "treemap"}, + {Key: "inuse_objects", Label: "In-use objects", Unit: "count", View: "table"}, + {Key: "alloc_objects", Label: "Allocated objects", Unit: "count", View: "table"}, + {Key: "samples", Label: "Samples", Unit: "count", View: "flamegraph"}, + {Key: "goroutine", Label: "Goroutines", Unit: "count", View: "table"}, + {Key: "contentions", Label: "Contentions", Unit: "count", View: "table"}, + {Key: "delay", Label: "Delay", Unit: "nanoseconds", View: "flamegraph"}, + }} +} diff --git a/pkg/tracing/groups/catalog.go b/pkg/tracing/groups/catalog.go index 9cb20ea..e939c43 100644 --- a/pkg/tracing/groups/catalog.go +++ b/pkg/tracing/groups/catalog.go @@ -53,6 +53,10 @@ type Rule struct { // TargetSession, or a flag such as --session whose value it is. Target string `yaml:"target,omitempty" json:"target,omitempty"` Match []Match `yaml:"match,omitempty" json:"match,omitempty"` + // SkipAncestry marks a runner, such as gavel, whose groups do not own the + // processes they start by ancestry alone: what it publishes about its + // tasks claims them first, by pid, wherever they run. + SkipAncestry bool `yaml:"skipAncestry,omitempty" json:"skipAncestry,omitempty"` // Disabled drops the built-in rule of this name. Disabled bool `yaml:"disabled,omitempty" json:"-"` } diff --git a/pkg/tracing/groups/catalog.yaml b/pkg/tracing/groups/catalog.yaml index e46d2b5..249fe4d 100644 --- a/pkg/tracing/groups/catalog.yaml +++ b/pkg/tracing/groups/catalog.yaml @@ -26,6 +26,7 @@ groups: - name: gavel kind: test target: positional + skipAncestry: true match: - exec: [gavel] - name: go-test diff --git a/pkg/tracing/groups/catalog_test.go b/pkg/tracing/groups/catalog_test.go index 1baf6b8..0dcda4c 100644 --- a/pkg/tracing/groups/catalog_test.go +++ b/pkg/tracing/groups/catalog_test.go @@ -40,10 +40,21 @@ groups: - exec: [gavel-dev] - name: gopls disabled: true + - name: acme-runner + kind: build + skipAncestry: true + match: + - exec: [acme-runner] `)) Expect(err).NotTo(HaveOccurred()) + runner, ok := catalog.Rule("acme-runner") + Expect(ok).To(BeTrue()) + Expect(runner.SkipAncestry).To(BeTrue(), "groups.yaml can skip-list a runner") + builtinGavel, ok := builtin.Rule("gavel") + Expect(ok).To(BeTrue()) + Expect(builtinGavel.SkipAncestry).To(BeTrue(), "the built-in gavel rule is skip-listed") - expected := []string{"acme-server"} + expected := []string{"acme-server", "acme-runner"} for _, name := range ruleNames(builtin) { if name != "gopls" { expected = append(expected, name) diff --git a/pkg/tracing/groups/empty_read_test.go b/pkg/tracing/groups/empty_read_test.go new file mode 100644 index 0000000..ee2de03 --- /dev/null +++ b/pkg/tracing/groups/empty_read_test.go @@ -0,0 +1,21 @@ +package groups + +import ( + "context" + "time" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = ginkgo.Describe("an empty process observation", func() { + ginkgo.It("does not poll memory or resources after the last process exits", func() { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + top := &Top{rates: NewRates()} + readings, held, err := top.read(ctx, nil, time.Now()) + Expect(err).NotTo(HaveOccurred()) + Expect(readings).To(BeEmpty()) + Expect(held).To(BeEmpty()) + }) +}) diff --git a/pkg/tracing/groups/gavel.go b/pkg/tracing/groups/gavel.go new file mode 100644 index 0000000..9e311ad --- /dev/null +++ b/pkg/tracing/groups/gavel.go @@ -0,0 +1,151 @@ +package groups + +import ( + "cmp" + "context" + "errors" + "os" + "slices" + "sync" + + "github.com/flanksource/gavel/sdk/status" +) + +// gavelRule names the catalog rule whose groups are gavel processes. +const gavelRule = "gavel" + +// SlotPool is one gavel slot pool as the gavel processes shown use it. +type SlotPool struct { + // Name is "global" or "project/". + Name string `json:"name"` + // Project is the git common directory of a project pool. + Project string `json:"project,omitempty"` + Limit int `json:"limit"` + Held int `json:"held"` + Waiting int `json:"waiting"` +} + +// SlotPools adds up the leases of statuses by pool, global first. +func SlotPools(statuses []status.Status) []SlotPool { + byName := map[string]*SlotPool{} + for _, s := range statuses { + for _, lease := range s.Slots { + pool, ok := byName[lease.Pool] + if !ok { + pool = &SlotPool{Name: lease.Pool} + byName[lease.Pool] = pool + } + pool.Project = cmp.Or(pool.Project, lease.Project) + pool.Limit = max(pool.Limit, lease.Limit) + if lease.State == status.LeaseWaiting { + pool.Waiting++ + } else { + pool.Held++ + } + } + } + pools := make([]SlotPool, 0, len(byName)) + for _, pool := range byName { + pools = append(pools, *pool) + } + rank := func(p SlotPool) int { + if p.Name == "global" { + return 0 + } + return 1 + } + slices.SortFunc(pools, func(a, b SlotPool) int { + return cmp.Or(cmp.Compare(rank(a), rank(b)), cmp.Compare(a.Project, b.Project), cmp.Compare(a.Name, b.Name)) + }) + return pools +} + +// gavelStatuses follows what each gavel process shown publishes, by pid, +// until it is closed. +type gavelStatuses struct { + cache string + ctx context.Context + cancel context.CancelFunc + mu sync.Mutex + byPID map[int]*gavelWatch +} + +type gavelWatch struct { + cancel context.CancelFunc + latest *status.Status + err error +} + +func newGavelStatuses(cache string) *gavelStatuses { + ctx, cancel := context.WithCancel(context.Background()) + return &gavelStatuses{cache: cache, ctx: ctx, cancel: cancel, byPID: map[int]*gavelWatch{}} +} + +// GavelReading is what a gavel process published, or why it could not be read. +type GavelReading struct { + Status *status.Status + Err error +} + +// of follows the processes pids, stops following the others, and returns +// what each has published so far. A process that has published nothing yet +// is retried on the next call. +func (g *gavelStatuses) of(pids []int) map[int]GavelReading { + g.mu.Lock() + defer g.mu.Unlock() + wanted := map[int]bool{} + for _, pid := range pids { + wanted[pid] = true + if _, ok := g.byPID[pid]; !ok { + g.follow(pid) + } + } + readings := map[int]GavelReading{} + for pid, watch := range g.byPID { + if !wanted[pid] { + watch.cancel() + delete(g.byPID, pid) + continue + } + readings[pid] = GavelReading{Status: watch.latest, Err: watch.err} + } + return readings +} + +// follow starts watching pid, once it has published its status. +func (g *gavelStatuses) follow(pid int) { + if _, err := os.Stat(status.Path(g.cache, pid)); errors.Is(err, os.ErrNotExist) { + return + } + ctx, cancel := context.WithCancel(g.ctx) + watch := &gavelWatch{cancel: cancel} + g.byPID[pid] = watch + watcher, err := status.Watch(ctx, status.WatchOptions{Cache: g.cache, PID: pid}) + if err != nil { + watch.err = err + return + } + go func() { + for update := range watcher.Updates() { + g.mu.Lock() + watch.latest = &update + g.mu.Unlock() + } + g.mu.Lock() + defer g.mu.Unlock() + watch.err = watcher.Err() + // A process whose status went away has finished; it is followed + // again should it publish once more. + if g.byPID[pid] == watch && watch.err == nil { + delete(g.byPID, pid) + } + }() +} + +// close stops following every process. +func (g *gavelStatuses) close() { + g.cancel() + g.mu.Lock() + defer g.mu.Unlock() + clear(g.byPID) +} diff --git a/pkg/tracing/groups/gavel_render.go b/pkg/tracing/groups/gavel_render.go new file mode 100644 index 0000000..8b7a473 --- /dev/null +++ b/pkg/tracing/groups/gavel_render.go @@ -0,0 +1,152 @@ +package groups + +import ( + "cmp" + "fmt" + "path/filepath" + "strings" + "time" + + "github.com/flanksource/gavel/sdk/status" +) + +// GavelText is a gavel process's progress in a line: each task group's and +// its tests' finished of total and failures, then the slots it holds (●) +// and waits for (⧗). +func GavelText(s *status.Status) string { + if s == nil { + return "" + } + var parts []string + for _, group := range s.Groups { + if group.Summary.Total > 0 { + parts = append(parts, summaryText(cmp.Or(group.Kind, group.Name), group.Summary)) + } + } + if s.Tests != nil && s.Tests.Total > 0 { + parts = append(parts, summaryText("tests", *s.Tests)) + } + if slots := SlotGlyphs(s); slots != "" { + parts = append(parts, slots) + } + return strings.Join(parts, " · ") +} + +func leaseGlyph(lease status.Lease) string { + if lease.State == status.LeaseWaiting { + return "⧗" + } + return "●" +} + +// rowBar is the width of a progress bar on a tree row. +const rowBar = 8 + +// barPart is n of a bar's total, drawn as glyph. +type barPart struct { + n int + glyph string +} + +// progressBar draws parts of total across width cells, in order, leaving the +// rest ░. Each part ends where its running sum rounds to, so the parts never +// overflow the bar, but a part that has any takes at least a cell: one +// failure in hundreds still shows. +func progressBar(width, total int, parts ...barPart) string { + var b strings.Builder + b.WriteString("[") + drawn, sum := 0, 0 + for _, part := range parts { + sum += part.n + end := width + if total > 0 { + end = min(width, (sum*width+total/2)/total) + } + if part.n > 0 && end <= drawn && drawn < width { + end = drawn + 1 + } + if end > drawn { + b.WriteString(strings.Repeat(part.glyph, end-drawn)) + drawn = end + } + } + b.WriteString(strings.Repeat("░", width-drawn) + "]") + return b.String() +} + +// summaryBar is a summary as a bar: █ finished, ▓ failed, ▒ running, ░ pending. +func summaryBar(width int, s status.Summary) string { + return progressBar(width, s.Total, barPart{s.Done() - s.Failed, "█"}, barPart{s.Failed, "▓"}, barPart{s.Running, "▒"}) +} + +func summaryText(label string, s status.Summary) string { + text := fmt.Sprintf("%s %s %d/%d", label, summaryBar(rowBar, s), s.Done(), s.Total) + if s.Failed > 0 { + text += fmt.Sprintf(" ✗%d", s.Failed) + } + return text +} + +// poolLabel is "global", or a project pool's repository name. +func poolLabel(pool, project string) string { + if pool == "global" || project == "" { + return pool + } + if filepath.Base(project) == ".git" { + project = filepath.Dir(project) + } + return strings.TrimSuffix(filepath.Base(project), ".git") +} + +// SlotsText is each slot pool's held of limit and how many wait for it. +func SlotsText(pools []SlotPool) string { + var parts []string + for _, pool := range pools { + text := fmt.Sprintf("%s %s %d/%d", poolLabel(pool.Name, pool.Project), progressBar(rowBar, pool.Limit, barPart{pool.Held, "█"}), pool.Held, pool.Limit) + if pool.Waiting > 0 { + text += fmt.Sprintf(" +%d waiting", pool.Waiting) + } + parts = append(parts, text) + } + return strings.Join(parts, " · ") +} + +// SlotGlyphs are the slots a gavel process holds (●) and waits for (⧗). +func SlotGlyphs(s *status.Status) string { + if s == nil { + return "" + } + var leases []string + for _, lease := range s.Slots { + leases = append(leases, leaseGlyph(lease)+poolLabel(lease.Pool, lease.Project)) + } + return strings.Join(leases, " ") +} + +// TaskText is what a row WithGavelTasks added says at now: a task group's +// progress, or a running task, how long it has run and its own progress. +func TaskText(group GroupSample, now time.Time) string { + task := group.Task + switch { + case task == nil: + return "" + case task.Summary != nil: + return summaryText(group.Name, *task.Summary) + } + item := task.Item + text := fmt.Sprintf("▶ %s %s", item.Name, age(now, item.Started)) + if item.Total > 0 { + text += fmt.Sprintf(" %s %d/%d", progressBar(rowBar, item.Total, barPart{item.Done, "█"}), item.Done, item.Total) + } + if item.Description != "" { + text += " · " + item.Description + } + return text +} + +func age(now, since time.Time) string { + if since.IsZero() { + return "-" + } + return now.Sub(since).Round(time.Second).String() +} diff --git a/pkg/tracing/groups/gavel_tasks.go b/pkg/tracing/groups/gavel_tasks.go new file mode 100644 index 0000000..fe3438a --- /dev/null +++ b/pkg/tracing/groups/gavel_tasks.go @@ -0,0 +1,114 @@ +package groups + +import ( + "cmp" + "math" + "strconv" + + "github.com/flanksource/gavel/sdk/status" +) + +// GavelTask is what a row WithGavelTasks adds shows: a task group's +// progress, or one running task. Order keeps the rows in gavel's order. +type GavelTask struct { + Order int + Summary *status.Summary + Item *status.Item +} + +// taskRank orders a gavel's task rows first, in gavel's order. +func taskRank(group GroupSample) int { + if group.Task == nil { + return math.MaxInt + } + return group.Task.Order +} + +// WithGavelTasks nests what each gavel group publishes under it: a row per +// task group with progress, and under each a row per task it is running. +// A skip-listed gavel's tasks claim the group of the process each runs, by +// pid, wherever that group is (a detached process included); its other +// groups stay where ancestry put them. The frame is rolled up again, so a +// claimed group's use counts once, under its task. +func (f Frame) WithGavelTasks() Frame { + byKey := map[string]GroupSample{} + for _, group := range f.Groups { + byKey[group.Key] = group + } + var added []GroupSample + claims := map[int]string{} + ancestors := map[int]map[string]bool{} + for _, group := range f.Groups { + if group.Gavel == nil || group.Kind == KindBucket { + continue + } + rows, itemPIDs := gavelTaskRows(group) + added = append(added, rows...) + if !group.SkipAncestry { + continue + } + lineage := map[string]bool{} + for key := group.Key; key != ""; key = byKey[key].ParentKey { + if lineage[key] { + break + } + lineage[key] = true + } + for pid, key := range itemPIDs { + claims[pid], ancestors[pid] = key, lineage + } + } + if len(added) == 0 { + return f + } + groups := make([]GroupSample, 0, len(f.Groups)+len(added)) + for _, group := range f.Groups { + if key, ok := claims[group.PID]; ok && group.PID != 0 && group.Kind != KindBucket && !ancestors[group.PID][group.Key] { + group.ParentKey = key + } + groups = append(groups, group) + } + f.Groups = Rollup(append(groups, added...)) + return f +} + +// gavelTaskRows are the rows of gavel's task groups and running tasks, and +// the row of each task that runs a process, by its pid. +func gavelTaskRows(gavel GroupSample) ([]GroupSample, map[int]string) { + row := func(key, parent, name, kind string, task *GavelTask) GroupSample { + return GroupSample{ + SampledAt: gavel.SampledAt, Tick: gavel.Tick, Key: key, ParentKey: parent, Name: name, Kind: kind, + Context: gavel.Context, Origin: gavel.Origin, Task: task, + } + } + var rows []GroupSample + byName := map[string]string{} + for _, group := range gavel.Gavel.Groups { + if group.Summary.Total == 0 { + continue + } + name := cmp.Or(group.Kind, group.Name) + key := gavel.Key + "/gavel/" + name + summary := group.Summary + rows = append(rows, row(key, gavel.Key, name, KindGavelTask, &GavelTask{Order: len(rows), Summary: &summary})) + byName[group.Name] = key + } + if tests := gavel.Gavel.Tests; tests != nil && tests.Total > 0 { + summary := *tests + rows = append(rows, row(gavel.Key+"/gavel/tests", gavel.Key, "tests", KindGavelTask, &GavelTask{Order: len(rows), Summary: &summary})) + } + pids := map[int]string{} + for index, item := range gavel.Gavel.Running { + parent, ok := byName[item.Group] + if !ok { + parent = gavel.Key + } + key := parent + "/" + strconv.Itoa(index) + running := item + rows = append(rows, row(key, parent, item.Name, KindGavelItem, &GavelTask{Order: index, Item: &running})) + if item.PID > 0 { + pids[item.PID] = key + } + } + return rows, pids +} diff --git a/pkg/tracing/groups/gavel_tasks_test.go b/pkg/tracing/groups/gavel_tasks_test.go new file mode 100644 index 0000000..1307608 --- /dev/null +++ b/pkg/tracing/groups/gavel_tasks_test.go @@ -0,0 +1,112 @@ +package groups_test + +import ( + "github.com/flanksource/gavel/sdk/status" + "github.com/flanksource/telltale/pkg/tracing/groups" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func parentsOf(frame groups.Frame) map[string]string { + parents := map[string]string{} + for _, group := range frame.Groups { + parents[group.Key] = group.ParentKey + } + return parents +} + +func groupByKey(frame groups.Frame, key string) groups.GroupSample { + GinkgoHelper() + for _, group := range frame.Groups { + if group.Key == key { + return group + } + } + Fail("no group " + key) + return groups.GroupSample{} +} + +var _ = Describe("WithGavelTasks", func() { + usage := func(processes int, rss uint64) groups.Usage { return groups.Usage{Processes: processes, RSSBytes: rss} } + var frame groups.Frame + BeforeEach(func() { + gavel := &status.Status{ + Groups: []status.Group{ + {Name: "Running tests across packages", Kind: status.GroupPackages, Summary: status.Summary{Total: 3, Passed: 1, Running: 2}}, + {Name: "Running linters", Kind: status.GroupLint, Summary: status.Summary{Total: 0}}, + }, + Tests: &status.Summary{Total: 40, Passed: 10, Pending: 30}, + Running: []status.Item{ + {Name: "go test ./pkg/a", Group: "Running tests across packages", PID: 300}, + {Name: "go test ./pkg/b", Group: "Running tests across packages", PID: 400}, + {Name: "prepare", Group: "Unknown group"}, + }, + } + frame = groups.Frame{Groups: []groups.GroupSample{ + {Key: "agent", Name: "claude", Kind: groups.KindAgent, PID: 100, Self: usage(1, 10)}, + {Key: "gavel", ParentKey: "agent", Name: "gavel", Kind: groups.KindTest, PID: 200, Gavel: gavel, SkipAncestry: true, Self: usage(1, 20)}, + {Key: "go-a", ParentKey: "gavel", Name: "go-test", Kind: groups.KindTest, PID: 300, Self: usage(2, 300)}, + {Key: "go-c", ParentKey: "gavel", Name: "go-test", Kind: groups.KindTest, PID: 500, Self: usage(1, 50)}, + {Key: "launchd", Name: "launchd", Kind: groups.KindOther, PID: 1, Self: usage(1, 1)}, + {Key: "go-b", ParentKey: "launchd", Name: "go-test", Kind: groups.KindTest, PID: 400, Self: usage(1, 40)}, + }} + }) + + It("nests task groups, then running tasks, under the gavel, and each task's process group under its task, detached ones included", func() { + nested := frame.WithGavelTasks() + Expect(parentsOf(nested)).To(Equal(map[string]string{ + "agent": "", "launchd": "", + "gavel": "agent", + "gavel/gavel/packages": "gavel", + "gavel/gavel/tests": "gavel", + "gavel/gavel/packages/0": "gavel/gavel/packages", + "gavel/gavel/packages/1": "gavel/gavel/packages", + "gavel/2": "gavel", + "go-a": "gavel/gavel/packages/0", + "go-b": "gavel/gavel/packages/1", + "go-c": "gavel", + }), "an empty lint group has no row; a task of an unknown group sits under the gavel; go-c, which no task claims, stays") + Expect(groupByKey(nested, "gavel/gavel/packages").Total).To(Equal(usage(3, 340)), "the claimed groups' use counts once, under their tasks") + Expect(groupByKey(nested, "gavel").Total).To(Equal(usage(5, 410))) + Expect(groupByKey(nested, "launchd").Total).To(Equal(usage(1, 1)), "the detached go-b moved under its task") + Expect(groupByKey(nested, "gavel/gavel/packages").Task.Summary).To(HaveValue(HaveField("Passed", 1))) + Expect(groupByKey(nested, "gavel/gavel/packages/1").Task.Item).To(HaveValue(HaveField("PID", 400))) + }) + + It("keeps the tasks of a gavel that is not skip-listed from claiming groups", func() { + frame.Groups[1].SkipAncestry = false + parents := parentsOf(frame.WithGavelTasks()) + Expect(parents).To(HaveKeyWithValue("go-a", "gavel")) + Expect(parents).To(HaveKeyWithValue("go-b", "launchd")) + Expect(parents).To(HaveKeyWithValue("gavel/gavel/packages/0", "gavel/gavel/packages")) + }) + + It("never lets a task claim its gavel or the gavel's ancestors", func() { + frame.Groups[1].Gavel.Running[0].PID = 100 + frame.Groups[1].Gavel.Running[1].PID = 200 + parents := parentsOf(frame.WithGavelTasks()) + Expect(parents).To(HaveKeyWithValue("agent", "")) + Expect(parents).To(HaveKeyWithValue("gavel", "agent")) + }) + + It("leaves a frame without gavel status as it is, and composes after a grouping", func() { + frame.Groups[1].Gavel = nil + Expect(frame.WithGavelTasks()).To(Equal(frame)) + + frame.Groups[1].Gavel = &status.Status{Running: []status.Item{{Name: "go test ./pkg/a", PID: 300}}} + for _, by := range []groups.GroupBy{groups.GroupRepo, groups.GroupSlots} { + projected, err := frame.Grouped(by) + Expect(err).NotTo(HaveOccurred()) + Expect(parentsOf(projected.WithGavelTasks())).To(HaveKeyWithValue("go-a", "gavel/0"), "grouped by %s", by) + } + }) +}) + +var _ = DescribeTable("ShortCommand drops the executable's directory", + func(command, short string) { Expect(groups.ShortCommand(command)).To(Equal(short)) }, + Entry("a program on a path", "/opt/homebrew/bin/golangci-lint run ./...", "golangci-lint run ./..."), + Entry("an app bundle's helper, whose name holds spaces", + "/Applications/Acme Browser.app/Contents/Frameworks/Acme.framework/Helpers/Acme Helper (Renderer).app/Contents/MacOS/Acme Helper (Renderer) --type=renderer", + "Acme Helper (Renderer) --type=renderer"), + Entry("a bare program", "gavel test ./...", "gavel test ./..."), +) diff --git a/pkg/tracing/groups/gavel_test.go b/pkg/tracing/groups/gavel_test.go new file mode 100644 index 0000000..4886d7a --- /dev/null +++ b/pkg/tracing/groups/gavel_test.go @@ -0,0 +1,110 @@ +package groups_test + +import ( + "encoding/json" + "os" + "os/exec" + "time" + + "github.com/flanksource/gavel/sdk/status" + "github.com/flanksource/telltale/pkg/tracing/groups" + "github.com/flanksource/telltale/pkg/tracing/process" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func publish(cache string, s status.Status) { + GinkgoHelper() + data, err := json.Marshal(s) + Expect(err).NotTo(HaveOccurred()) + Expect(os.MkdirAll(status.Dir(cache), 0o700)).To(Succeed()) + Expect(os.WriteFile(status.Path(cache, s.PID), data, 0o600)).To(Succeed()) +} + +var ( + acme = "/src/acme/.git" + global = func(state string, index int) status.Lease { + return status.Lease{Pool: "global", Limit: 4, Index: index, State: state} + } + project = func(state string, index int) status.Lease { + return status.Lease{Pool: "project/abc", Project: acme, Limit: 2, Index: index, State: state} + } +) + +var _ = Describe("gavel status in Top", func() { + It("shows what a gavel group's process publishes, until it stops publishing", func(ctx SpecContext) { + cache := GinkgoT().TempDir() + catalog, err := groups.LoadCatalog(writeCatalog("groups:\n - name: gavel\n kind: test\n target: positional\n match: [{exec: [sleep]}]\n")) + Expect(err).NotTo(HaveOccurred()) + gavel := exec.Command("sleep", "31") + Expect(gavel.Start()).To(Succeed()) + DeferCleanup(func() { _ = gavel.Process.Kill(); _ = gavel.Wait() }) + pid := gavel.Process.Pid + publish(cache, status.Status{PID: pid, Command: "gavel lint", + Groups: []status.Group{{Name: "Running linters", Kind: status.GroupLint, Summary: status.Summary{Total: 4, Passed: 1, Running: 3}}}, + Slots: []status.Lease{global(status.LeaseHeld, 0), project(status.LeaseWaiting, -1)}, + }) + + top, err := groups.NewTop(ctx, process.NewSampler(), groups.TopOptions{Catalog: catalog, Groups: []string{"gavel"}, GavelCache: cache}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(top.Close) + var frame groups.Frame + gavelGroup := func() groups.GroupSample { + frame, err = top.Tick(ctx) + Expect(err).NotTo(HaveOccurred()) + for _, group := range frame.Groups { + if group.PID == pid { + return group + } + } + Fail("the gavel process has no group") + return groups.GroupSample{} + } + + Eventually(gavelGroup).WithTimeout(5 * time.Second).Should(SatisfyAll( + HaveField("Gavel", HaveValue(HaveField("Command", "gavel lint"))), + HaveField("Gavel.Groups", ConsistOf(HaveField("Summary.Passed", 1))), + )) + Expect(frame.Slots).To(Equal([]groups.SlotPool{ + {Name: "global", Limit: 4, Held: 1}, + {Name: "project/abc", Project: acme, Limit: 2, Waiting: 1}, + })) + + publish(cache, status.Status{PID: pid, Command: "gavel lint", + Groups: []status.Group{{Name: "Running linters", Kind: status.GroupLint, Summary: status.Summary{Total: 4, Passed: 4}}}}) + Eventually(gavelGroup).WithTimeout(5 * time.Second).Should(HaveField("Gavel.Groups", ConsistOf(HaveField("Summary.Passed", 4)))) + + Expect(os.Remove(status.Path(cache, pid))).To(Succeed()) + Eventually(gavelGroup).WithTimeout(5 * time.Second).Should(HaveField("Gavel", BeNil())) + }, SpecTimeout(30*time.Second)) +}) + +var _ = Describe("gavel slot pools", func() { + It("adds up held and waiting leases per pool across processes, global first", func() { + pools := groups.SlotPools([]status.Status{ + {PID: 1, Slots: []status.Lease{project(status.LeaseHeld, 0), global(status.LeaseHeld, 0)}}, + {PID: 2, Slots: []status.Lease{project(status.LeaseHeld, 1), global(status.LeaseHeld, 1)}}, + {PID: 3, Slots: []status.Lease{project(status.LeaseWaiting, -1)}}, + }) + Expect(pools).To(Equal([]groups.SlotPool{ + {Name: "global", Limit: 4, Held: 2}, + {Name: "project/abc", Project: acme, Limit: 2, Held: 2, Waiting: 1}, + })) + Expect(groups.SlotsText(pools)).To(Equal("global [████░░░░] 2/4 · acme [████████] 2/2 +1 waiting")) + }) + + It("says a process's progress in a line: groups, tests, then held and awaited slots", func() { + text := groups.GavelText(&status.Status{ + Groups: []status.Group{ + {Kind: status.GroupPackages, Summary: status.Summary{Total: 40, Passed: 10, Failed: 2, Running: 4, Pending: 24}}, + {Kind: status.GroupLint, Summary: status.Summary{}}, + {Name: "nested", Summary: status.Summary{Total: 1, Passed: 1}}, + }, + Tests: &status.Summary{Total: 400, Passed: 118, Failed: 2, Pending: 280}, + Slots: []status.Lease{global(status.LeaseHeld, 0), project(status.LeaseWaiting, -1)}, + }) + // packages: 10 passed → 2 cells █, 2 failed → 1 ▓, 4 running → 1 ▒; tests: one ▓ cell for 2 failures in 400. + Expect(text).To(Equal("packages [██▓▒░░░░] 12/40 ✗2 · nested [████████] 1/1 · tests [██▓░░░░░] 120/400 ✗2 · ●global ⧗acme")) + Expect(groups.GavelText(nil)).To(BeEmpty()) + }) +}) diff --git a/pkg/tracing/groups/grouping.go b/pkg/tracing/groups/grouping.go index 5331898..6710142 100644 --- a/pkg/tracing/groups/grouping.go +++ b/pkg/tracing/groups/grouping.go @@ -4,6 +4,8 @@ import ( "fmt" "path/filepath" "strings" + + "github.com/flanksource/gavel/sdk/status" ) type GroupBy string @@ -13,14 +15,23 @@ const ( GroupRepo GroupBy = "repo" GroupWorktree GroupBy = "worktree" GroupClaudeSession GroupBy = "claude-session" - KindBucket = "bucket" + // GroupSlots buckets by the gavel slot pool a group's gavel process holds. + GroupSlots GroupBy = "slots" +) + +const ( + KindBucket = "bucket" + // KindGavelTask and KindGavelItem are the rows WithGavelTasks adds: a + // gavel task group and one of its running tasks. + KindGavelTask = "gavel-task" + KindGavelItem = "gavel-item" ) -var Groupings = []GroupBy{GroupTree, GroupRepo, GroupWorktree, GroupClaudeSession} +var Groupings = []GroupBy{GroupTree, GroupRepo, GroupWorktree, GroupClaudeSession, GroupSlots} func (by GroupBy) Validate() error { switch by { - case "", GroupTree, GroupRepo, GroupWorktree, GroupClaudeSession: + case "", GroupTree, GroupRepo, GroupWorktree, GroupClaudeSession, GroupSlots: return nil default: return fmt.Errorf("group-by %q: expected one of %v", by, Groupings) @@ -38,11 +49,19 @@ func (f Frame) Grouped(by GroupBy) (Frame, error) { if err != nil { return Frame{}, err } + identities := map[string]bucket{} + if by == GroupSlots { + identities = slotBuckets(f.Groups) + } else { + for _, group := range f.Groups { + identity, label := bucketIdentity(by, contexts[group.Key]) + identities[group.Key] = bucket{identity, label} + } + } buckets := map[string]string{} skipped := map[string]bool{} for _, group := range f.Groups { - identity, _ := bucketIdentity(by, contexts[group.Key]) - buckets[group.Key] = "bucket:" + string(by) + ":" + identity + buckets[group.Key] = "bucket:" + string(by) + ":" + identities[group.Key].identity skipped[group.Key] = group.PID == 0 && (group.Kind == KindOther || group.Kind == KindApp) } var projected []GroupSample @@ -53,8 +72,8 @@ func (f Frame) Grouped(by GroupBy) (Frame, error) { } key := buckets[group.Key] if !created[key] { - identity, label := bucketIdentity(by, contexts[group.Key]) - projected = append(projected, GroupSample{Key: key, Kind: KindBucket, Name: label, Target: identity, Command: identity, SampledAt: f.SampledAt, Tick: f.Tick}) + id := identities[group.Key] + projected = append(projected, GroupSample{Key: key, Kind: KindBucket, Name: id.label, Target: id.identity, Command: id.identity, SampledAt: f.SampledAt, Tick: f.Tick}) created[key] = true } if group.ParentKey == "" || buckets[group.ParentKey] != key || skipped[group.ParentKey] { @@ -67,6 +86,69 @@ func (f Frame) Grouped(by GroupBy) (Frame, error) { return f, nil } +// bucket is what a group is bucketed by, and the bucket's name. +type bucket struct{ identity, label string } + +// slotBuckets puts each group in the slot pool its gavel process holds — a +// project pool before the global one — or, holding none, the pool it waits +// for. A group with no gavel status of its own takes its parent's bucket, so +// a gavel's tests share its pool. +func slotBuckets(groups []GroupSample) map[string]bucket { + byKey := map[string]GroupSample{} + for _, group := range groups { + byKey[group.Key] = group + } + resolved := map[string]bucket{} + // ResolveContexts has refused a cycle by now. + var resolve func(key string) bucket + resolve = func(key string) bucket { + if found, ok := resolved[key]; ok { + return found + } + group, ok := byKey[key] + if !ok { + return bucket{"", "Unattributed"} + } + found, own := slotBucket(group.Gavel) + if !own { + found = resolve(group.ParentKey) + } + resolved[key] = found + return found + } + for _, group := range groups { + resolve(group.Key) + } + return resolved +} + +// slotBucket is the pool a gavel status's leases name, and whether it has any. +func slotBucket(s *status.Status) (bucket, bool) { + if s == nil || len(s.Slots) == 0 { + return bucket{}, false + } + best := s.Slots[0] + rank := func(l status.Lease) int { + score := 0 + if l.State != status.LeaseWaiting { + score += 2 + } + if l.Pool != "global" { + score++ + } + return score + } + for _, lease := range s.Slots[1:] { + if rank(lease) > rank(best) { + best = lease + } + } + if best.State == status.LeaseWaiting { + return bucket{best.Pool + ":waiting", poolLabel(best.Pool, best.Project) + " (waiting)"}, true + } + return bucket{best.Pool, poolLabel(best.Pool, best.Project)}, true +} + func bucketIdentity(by GroupBy, c Context) (string, string) { switch by { case GroupRepo: diff --git a/pkg/tracing/groups/grouping_test.go b/pkg/tracing/groups/grouping_test.go index 7404a53..ecf3bd5 100644 --- a/pkg/tracing/groups/grouping_test.go +++ b/pkg/tracing/groups/grouping_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "os" + "github.com/flanksource/gavel/sdk/status" "github.com/flanksource/telltale/pkg/tracing/groups" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -114,4 +115,43 @@ var _ = Describe("Frame.Grouped", func() { Expect(projected.Groups[0].Total.NetRx).To(HaveValue(Equal(float64(1024)))) Expect(projected.Groups[2].Total.NetRx).To(BeNil()) }) + + It("buckets gavel groups by the slot pool they hold, their tests with them, and queued ones by the pool they wait for", func() { + held := &status.Status{Slots: []status.Lease{ + {Pool: "global", Limit: 4, State: status.LeaseHeld}, + {Pool: "project/abc", Project: "/src/acme/.git", Limit: 2, State: status.LeaseHeld}, + }} + queued := &status.Status{Slots: []status.Lease{{Pool: "project/abc", Project: "/src/acme/.git", Limit: 2, Index: -1, State: status.LeaseWaiting}}} + usage := groups.Usage{Processes: 1, RSSBytes: 10} + frame.Groups = []groups.GroupSample{ + {Key: "agent", Name: "claude", Kind: groups.KindAgent, PID: 1, Self: usage}, + {Key: "gavel-a", ParentKey: "agent", Name: "gavel", Kind: groups.KindTest, PID: 2, Gavel: held, Self: usage}, + {Key: "go-test", ParentKey: "gavel-a", Name: "go-test", Kind: groups.KindTest, PID: 3, Self: usage}, + {Key: "gavel-b", ParentKey: "agent", Name: "gavel", Kind: groups.KindTest, PID: 4, Gavel: queued, Self: usage}, + } + projected, err := frame.Grouped(groups.GroupSlots) + Expect(err).NotTo(HaveOccurred()) + parents := map[string]string{} + for _, group := range projected.Groups { + parents[group.Key] = group.ParentKey + } + Expect(parents).To(Equal(map[string]string{ + "bucket:slots:": "", + "agent": "bucket:slots:", + "bucket:slots:project/abc": "", + "gavel-a": "bucket:slots:project/abc", + "go-test": "gavel-a", + "bucket:slots:project/abc:waiting": "", + "gavel-b": "bucket:slots:project/abc:waiting", + })) + names := map[string]string{} + for _, group := range projected.Groups { + if group.Kind == groups.KindBucket { + names[group.Key] = group.Name + } + } + Expect(names).To(Equal(map[string]string{ + "bucket:slots:": "Unattributed", "bucket:slots:project/abc": "acme", "bucket:slots:project/abc:waiting": "acme (waiting)", + })) + }) }) diff --git a/pkg/tracing/groups/hogs.go b/pkg/tracing/groups/hogs.go new file mode 100644 index 0000000..f91cd9f --- /dev/null +++ b/pkg/tracing/groups/hogs.go @@ -0,0 +1,82 @@ +package groups + +import ( + "maps" + "strings" + "time" +) + +// Hogs keeps each resource's pin expiry, measured from its last excessive sample. +type Hogs map[string]time.Time + +const HogRetention = 120 * time.Second + +func (h Hogs) Pinned(at time.Time) bool { + for _, until := range h { + if until.After(at) { + return true + } + } + return false +} + +func (h Hogs) Label(at time.Time) string { + var resources []string + for _, resource := range []string{"CPU", "memory", "swap", "compressed"} { + if h[resource].After(at) { + resources = append(resources, resource) + } + } + if len(resources) == 0 { + return "" + } + return strings.Join(resources, "/") + " hog" +} + +func (h Hogs) merge(other Hogs) Hogs { + merged := maps.Clone(h) + for resource, until := range other { + if until.After(merged[resource]) { + if merged == nil { + merged = Hogs{} + } + merged[resource] = until + } + } + return merged +} + +type hogTracker struct { + last map[string]Hogs +} + +func (h *hogTracker) observe(readings map[string]Reading, host HostUsage, at time.Time) { + next := make(map[string]Hogs, len(readings)) + for key, reading := range readings { + var recent Hogs + for resource, until := range h.last[key] { + if until.After(at) { + if recent == nil { + recent = Hogs{} + } + recent[resource] = until + } + } + for resource, hogging := range map[string]bool{ + "CPU": host.Cores > 0 && reading.CPUPercent != nil && *reading.CPUPercent > 25*float64(host.Cores), + "memory": host.MemoryTotalBytes > 0 && reading.RSSBytes > host.MemoryTotalBytes/4, + "swap": host.SwapTotalBytes > 0 && reading.SwapBytes != nil && *reading.SwapBytes > host.SwapTotalBytes/4, + "compressed": host.MemoryTotalBytes > 0 && reading.CompressedBytes != nil && *reading.CompressedBytes > host.MemoryTotalBytes/4, + } { + if hogging { + if recent == nil { + recent = Hogs{} + } + recent[resource] = at.Add(HogRetention) + } + } + reading.Hogs = recent + readings[key], next[key] = reading, recent + } + h.last = next +} diff --git a/pkg/tracing/groups/hogs_test.go b/pkg/tracing/groups/hogs_test.go new file mode 100644 index 0000000..091c831 --- /dev/null +++ b/pkg/tracing/groups/hogs_test.go @@ -0,0 +1,87 @@ +package groups + +import ( + "time" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = ginkgo.Describe("resource hogs", func() { + start := time.Date(2026, 11, 6, 10, 0, 0, 0, time.UTC) + host := HostUsage{Cores: 8, MemoryTotalBytes: 16 << 30, SwapTotalBytes: 4 << 30} + ginkgo.DescribeTable("detects excessive use of each resource", func(reading Reading, resource string) { + readings := map[string]Reading{"worker@1": reading} + var tracker hogTracker + tracker.observe(readings, host, start) + Expect(readings["worker@1"].Hogs).To(Equal(Hogs{resource: start.Add(HogRetention)})) + }, + ginkgo.Entry("CPU across all cores", Reading{CPUPercent: new(201.0)}, "CPU"), + ginkgo.Entry("resident memory", Reading{RSSBytes: 5 << 30}, "memory"), + ginkgo.Entry("swap", Reading{SwapBytes: new(uint64(2 << 30))}, "swap"), + ginkgo.Entry("compressed memory", Reading{CompressedBytes: new(uint64(5 << 30))}, "compressed"), + ) + + ginkgo.It("uses strict thresholds and never flags unavailable capacities", func() { + readings := map[string]Reading{"worker@1": {CPUPercent: new(200.0), RSSBytes: 4 << 30, SwapBytes: new(uint64(1 << 30)), CompressedBytes: new(uint64(4 << 30))}} + var tracker hogTracker + tracker.observe(readings, host, start) + Expect(readings["worker@1"].Hogs).To(BeEmpty()) + tracker.observe(readings, HostUsage{}, start) + Expect(readings["worker@1"].Hogs).To(BeEmpty()) + }) + + ginkgo.It("retains a spike for exactly 120 seconds and extends only the resource still hogging", func() { + var tracker hogTracker + readings := map[string]Reading{"worker@1": {CPUPercent: new(201.0), RSSBytes: 5 << 30}} + tracker.observe(readings, host, start) + first := readings["worker@1"].Hogs + readings["worker@1"] = Reading{RSSBytes: 5 << 30} + tracker.observe(readings, host, start.Add(time.Minute)) + Expect(first).To(Equal(Hogs{"CPU": start.Add(HogRetention), "memory": start.Add(HogRetention)}), "previous frames must not mutate") + readings["worker@1"] = Reading{} + tracker.observe(readings, host, start.Add(HogRetention)) + Expect(readings["worker@1"].Hogs).To(Equal(Hogs{"memory": start.Add(3 * time.Minute)})) + Expect(readings["worker@1"].Hogs.Pinned(start.Add(3*time.Minute - time.Nanosecond))).To(BeTrue()) + tracker.observe(readings, host, start.Add(3*time.Minute)) + Expect(readings["worker@1"].Hogs.Pinned(start.Add(3 * time.Minute))).To(BeFalse()) + }) + + ginkgo.It("forgets exited processes and does not transfer pins to a reused pid", func() { + var tracker hogTracker + readings := map[string]Reading{"100@1": {RSSBytes: 5 << 30}} + tracker.observe(readings, host, start) + readings = map[string]Reading{"100@2": {}} + tracker.observe(readings, host, start.Add(time.Second)) + Expect(readings["100@2"].Hogs).To(BeEmpty()) + Expect(tracker.last).NotTo(HaveKey("100@1")) + }) + + ginkgo.It("rolls up swap, compressed memory and descendant pins without double counting", func() { + hogs := Hogs{"swap": start.Add(HogRetention)} + groups := Rollup([]GroupSample{ + {Key: "parent", Self: Usage{Processes: 1, RSSBytes: 100, SwapBytes: new(uint64(10)), CompressedBytes: new(uint64(20))}}, + {Key: "child", ParentKey: "parent", Self: Usage{Processes: 1, RSSBytes: 200, SwapBytes: new(uint64(30)), CompressedBytes: new(uint64(40)), Hogs: hogs}}, + }) + Expect(groups[0].Total).To(Equal(Usage{Processes: 2, RSSBytes: 300, SwapBytes: new(uint64(40)), CompressedBytes: new(uint64(60)), Hogs: hogs})) + Expect(groups[0].Self.Hogs).To(BeNil()) + }) + + ginkgo.It("pins the branch containing a hog ahead of every sort direction", func() { + frame := Frame{SampledAt: start, Groups: []GroupSample{ + {Key: "busy", Name: "a", Total: Usage{CPUPercent: new(100.0), RSSBytes: 1000}}, + {Key: "hog", Name: "z", Total: Usage{Hogs: Hogs{"memory": start.Add(HogRetention)}}}, + }} + for _, by := range Sorts { + for _, reverse := range []bool{false, true} { + sorted, err := frame.Sorted(Order{By: by, Reverse: reverse}) + Expect(err).NotTo(HaveOccurred()) + Expect(sorted.Groups[0].Key).To(Equal("hog"), "%s reverse=%t", by, reverse) + } + } + frame.SampledAt = start.Add(HogRetention) + sorted, err := frame.Sorted(Order{By: SortMemory}) + Expect(err).NotTo(HaveOccurred()) + Expect(sorted.Groups[0].Key).To(Equal("busy")) + }) +}) diff --git a/pkg/tracing/groups/memory.go b/pkg/tracing/groups/memory.go new file mode 100644 index 0000000..e11367b --- /dev/null +++ b/pkg/tracing/groups/memory.go @@ -0,0 +1,68 @@ +package groups + +import ( + "bufio" + "fmt" + "io" + "math" + "strconv" + "strings" +) + +func parseCompressed(reader io.Reader) (map[int]*uint64, error) { + values := map[int]*uint64{} + scanner := bufio.NewScanner(reader) + header := false + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if strings.Join(fields, " ") == "PID CMPRS" { + header = true + continue + } + if !header || len(fields) == 0 { + continue + } + if len(fields) != 2 { + return nil, fmt.Errorf("compressed memory: expected PID CMPRS, got %q", scanner.Text()) + } + pid, err := strconv.Atoi(fields[0]) + if err != nil || pid < 0 { + return nil, fmt.Errorf("compressed memory: invalid PID %q", fields[0]) + } + size, err := parseMemorySize(fields[1]) + if err != nil { + return nil, err + } + if pid > 0 { + values[pid] = &size + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("compressed memory listing: %w", err) + } + if !header || len(values) == 0 { + return nil, fmt.Errorf("compressed memory listing has no PID CMPRS readings") + } + return values, nil +} + +func parseMemorySize(raw string) (uint64, error) { + if len(raw) < 2 { + return 0, fmt.Errorf("compressed memory: invalid size %q", raw) + } + unit := strings.IndexByte("BKMGTPE", raw[len(raw)-1]) + value, err := strconv.ParseFloat(raw[:len(raw)-1], 64) + value *= math.Pow(1024, float64(unit)) + if err != nil || unit < 0 || math.IsNaN(value) || value < 0 || value >= math.Exp2(64) { + return 0, fmt.Errorf("compressed memory: invalid size %q", raw) + } + return uint64(value), nil +} + +// MemorySize preserves unavailable readings as a dash. +func MemorySize(value *uint64) string { + if value == nil { + return "-" + } + return Size(float64(*value)) +} diff --git a/pkg/tracing/groups/memory_darwin.go b/pkg/tracing/groups/memory_darwin.go new file mode 100644 index 0000000..55b2b42 --- /dev/null +++ b/pkg/tracing/groups/memory_darwin.go @@ -0,0 +1,36 @@ +package groups + +import ( + "bytes" + "context" + "fmt" + "os/exec" + "strings" +) + +func readMemory(ctx context.Context, classified []Group, readings map[string]Reading) error { + var stdout, stderr bytes.Buffer + // macOS accounts compressed and swapped-out pages together in CMPRS; + // it does not expose a separate per-process disk-swap total. + cmd := exec.CommandContext(ctx, "/usr/bin/top", "-l", "1", "-stats", "pid,cmprs", "-R", "-F") + cmd.Stdout, cmd.Stderr = &stdout, &stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("read compressed memory: %w: %s", err, strings.TrimSpace(stderr.String())) + } + compressed, err := parseCompressed(&stdout) + if err != nil { + return err + } + for _, group := range classified { + for _, member := range group.Members { + key := keyOf(member) + reading := readings[key] + reading.CompressedBytes = compressed[member.PID] + if reading.CompressedBytes == nil { + reading.MemoryError = fmt.Sprintf("PID %d absent from compressed-memory listing", member.PID) + } + readings[key] = reading + } + } + return nil +} diff --git a/pkg/tracing/groups/memory_linux.go b/pkg/tracing/groups/memory_linux.go new file mode 100644 index 0000000..04b104a --- /dev/null +++ b/pkg/tracing/groups/memory_linux.go @@ -0,0 +1,29 @@ +package groups + +import ( + "context" + "fmt" + + "github.com/shirou/gopsutil/v4/process" +) + +func readMemory(ctx context.Context, classified []Group, readings map[string]Reading) error { + for _, group := range classified { + for _, member := range group.Members { + if err := ctx.Err(); err != nil { + return err + } + key := keyOf(member) + reading := readings[key] + p := process.Process{Pid: int32(member.PID)} + memory, err := p.MemoryInfoWithContext(ctx) + if err != nil { + reading.MemoryError = fmt.Sprintf("PID %d swap: %v", member.PID, err) + } else { + reading.SwapBytes = &memory.Swap + } + readings[key] = reading + } + } + return nil +} diff --git a/pkg/tracing/groups/memory_other.go b/pkg/tracing/groups/memory_other.go new file mode 100644 index 0000000..ae8bd5b --- /dev/null +++ b/pkg/tracing/groups/memory_other.go @@ -0,0 +1,10 @@ +//go:build !darwin && !linux + +package groups + +import "context" + +// Per-process swap and compressed memory are unavailable on other platforms. +func readMemory(ctx context.Context, _ []Group, _ map[string]Reading) error { + return ctx.Err() +} diff --git a/pkg/tracing/groups/memory_test.go b/pkg/tracing/groups/memory_test.go new file mode 100644 index 0000000..197ada6 --- /dev/null +++ b/pkg/tracing/groups/memory_test.go @@ -0,0 +1,31 @@ +package groups + +import ( + "strings" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = ginkgo.Describe("compressed memory listing", func() { + ginkgo.It("reads macOS top's compressed bytes independently of resident memory and swap", func() { + listing := "Processes: 3 total\nPhysMem: 16G used\n\nPID CMPRS\n100 0B\n200 1.5G\n300 8384K\n0 0B\n" + values, err := parseCompressed(strings.NewReader(listing)) + Expect(err).NotTo(HaveOccurred()) + Expect(values).To(Equal(map[int]*uint64{100: new(uint64(0)), 200: new(uint64(1536 << 20)), 300: new(uint64(8384 << 10))})) + }) + + ginkgo.DescribeTable("refuses malformed listings", func(listing string) { + _, err := parseCompressed(strings.NewReader(listing)) + Expect(err).To(HaveOccurred()) + }, + ginkgo.Entry("no header", "100 20M\n"), + ginkgo.Entry("no processes", "PID CMPRS\n"), + ginkgo.Entry("invalid units", "PID CMPRS\n100 20X\n"), + ginkgo.Entry("invalid pid", "PID CMPRS\nworker 20M\n"), + ginkgo.Entry("missing column", "PID CMPRS\n100\n"), + ginkgo.Entry("negative memory", "PID CMPRS\n100 -1M\n"), + ginkgo.Entry("non-finite memory", "PID CMPRS\n100 NaNG\n"), + ginkgo.Entry("overflow", "PID CMPRS\n100 999999999999999999G\n"), + ) +}) diff --git a/pkg/tracing/groups/render.go b/pkg/tracing/groups/render.go index b364847..32f4db0 100644 --- a/pkg/tracing/groups/render.go +++ b/pkg/tracing/groups/render.go @@ -14,16 +14,18 @@ import ( // The columns a Frame's siblings are sorted by, by their Total but for the // name. const ( - SortCPU = "cpu" - SortMemory = "mem" - SortIO = "io" - SortNet = "net" - SortProcesses = "procs" - SortName = "name" + SortCPU = "cpu" + SortMemory = "mem" + SortSwap = "swap" + SortCompressed = "compressed" + SortIO = "io" + SortNet = "net" + SortProcesses = "procs" + SortName = "name" ) // Sorts are every column a Frame sorts by. -var Sorts = []string{SortCPU, SortMemory, SortIO, SortNet, SortProcesses, SortName} +var Sorts = []string{SortCPU, SortMemory, SortSwap, SortCompressed, SortIO, SortNet, SortProcesses, SortName} // Order is the column siblings are sorted by: a usage column highest first, // the name A first, unless Reverse turns it. @@ -57,7 +59,19 @@ func (f Frame) Sorted(order Order) (Frame, error) { var walk func(parent string) walk = func(parent string) { siblings := children[parent] - slices.SortStableFunc(siblings, compare) + slices.SortStableFunc(siblings, func(a, b GroupSample) int { + // A gavel's task and running-task rows come first, in gavel's order. + if a.Task != nil || b.Task != nil { + return cmp.Compare(taskRank(a), taskRank(b)) + } + if a.Total.Hogs.Pinned(f.SampledAt) != b.Total.Hogs.Pinned(f.SampledAt) { + if a.Total.Hogs.Pinned(f.SampledAt) { + return -1 + } + return 1 + } + return compare(a, b) + }) for _, group := range siblings { sorted.Groups = append(sorted.Groups, group) walk(group.Key) @@ -87,6 +101,20 @@ func comparer(order Order) (func(a, b GroupSample) int, error) { value = func(u Usage) float64 { return deref(u.CPUPercent) } case SortMemory: value = func(u Usage) float64 { return float64(u.RSSBytes) } + case SortSwap: + value = func(u Usage) float64 { + if u.SwapBytes == nil { + return 0 + } + return float64(*u.SwapBytes) + } + case SortCompressed: + value = func(u Usage) float64 { + if u.CompressedBytes == nil { + return 0 + } + return float64(*u.CompressedBytes) + } case SortIO: value = func(u Usage) float64 { return deref(u.DiskRead) + deref(u.DiskWrite) } case SortNet: @@ -110,8 +138,8 @@ const nameWidth = 44 // Pretty is the frame as a top table: each group's Total, indented under its // parent, with where it runs and what started it. func (f Frame) Pretty() api.Text { - text := api.Text{}.AddText(fmt.Sprintf("%-*s %5s %7s %8s %9s %9s %9s %9s %5s %-24s %s", - nameWidth, "GROUP", "PROCS", "CPU%", "RSS", "DISK R/s", "DISK W/s", "NET IN/s", "NET OUT/s", "SOCK", "ORIGIN", "CONTEXT"), "font-bold") + text := api.Text{}.AddText(fmt.Sprintf("%-*s %5s %7s %8s %8s %8s %9s %9s %9s %9s %5s %-24s %s", + nameWidth, "GROUP", "PROCS", "CPU%", "RSS", "CMPRS", "SWAP", "DISK R/s", "DISK W/s", "NET IN/s", "NET OUT/s", "SOCK", "ORIGIN", "CONTEXT"), "font-bold") depth := map[string]int{} for _, group := range f.Groups { level, ok := depth[group.ParentKey] @@ -121,11 +149,20 @@ func (f Frame) Pretty() api.Text { depth[group.Key] = level usage := group.Total name := strings.Repeat(" ", level) + Label(group) - text = text.NewLine().AddText(fmt.Sprintf("%-*s %5d %7s %8s %9s %9s %9s %9s %5s %-24s ", - nameWidth, Truncate(name, nameWidth), usage.Processes, Percent(usage.CPUPercent), Size(float64(usage.RSSBytes)), + if label := usage.Hogs.Label(f.SampledAt); label != "" { + name = "! " + label + " · " + name + } + text = text.NewLine().AddText(fmt.Sprintf("%-*s %5d %7s %8s %8s %8s %9s %9s %9s %9s %5s %-24s ", + nameWidth, Truncate(name, nameWidth), usage.Processes, Percent(usage.CPUPercent), Size(float64(usage.RSSBytes)), MemorySize(usage.CompressedBytes), MemorySize(usage.SwapBytes), Rate(usage.DiskRead), Rate(usage.DiskWrite), Rate(usage.NetRx), Rate(usage.NetTx), Count(usage.Sockets), Truncate(OriginText(group.Origin), 24)), kindStyle(group.Kind)). AddText(ContextText(group.Context), "text-gray-500") + if progress := GavelText(group.Gavel); progress != "" { + text = text.AddText(" · "+progress, "text-green-600") + } + } + if len(f.Slots) > 0 { + text = text.NewLine().AddText("slots "+SlotsText(f.Slots), "text-gray-500") } return text.NewLine().AddText(fmt.Sprintf("%d groups · tick %d · %s", len(f.Groups), f.Tick, f.SampledAt.Local().Format("15:04:05")), "text-gray-500") } @@ -203,6 +240,21 @@ func OriginText(o Origin) string { } // Truncate cuts text to width, ending it with an ellipsis where it cut. +// macExecutable is where an app bundle keeps its executables; the name that +// follows can hold spaces, as Google Chrome Helper (Renderer) does. +const macExecutable = "/Contents/MacOS/" + +// ShortCommand is a command line without its executable's directory; an +// app bundle's executable starts at its name, after Contents/MacOS. +func ShortCommand(command string) string { + if index := strings.LastIndex(command, macExecutable); index >= 0 { + return command[index+len(macExecutable):] + } + fields := strings.SplitN(command, " ", 2) + fields[0] = filepath.Base(fields[0]) + return strings.Join(fields, " ") +} + func Truncate(text string, width int) string { runes := []rune(text) if len(runes) <= width { diff --git a/pkg/tracing/groups/snapshot.go b/pkg/tracing/groups/snapshot.go index cd9f1c8..5b1695e 100644 --- a/pkg/tracing/groups/snapshot.go +++ b/pkg/tracing/groups/snapshot.go @@ -2,13 +2,19 @@ package groups import ( "context" + clickyprocess "github.com/flanksource/clicky/process" + "github.com/flanksource/telltale/pkg/tracing/process" + "maps" "slices" ) // Snapshot holds attribution and measurements from the same unfiltered tick. type Snapshot struct { - Frame Frame `json:"frame"` - Processes []ProcessSample `json:"processes"` + Frame Frame `json:"frame"` + Processes []ProcessSample `json:"processes"` + Observed []process.Observed `json:"-"` + Descriptors map[int][]clickyprocess.Descriptor `json:"-"` + IO map[int]clickyprocess.Process `json:"-"` } func (t *Top) Snapshot(ctx context.Context) (Snapshot, error) { @@ -18,7 +24,7 @@ func (t *Top) Snapshot(ctx context.Context) (Snapshot, error) { if err != nil { return Snapshot{}, err } - return Snapshot{Frame: frame, Processes: append([]ProcessSample(nil), t.processes...)}, nil + return Snapshot{Frame: frame, Processes: slices.Clone(t.processes), Observed: slices.Clone(t.observed), Descriptors: maps.Clone(t.held), IO: maps.Clone(t.io)}, nil } func (t *Top) Tick(ctx context.Context) (Frame, error) { @@ -29,7 +35,7 @@ func (t *Top) Tick(ctx context.Context) (Frame, error) { shown := snapshot.Frame.Groups[:0] kept := map[string]bool{} for _, group := range snapshot.Frame.Groups { - if group.Kind == KindOther && !t.options.All { + if group.Kind == KindOther && !t.options.All && !group.Total.Hogs.Pinned(snapshot.Frame.SampledAt) { continue } if len(t.options.Groups) > 0 && !slices.Contains(t.options.Groups, group.Name) && !kept[group.ParentKey] { diff --git a/pkg/tracing/groups/summary.go b/pkg/tracing/groups/summary.go new file mode 100644 index 0000000..fd09179 --- /dev/null +++ b/pkg/tracing/groups/summary.go @@ -0,0 +1,29 @@ +package groups + +import "iter" + +func (kind) Summarize(rows iter.Seq[GroupSample]) (map[string]any, error) { + var tick int64 + var usage Usage + groups := 0 + for row := range rows { + if row.Tick < tick { + continue + } + if row.Tick > tick { + tick = row.Tick + usage = Usage{} + groups = 0 + } + groups++ + usage.merge(row.Self) + } + result := map[string]any{"groups": groups, "processes": usage.Processes, "rssBytes": usage.RSSBytes} + if usage.CPUPercent != nil { + result["cpuPercent"] = *usage.CPUPercent + } + if usage.SwapBytes != nil { + result["swapBytes"] = *usage.SwapBytes + } + return result, nil +} diff --git a/pkg/tracing/groups/top.go b/pkg/tracing/groups/top.go index 77a13fc..d6f0181 100644 --- a/pkg/tracing/groups/top.go +++ b/pkg/tracing/groups/top.go @@ -12,6 +12,7 @@ import ( "time" clickyprocess "github.com/flanksource/clicky/process" + "github.com/flanksource/gavel/sdk/status" "github.com/flanksource/telltale/pkg/datadir" "github.com/flanksource/telltale/pkg/tracing/process" ) @@ -32,6 +33,16 @@ type GroupSample struct { Origin Origin `json:"origin" filter:"none"` Self Usage `json:"self" filter:"none"` Total Usage `json:"total" filter:"none"` + // Gavel is the progress and slots a gavel group's process publishes, and + // GavelError why it could not be read. + Gavel *status.Status `json:"gavel,omitempty" filter:"none" pretty:"hide"` + GavelError string `json:"gavelError,omitempty" filter:"none" pretty:"hide"` + // SkipAncestry is the group's rule's: its tasks claim their processes' + // groups by pid before ancestry does. + SkipAncestry bool `json:"skipAncestry,omitempty" filter:"none" pretty:"hide"` + // Task is what a row WithGavelTasks adds shows: a task group's progress, + // or one running task. It is never recorded. + Task *GavelTask `json:"-" filter:"none" pretty:"hide"` } // Frame is every group shown at one tick, parents before their children, @@ -43,6 +54,8 @@ type Frame struct { // Overhead is what reading the frame cost, when read live. Overhead *Overhead `json:"overhead,omitempty"` Groups []GroupSample `json:"groups"` + // Slots are the gavel slot pools the gavel groups hold or wait for. + Slots []SlotPool `json:"slots,omitempty"` } // ProcessSample is one process at one tick: who runs it, the group it is in, @@ -77,7 +90,9 @@ func (p ProcessSample) holds(query string) bool { // TopOptions choose what a Top reads and shows. type TopOptions struct { - Catalog Catalog + Catalog Catalog + RootPID int + SkipDescriptors bool // All shows, and reads the disk and descriptors of, the processes in no // tool's group too. All bool @@ -86,6 +101,9 @@ type TopOptions struct { // Net reads each process's network bytes with nettop, every Interval. Net bool Interval time.Duration + // GavelCache is where gavel processes publish their status; defaults to + // gavel's user cache directory. + GavelCache string } // Top reads the host's process groups at each Tick. @@ -100,9 +118,14 @@ type Top struct { self int overhead overheadReader host hostReader + hogs hogTracker net *NetMonitor tick int64 processes []ProcessSample + gavel *gavelStatuses + observed []process.Observed + held map[int][]clickyprocess.Descriptor + io map[int]clickyprocess.Process } // NewTop is a Top over sampler, refusing a group no rule names; with Net it @@ -113,7 +136,15 @@ func NewTop(ctx context.Context, sampler *process.Sampler, options TopOptions) ( return nil, fmt.Errorf("group %q is no rule of the catalog", name) } } - top := &Top{sampler: sampler, options: options, rates: NewRates(), contexts: newContexts(), brew: BrewPrefixes(), self: os.Getpid()} + if options.GavelCache == "" { + cache, err := status.CacheDir() + if err != nil { + return nil, fmt.Errorf("gavel status directory: %w", err) + } + options.GavelCache = cache + } + top := &Top{sampler: sampler, options: options, rates: NewRates(), contexts: newContexts(), brew: BrewPrefixes(), self: os.Getpid(), + gavel: newGavelStatuses(options.GavelCache)} if options.Net { monitor, err := StartNetMonitor(ctx, options.Interval) if err != nil { @@ -140,12 +171,13 @@ func (t *Top) Close() { if t.net != nil { t.net.Close() } + t.gavel.close() } // Tick lists the host, sorts it into groups and reads what each uses. func (t *Top) collect(ctx context.Context) (Frame, error) { started := time.Now() - listing, at, err := t.sampler.Observe(ctx, 0) + listing, at, err := t.sampler.Observe(ctx, t.options.RootPID) if err != nil { return Frame{}, err } @@ -162,10 +194,12 @@ func (t *Top) collect(ctx context.Context) (Frame, error) { if err != nil { return Frame{}, err } + t.observed, t.held = listing, held host, err := t.host.read(ctx) if err != nil { return Frame{}, err } + t.hogs.observe(readings, host, at) totals := Aggregate(classified, readings) contexts := t.contexts.of(ctx, classified, held) t.tick++ @@ -186,14 +220,33 @@ func (t *Top) collect(ctx context.Context) (Frame, error) { return Frame{}, err } frame := Frame{SampledAt: at, Tick: t.tick, Host: &host, Overhead: &overhead} + var gavelPIDs []int + for _, group := range classified { + if group.Rule.Name == gavelRule { + gavelPIDs = append(gavelPIDs, group.Anchor.PID) + } + } + gavels := t.gavel.of(gavelPIDs) + var published []status.Status for _, group := range classified { - frame.Groups = append(frame.Groups, GroupSample{ + sample := GroupSample{ SampledAt: at, Tick: t.tick, Key: group.Key, ParentKey: group.ParentKey, Name: group.Rule.Name, Kind: group.Rule.Kind, Target: group.Target, PID: group.Anchor.PID, Command: group.Anchor.Command, Context: contexts[group.Key], Origin: group.Origin, - Self: totals[group.Key].Self, Total: totals[group.Key].Total, - }) + Self: totals[group.Key].Self, Total: totals[group.Key].Total, SkipAncestry: group.Rule.SkipAncestry, + } + if reading, ok := gavels[group.Anchor.PID]; ok && group.Rule.Name == gavelRule { + sample.Gavel = reading.Status + if reading.Err != nil { + sample.GavelError = reading.Err.Error() + } + if reading.Status != nil { + published = append(published, *reading.Status) + } + } + frame.Groups = append(frame.Groups, sample) } + frame.Slots = SlotPools(published) return frame, nil } @@ -201,13 +254,17 @@ func (t *Top) collect(ctx context.Context) (Frame, error) { func (t *Top) Processes(filter ProcessFilter) []ProcessSample { t.mu.Lock() defer t.mu.Unlock() + return FilterProcesses(t.processes, filter) +} + +func FilterProcesses(processes []ProcessSample, filter ProcessFilter) []ProcessSample { picked := map[int]bool{} if filter.PID != 0 { picked[filter.PID] = true // Parents may be listed after their children: grow the subtree until it holds. for grew := true; grew; { grew = false - for _, p := range t.processes { + for _, p := range processes { if picked[p.PPID] && !picked[p.PID] && p.PID != p.PPID { picked[p.PID], grew = true, true } @@ -215,7 +272,7 @@ func (t *Top) Processes(filter ProcessFilter) []ProcessSample { } } var result []ProcessSample - for _, p := range t.processes { + for _, p := range processes { switch { case filter.PID != 0 && !picked[p.PID]: case filter.GroupKey != "" && p.GroupKey != filter.GroupKey: @@ -247,6 +304,13 @@ func (t *Top) read(ctx context.Context, classified []Group, at time.Time) (map[s } } t.rates.Forget(alive) + if len(readings) == 0 { + t.io = nil + return readings, nil, nil + } + if err := readMemory(ctx, classified, readings); err != nil { + return nil, nil, err + } pids := make([]int, 0, len(measured)) for _, p := range measured { pids = append(pids, p.PID) @@ -255,12 +319,21 @@ func (t *Top) read(ctx context.Context, classified []Group, at time.Time) (map[s if err := snapshot.PopulateIO(ctx, pids); err != nil { return nil, nil, err } - descriptors, err := clickyprocess.Descriptors(ctx, pids...) - if err != nil { - return nil, nil, fmt.Errorf("read sockets: %w", err) + t.io = make(map[int]clickyprocess.Process, len(pids)) + for _, p := range snapshot.All() { + t.io[p.PID] = p + } + var descriptors map[int][]clickyprocess.Descriptor + if !t.options.SkipDescriptors && len(pids) > 0 { + var err error + descriptors, err = clickyprocess.Descriptors(ctx, pids...) + if err != nil { + return nil, nil, fmt.Errorf("read sockets: %w", err) + } } var network map[int]NetBytes // nil leaves every process's network unknown if t.net != nil { + var err error if network, err = t.net.Counters(); err != nil { return nil, nil, err } diff --git a/pkg/tracing/groups/usage.go b/pkg/tracing/groups/usage.go index 45ab43b..aa15d6b 100644 --- a/pkg/tracing/groups/usage.go +++ b/pkg/tracing/groups/usage.go @@ -1,6 +1,7 @@ package groups import ( + "strings" "time" ) @@ -15,8 +16,11 @@ type PerSecond struct { // Reading is one process's use at one tick. type Reading struct { - CPUPercent *float64 - RSSBytes uint64 + CPUPercent *float64 + RSSBytes uint64 + SwapBytes, CompressedBytes *uint64 + Hogs Hogs + MemoryError string PerSecond Sockets, Connections *int } @@ -24,9 +28,13 @@ type Reading struct { // Usage is what a set of processes use at one tick: the sums of what is // known of them, nil where nothing is. type Usage struct { - Processes int `json:"processes"` - CPUPercent *float64 `json:"cpuPercent,omitempty" pretty:"label=CPU %"` - RSSBytes uint64 `json:"rssBytes" pretty:"label=RSS,type=bytes"` + Processes int `json:"processes"` + CPUPercent *float64 `json:"cpuPercent,omitempty" pretty:"label=CPU %"` + RSSBytes uint64 `json:"rssBytes" pretty:"label=RSS,type=bytes"` + SwapBytes *uint64 `json:"swapBytes,omitempty" pretty:"label=Swap,type=bytes"` + CompressedBytes *uint64 `json:"compressedBytes,omitempty" pretty:"label=Compressed,type=bytes"` + Hogs Hogs `json:"hogs,omitempty"` + MemoryError string `json:"memoryError,omitempty"` PerSecond Sockets *int `json:"sockets,omitempty"` Connections *int `json:"connections,omitempty"` @@ -36,6 +44,10 @@ func (u *Usage) add(r Reading) { u.Processes++ u.CPUPercent = sum(u.CPUPercent, r.CPUPercent) u.RSSBytes += r.RSSBytes + u.SwapBytes = sumBytes(u.SwapBytes, r.SwapBytes) + u.CompressedBytes = sumBytes(u.CompressedBytes, r.CompressedBytes) + u.Hogs = u.Hogs.merge(r.Hogs) + u.MemoryError = strings.TrimSpace(u.MemoryError + " " + r.MemoryError) u.PerSecond = u.PerSecond.add(r.PerSecond) u.Sockets, u.Connections = sumInt(u.Sockets, r.Sockets), sumInt(u.Connections, r.Connections) } @@ -44,6 +56,10 @@ func (u *Usage) merge(other Usage) { u.Processes += other.Processes u.CPUPercent = sum(u.CPUPercent, other.CPUPercent) u.RSSBytes += other.RSSBytes + u.SwapBytes = sumBytes(u.SwapBytes, other.SwapBytes) + u.CompressedBytes = sumBytes(u.CompressedBytes, other.CompressedBytes) + u.Hogs = u.Hogs.merge(other.Hogs) + u.MemoryError = strings.TrimSpace(u.MemoryError + " " + other.MemoryError) u.PerSecond = u.PerSecond.add(other.PerSecond) u.Sockets, u.Connections = sumInt(u.Sockets, other.Sockets), sumInt(u.Connections, other.Connections) } @@ -77,6 +93,17 @@ func sumInt(a, b *int) *int { return &total } +func sumBytes(a, b *uint64) *uint64 { + if a == nil { + return clone(b) + } + if b == nil { + return a + } + total := *a + *b + return &total +} + func clone[T any](value *T) *T { if value == nil { return nil diff --git a/pkg/tracing/http/agent_internal_test.go b/pkg/tracing/http/agent_internal_test.go new file mode 100644 index 0000000..b717a17 --- /dev/null +++ b/pkg/tracing/http/agent_internal_test.go @@ -0,0 +1,34 @@ +package http_test + +import ( + "encoding/json" + + "github.com/flanksource/telltale/pkg/tracing/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("HTTP agent records", func() { + It("preserves JVM direction, request identity and trace correlation", func() { + record, err := http.DecodeRecord(json.RawMessage(`{"startedDateTime":"2026-01-02T03:04:05Z","time":12,"request":{"method":"GET","url":"https://example.com/orders/7"},"response":{"status":200},"direction":"incoming","requestId":"request-7","route":"/orders/{id}","handler":"example.OrderController#get","threadId":19,"thread":"request-worker","traceId":"trace-7","spanId":"span-7"}`)) + Expect(err).NotTo(HaveOccurred()) + Expect(record.Metadata).To(Equal(http.Metadata{Direction: http.Incoming, RequestID: "request-7", Route: "/orders/{id}", Handler: "example.OrderController#get", ThreadID: 19, Thread: "request-worker", TraceID: "trace-7", SpanID: "span-7"})) + Expect(http.ToEntry(record).Metadata).To(Equal(record.Metadata)) + Expect(record.Method).To(Equal("GET")) + Expect(record.Host).To(Equal("example.com")) + Expect(record.Status).To(Equal(200)) + Expect(record.DurationMs).To(Equal(float64(12))) + }) + + It("does not deduplicate incoming and outgoing copies of the same exchange", func() { + incoming := http.Exchange{Metadata: http.Metadata{Direction: http.Incoming}, Method: "GET", URL: "https://example.com/orders", Status: 200} + outgoing := incoming + outgoing.Direction = http.Outgoing + Expect(http.Key(incoming)).NotTo(Equal(http.Key(outgoing))) + }) + + DescribeTable("rejects invalid direction values", func(direction http.Direction) { + Expect((http.Options{Features: []string{"http"}, Selection: http.Selection{Direction: direction}}).Validate()).NotTo(Succeed()) + }, Entry("unknown", http.Direction("sideways")), Entry("blank whitespace", http.Direction(" "))) +}) diff --git a/pkg/tracing/http/export.go b/pkg/tracing/http/export.go index 1d8168e..7360554 100644 --- a/pkg/tracing/http/export.go +++ b/pkg/tracing/http/export.go @@ -67,20 +67,24 @@ func Export(format string, rows iter.Seq[Exchange], w io.Writer) error { // the encoding of a response body that is not text. type Entry struct { har.Entry - ServerIPAddress string `json:"serverIPAddress,omitempty"` - Comment string `json:"comment,omitempty"` - Tunnelled bool `json:"_tunnelled,omitempty"` - BytesIn int64 `json:"_bytesIn,omitempty"` - BytesOut int64 `json:"_bytesOut,omitempty"` - Encoding string `json:"_encoding,omitempty"` + Metadata Metadata `json:"_telltale,omitzero"` + ServerIPAddress string `json:"serverIPAddress,omitempty"` + Comment string `json:"comment,omitempty"` + Tunnelled bool `json:"_tunnelled,omitempty"` + BytesIn int64 `json:"_bytesIn,omitempty"` + BytesOut int64 `json:"_bytesOut,omitempty"` + Encoding string `json:"_encoding,omitempty"` + RequestEncoding string `json:"_requestEncoding,omitempty"` } // ToEntry is exchange as the HAR entry it was captured from. func ToEntry(exchange Exchange) Entry { return Entry{ Entry: exchange.Entry, + Metadata: exchange.Metadata, ServerIPAddress: exchange.ServerIPAddress, Comment: exchange.Comment, Tunnelled: exchange.Tunnelled, BytesIn: exchange.BytesIn, BytesOut: exchange.BytesOut, Encoding: exchange.Encoding, + RequestEncoding: exchange.RequestEncoding, } } @@ -92,8 +96,10 @@ func Summarize(rows iter.Seq[Exchange]) (map[string]any, error) { methods, statuses, hosts := map[string]int{}, map[string]int{}, map[string]bool{} requests := []map[string]any{} entries, errors := 0, 0 + durationMs := 0.0 for exchange := range rows { entries++ + durationMs += exchange.DurationMs host, path, err := splitURL(exchange.URL) if err != nil { return nil, err @@ -111,6 +117,7 @@ func Summarize(rows iter.Seq[Exchange]) (map[string]any, error) { return map[string]any{ "entries": entries, "errors": errors, "hosts": slices.Sorted(maps.Keys(hosts)), "methods": methods, "statuses": statuses, "requests": requests, + "durationMs": durationMs, }, nil } diff --git a/pkg/tracing/http/httptraffic.go b/pkg/tracing/http/httptraffic.go index fdc74d6..a6ea64f 100644 --- a/pkg/tracing/http/httptraffic.go +++ b/pkg/tracing/http/httptraffic.go @@ -22,7 +22,7 @@ import ( "github.com/flanksource/commons-db/recordstore/recordresults" "github.com/flanksource/telltale/pkg/agentclient" "github.com/flanksource/telltale/pkg/tracing/seam" - commonsdb "github.com/flanksource/telltale/sdk/commonsdb" + _ "github.com/flanksource/telltale/sdk/commonsdb" ) // RequestBody and ResponseBody are where an exchange holds its bodies, as @@ -34,6 +34,9 @@ const ( // Options choose the features whose exchanges a capture records. type Options struct { + Selection + Bodies string `json:"bodies,omitempty" clicky:"title=Bodies,desc=Retained bytes of each body default 256KiB maximum 4MiB; zero keeps none"` + Redact []string `json:"redact,omitempty" clicky:"title=Redact headers,desc=Additional header names to redact"` // Features name the transports to observe, as their HAR levels do: // http, prometheus, loki, opensearch, git, ... Features []string `json:"features" clicky:"required,title=Features,desc=Telltale's transports to observe by their HAR level: http prometheus loki opensearch git ..."` @@ -50,6 +53,12 @@ type Params struct { // Validate is the static check of the features a capture records. func (p Options) Validate() error { + if err := p.Selection.Validate(); err != nil { + return err + } + if _, err := bodyLimit(p.Bodies); err != nil { + return err + } if len(p.Features) == 0 { return errors.New("name at least one feature to observe") } @@ -77,6 +86,7 @@ func (p Options) window() (time.Duration, error) { // Exchange is one outbound request and its response. type Exchange struct { + Metadata tracing.Truncation har.Entry StartedAt time.Time `json:"startedAt" pretty:"label=Started" sort:"startedAt"` @@ -100,7 +110,8 @@ type Exchange struct { BytesOut int64 `json:"bytesOut,omitempty" pretty:"label=Bytes out"` // Encoding is the response content's encoding: base64 for a body that is // not UTF-8 text. - Encoding string `json:"encoding,omitempty" pretty:"hide"` + Encoding string `json:"encoding,omitempty" pretty:"hide"` + RequestEncoding string `json:"requestEncoding,omitempty" pretty:"hide"` // Comment says what the capture left out of the exchange, such as a body // it cut or an event stream still open when it ended. Comment string `json:"comment,omitempty" filter:"text"` @@ -129,7 +140,7 @@ func Key(exchange Exchange) string { } digest.Write([]byte{0}) digest.Write([]byte(exchange.Response.Content.Text)) - return exchange.Method + " " + exchange.URL + " " + strconv.Itoa(exchange.Status) + " " + hex.EncodeToString(digest.Sum(nil)) + return string(exchange.Direction) + " " + exchange.Method + " " + exchange.URL + " " + strconv.Itoa(exchange.Status) + " " + hex.EncodeToString(digest.Sum(nil)) } type traffic struct { @@ -169,15 +180,10 @@ func NewCapture(registry *agentclient.Registry) interface { AgentAvailable(string, target.Target) (bool, error) } { agent := seam.AgentSeam[Params, Exchange]{Probe: "http", Registry: registry, - Target: func(p Params) target.Target { return p.Target }, - ArmParams: func(p Params) (json.RawMessage, error) { return json.Marshal(p) }, - DecodeRecord: func(raw json.RawMessage) (Exchange, error) { - var record commonsdb.HTTPEntry - if err := json.Unmarshal(raw, &record); err != nil { - return Exchange{}, err - } - return FromEntry(record.Feature, record.Entry), nil - }, + Target: func(p Params) target.Target { return p.Target }, + ArmParams: func(p Params) (json.RawMessage, error) { return json.Marshal(p) }, + DecodeRecord: DecodeRecord, + DecodeBlob: decodeBlob, } return traffic{agent: agent} } diff --git a/pkg/tracing/http/httptraffic_spec_test.go b/pkg/tracing/http/httptraffic_spec_test.go index 8094a1f..6af3648 100644 --- a/pkg/tracing/http/httptraffic_spec_test.go +++ b/pkg/tracing/http/httptraffic_spec_test.go @@ -230,9 +230,10 @@ var _ = Describe("http trace kind", func() { Expect(summary).To(MatchAllKeys(Keys{ "count": Equal(3), "entries": Equal(3), "errors": Equal(2), "tunnelled": Equal(0), "bytes_in": BeEquivalentTo(0), "bytes_out": BeEquivalentTo(0), - "hosts": Equal([]string{host}), - "methods": Equal(map[string]int{"POST": 1, "GET": 2}), - "statuses": Equal(map[string]int{"200": 1, "404": 1, "0": 1}), + "durationMs": BeNumerically(">=", 0), + "hosts": Equal([]string{host}), + "methods": Equal(map[string]int{"POST": 1, "GET": 2}), + "statuses": Equal(map[string]int{"200": 1, "404": 1, "0": 1}), "requests": ConsistOf( MatchAllKeys(Keys{"method": Equal("POST"), "url": Equal(server.URL + "/api"), "host": Equal(host), "path": Equal("/api"), "status": Equal(200), "duration_ms": BeNumerically(">=", 0), "error": BeEmpty(), diff --git a/pkg/tracing/http/metadata.go b/pkg/tracing/http/metadata.go new file mode 100644 index 0000000..e0caf2a --- /dev/null +++ b/pkg/tracing/http/metadata.go @@ -0,0 +1,111 @@ +package http + +import ( + "encoding/json" + "fmt" + "path" + "time" + + "github.com/flanksource/commons/properties" + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/tracing/seam" +) + +type Direction string + +const ( + Incoming Direction = "incoming" + Outgoing Direction = "outgoing" + Both Direction = "both" +) + +func (Direction) JSONSchema() map[string]any { + return map[string]any{"type": "string", "enum": []string{string(Incoming), string(Outgoing), string(Both)}, "default": string(Both)} +} + +type Selection struct { + Direction Direction `json:"direction,omitempty" clicky:"title=Direction,desc=Incoming requests outgoing clients or both"` + Hosts []string `json:"hosts,omitempty" clicky:"title=Hosts,desc=Host globs to record"` + Paths []string `json:"paths,omitempty" clicky:"title=Paths,desc=URL path globs to record"` +} + +func (s Selection) Validate() error { + if s.Direction != "" && s.Direction != Incoming && s.Direction != Outgoing && s.Direction != Both { + return fmt.Errorf("direction %q must be incoming, outgoing or both", s.Direction) + } + for _, patterns := range []struct { + name string + values []string + }{{"hosts", s.Hosts}, {"paths", s.Paths}} { + for _, pattern := range patterns.values { + if _, err := path.Match(pattern, ""); err != nil || pattern == "" { + return fmt.Errorf("%s contains an invalid glob %q", patterns.name, pattern) + } + } + } + return nil +} + +type Metadata struct { + Direction Direction `json:"direction,omitempty" filter:"terms"` + RequestID string `json:"requestId,omitempty"` + Route string `json:"route,omitempty" filter:"text"` + Handler string `json:"handler,omitempty" filter:"text"` + Phase string `json:"phase,omitempty" filter:"terms"` + ThreadID int64 `json:"threadId,omitempty"` + Thread string `json:"thread,omitempty"` + TraceID string `json:"traceId,omitempty"` + SpanID string `json:"spanId,omitempty"` + Sampled *bool `json:"sampled,omitempty"` +} + +func bodyLimit(value string) (int64, error) { + if value == "" { + return 256 << 10, nil + } + limit, err := properties.ParseBytes(value) + if err != nil || limit < 0 || limit > 4<<20 { + return 0, fmt.Errorf("bodies %q must be between zero and 4MiB", value) + } + return limit, nil +} + +func DecodeRecord(raw json.RawMessage) (Exchange, error) { + var record Exchange + if err := json.Unmarshal(raw, &record); err != nil { + return record, fmt.Errorf("decode HTTP exchange: %w", err) + } + projected := FromEntry(record.Feature, record.Entry) + var err error + record.StartedAt, err = time.Parse(time.RFC3339Nano, record.StartedDateTime) + if err != nil { + return record, fmt.Errorf("HTTP exchange startedDateTime: %w", err) + } + record.Method, record.URL, record.Host = projected.Method, projected.URL, projected.Host + record.Status, record.DurationMs = projected.Status, projected.DurationMs + if record.Error == "" { + record.Error = record.Entry.Error + } + record.Entry.Error = record.Error + return record, nil +} + +func decodeBlob(blob agentclient.Blob, resource map[string]string) (seam.DecodedBlob[Exchange], error) { + if blob.Info.ContentType != "application/json" { + return seam.DecodedBlob[Exchange]{}, fmt.Errorf("HTTP exchanges require application/json, received %q", blob.Info.ContentType) + } + var raw []json.RawMessage + if err := json.Unmarshal(blob.Data, &raw); err != nil { + return seam.DecodedBlob[Exchange]{}, fmt.Errorf("decode HTTP exchange batch: %w", err) + } + records := make([]Exchange, 0, len(raw)) + for _, item := range raw { + record, err := DecodeRecord(item) + if err != nil { + return seam.DecodedBlob[Exchange]{}, err + } + record.Resource = resource + records = append(records, record) + } + return seam.DecodedBlob[Exchange]{Rows: records}, nil +} diff --git a/pkg/tracing/http/pretty.go b/pkg/tracing/http/pretty.go new file mode 100644 index 0000000..7736f3a --- /dev/null +++ b/pkg/tracing/http/pretty.go @@ -0,0 +1,22 @@ +package http + +import ( + "github.com/flanksource/clicky/api" + "github.com/flanksource/commons/har/harpretty" +) + +func (e Exchange) Pretty() api.Text { + t := harpretty.Entry(e.Entry).Pretty() + if e.Error != "" { + t = t.Space().Append(e.Error, "text-red-500") + } + return t +} + +func (e Exchange) PrettyFull() api.Textable { + t := e.Pretty() + if detail := harpretty.Entry(e.Entry).RowDetail(); detail != nil { + t = t.NewLine().Add(detail) + } + return t +} diff --git a/pkg/tracing/httpproxy/httpproxy_spec_test.go b/pkg/tracing/httpproxy/httpproxy_spec_test.go index 0e436d6..e449358 100644 --- a/pkg/tracing/httpproxy/httpproxy_spec_test.go +++ b/pkg/tracing/httpproxy/httpproxy_spec_test.go @@ -154,6 +154,30 @@ var _ = Describe("http_proxy trace kind", func() { Expect(string(stored)).ToNot(ContainSubstring(secret)) }) + It("records an exchange through the http_proxy preset started with no params", func() { + upstream := httptest.NewServer(nethttp.HandlerFunc(func(w nethttp.ResponseWriter, _ *nethttp.Request) { + _, _ = io.WriteString(w, "ok") + })) + DeferCleanup(upstream.Close) + presets := tracingtest.NewEnv(map[string]tracing.TracePlugin{httpproxy.ProxyKind: httpproxy.UnifiedKind(nil).Preset(httpproxy.ProxyPreset())}) + managed, err := presets.Runtime.Start(dbcontext.New(), tracing.StartRequest{Kind: httpproxy.ProxyKind}) + Expect(err).ToNot(HaveOccurred()) + tracingtest.Running(managed.Session()) + Expect(managed.Status().Metadata).To(ContainElement(MatchFields(IgnoreExtras, Fields{"Name": Equal("mode"), "Value": Equal("connect")}))) + proxied, _ := client(managed, nil) + Expect(read(proxied.Get(upstream.URL + "/preset"))).To(Equal("ok")) + + managed.Session().Stop("spec") + info := tracingtest.Ended(managed.Session()) + Expect(info.Error).To(BeEmpty()) + var recorded []tracehttp.Exchange + Expect(tracing.ReadRecords(context.Background(), presets.Runtime, tracing.TraceRef{Kind: httpproxy.ProxyKind, Stream: info.Events.Stream}, func(exchange tracehttp.Exchange) error { + recorded = append(recorded, exchange) + return nil + })).To(Succeed()) + Expect(recorded).To(ConsistOf(HaveField("URL", upstream.URL+"/preset"))) + }) + It("records only the hosts and paths its globs name, still proxying the rest", func() { upstream := httptest.NewServer(nethttp.HandlerFunc(func(w nethttp.ResponseWriter, _ *nethttp.Request) { _, _ = io.WriteString(w, "ok") diff --git a/pkg/tracing/httpproxy/unified.go b/pkg/tracing/httpproxy/unified.go index 9ef9a05..0862bec 100644 --- a/pkg/tracing/httpproxy/unified.go +++ b/pkg/tracing/httpproxy/unified.go @@ -19,8 +19,9 @@ import ( type UnifiedParams struct { Params - Features []string `json:"features,omitempty" clicky:"title=Features,desc=HTTP features to observe"` - DedupWindow string `json:"dedupWindow,omitempty" clicky:"title=Dedup window,desc=Store identical exchanges only once within this duration"` + Direction tracehttp.Direction `json:"direction,omitempty" clicky:"title=Direction,desc=Incoming requests outgoing clients or both"` + Features []string `json:"features,omitempty" clicky:"title=Features,desc=HTTP features to observe"` + DedupWindow string `json:"dedupWindow,omitempty" clicky:"title=Dedup window,desc=Store identical exchanges only once within this duration"` dedupDuration time.Duration } @@ -30,7 +31,7 @@ var ( ) func (p UnifiedParams) httpParams() tracehttp.Params { - return tracehttp.Params{Options: tracehttp.Options{Features: p.Features, DedupWindow: p.DedupWindow}, Target: p.Target} + return tracehttp.Params{Options: tracehttp.Options{Selection: tracehttp.Selection{Direction: p.Direction, Hosts: p.Hosts, Paths: p.Paths}, Bodies: p.Bodies, Redact: p.Redact, Features: p.Features, DedupWindow: p.DedupWindow}, Target: p.Target} } func (p *UnifiedParams) ValidateInput(raw json.RawMessage) error { @@ -76,9 +77,23 @@ type unified struct { func (u unified) AgentAvailable(probe string, t target.Target) (bool, error) { return u.self.AgentAvailable(probe, t) } -func (unified) Params() UnifiedParams { +func (unified) Params() UnifiedParams { return DefaultParams() } + +// ProxyKind is the `http` kind armed on its proxy seam (ProxyPreset). +const ProxyKind = "http_proxy" + +// DefaultParams are the `http` kind's defaults: telltale's HTTP feature on +// its self and agent seams, and the proxy's mode and body cap on its proxy +// seam. A host adjusts a copy of them for Handler.WithDefaults. +func DefaultParams() UnifiedParams { return UnifiedParams{Params: proxy{}.Params(), Features: []string{httpx.Feature}} } + +// ProxyPreset arms the `http` kind only on its proxy seam, as ProxyKind, +// without the features only its other seams observe. +func ProxyPreset() tracing.PresetOptions[UnifiedParams] { + return tracing.PresetOptions[UnifiedParams]{Title: "HTTP proxy", Seam: target.Proxy, Defaults: func(p *UnifiedParams) { p.Features = nil }} +} func (unified) Schema() recordresults.ResultType[tracehttp.Exchange] { return proxy{}.Schema() } func (u unified) Prepare(ctx dbcontext.Context, p UnifiedParams, out tracing.Emitter[tracehttp.Exchange]) (dbcontext.Context, func() error, error) { if p.Target.Seam == target.Proxy { diff --git a/pkg/tracing/httptop/checkpoint_test.go b/pkg/tracing/httptop/checkpoint_test.go new file mode 100644 index 0000000..9a0a9ab --- /dev/null +++ b/pkg/tracing/httptop/checkpoint_test.go @@ -0,0 +1,61 @@ +package httptop_test + +import ( + "context" + "encoding/json" + + dbcontext "github.com/flanksource/commons-db/context" + _ "github.com/flanksource/commons-db/query/providers/sqlite" + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/httptop" + "github.com/flanksource/telltale/pkg/tracing/tracingtest" + sdk "github.com/flanksource/telltale/sdk" + "github.com/flanksource/telltale/sdk/protocol" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type checkpointProbe struct{ received chan json.RawMessage } + +func (*checkpointProbe) Spec() protocol.ProbeSpec { + return protocol.ProbeSpec{Name: httptop.Kind, Checkpoints: true} +} + +func (*checkpointProbe) Arm(context.Context, json.RawMessage, sdk.Sink) (func() error, error) { + return func() error { return nil }, nil +} + +func (p *checkpointProbe) Checkpoint(_ context.Context, params json.RawMessage, _ sdk.Sink) error { + p.received <- params + return nil +} + +var _ = Describe("HTTP Top checkpoint parameters", func() { + It("forwards stop reasons through the runtime and rejects invalid parameters before contacting the agent", func() { + probe := &checkpointProbe{received: make(chan json.RawMessage, 16)} + sdk.RegisterProbe(probe) + server := sdk.Loopback() + DeferCleanup(func() { Expect(server.Close()).To(Succeed()) }) + agent, err := agentclient.Connect(context.Background(), server.Connect(), "") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { Expect(agent.Close()).To(Succeed()) }) + registry := agentclient.NewRegistry() + registry.Dir = GinkgoT().TempDir() + Expect(registry.Add(agent)).To(Succeed()) + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{httptop.Kind: httptop.NewKind(registry)}) + params, err := json.Marshal(map[string]any{"target": map[string]any{"seam": "agent", "agent": agent.Hello.AgentID}}) + Expect(err).NotTo(HaveOccurred()) + capture, err := env.Runtime.Start(dbcontext.New(), tracing.StartRequest{Kind: httptop.Kind, Params: params}) + Expect(err).NotTo(HaveOccurred()) + tracingtest.Running(capture.Session()) + checkpoint := tracing.WithCheckpointParams(context.Background(), json.RawMessage(`{"reason":"stop"}`)) + Expect(env.Runtime.Checkpoint(checkpoint, capture)).To(Succeed()) + Eventually(probe.received).Should(Receive(MatchJSON(`{"reason":"stop"}`))) + for _, raw := range []string{`{"reason":"stop","unknown":true}`, `{"reason":7}`} { + err := env.Runtime.Checkpoint(tracing.WithCheckpointParams(context.Background(), json.RawMessage(raw)), capture) + Expect(err).To(MatchError(ContainSubstring("invalid checkpoint params"))) + Expect(probe.received).To(BeEmpty()) + } + }) +}) diff --git a/pkg/tracing/httptop/decode.go b/pkg/tracing/httptop/decode.go new file mode 100644 index 0000000..3bea97d --- /dev/null +++ b/pkg/tracing/httptop/decode.go @@ -0,0 +1,74 @@ +package httptop + +import ( + "encoding/json" + "fmt" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/otlp/profiles" + "github.com/flanksource/telltale/pkg/tracing/seam" +) + +func Decode(blob agentclient.Blob, resource map[string]string) (seam.DecodedBlob[Sample], error) { + if blob.Info.ContentType != profiles.ContentType { + return seam.DecodedBlob[Sample]{}, fmt.Errorf("HTTP top requires OTLP Profiles, received %q", blob.Info.ContentType) + } + if blob.Info.Session == "" { + return seam.DecodedBlob[Sample]{}, fmt.Errorf("HTTP top agent blob is missing the authenticated capture session") + } + source := resource["telltale.source.instance"] + if source == "" { + return seam.DecodedBlob[Sample]{}, fmt.Errorf("HTTP top agent Hello is missing telltale.source.instance") + } + snapshots, err := profiles.Decode(blob.Data, profiles.Options{ExpectedKind: Kind, ExpectedCaptureID: blob.Info.Session, ExpectedSource: source}) + if err != nil { + return seam.DecodedBlob[Sample]{}, err + } + if len(snapshots) != 1 { + return seam.DecodedBlob[Sample]{}, fmt.Errorf("HTTP top requires one snapshot per blob, received %d", len(snapshots)) + } + snapshot := snapshots[0] + rows, err := FromSnapshot(snapshot) + if err != nil { + return seam.DecodedBlob[Sample]{}, err + } + for i := range rows { + for name, value := range resource { + rows[i].Resource[name] = value + } + } + frame := snapshot.Frame() + return seam.DecodedBlob[Sample]{Rows: rows, Frame: &frame}, nil +} + +func FromSnapshot(snapshot profiles.Snapshot) ([]Sample, error) { + if snapshot.Kind != Kind { + return nil, fmt.Errorf("HTTP top cannot decode snapshot kind %q", snapshot.Kind) + } + frame := snapshot.Frame() + rows := make([]Sample, 0, len(snapshot.Rows)) + for _, native := range snapshot.Rows { + attributes, err := json.Marshal(native.Attributes) + if err != nil { + return nil, err + } + var sample Sample + if err := json.Unmarshal(attributes, &sample); err != nil { + return nil, fmt.Errorf("HTTP top attributes: %w", err) + } + if sample.StartedAt.IsZero() { + return nil, fmt.Errorf("HTTP top operation %q has no startedAt", native.OperationID) + } + sample.RequestID = native.OperationID + sample.Source = frame.Source + sample.ElapsedMs = float64(native.ElapsedNS) / float64(1e6) + sample.SampledAt = native.SampledAt + sample.TraceID, sample.SpanID = native.TraceID, native.SpanID + sample.Resource = map[string]string{} + for name, value := range snapshot.Resource { + sample.Resource[name] = fmt.Sprint(value) + } + rows = append(rows, sample) + } + return rows, nil +} diff --git a/pkg/tracing/httptop/httptop_test.go b/pkg/tracing/httptop/httptop_test.go new file mode 100644 index 0000000..e568eef --- /dev/null +++ b/pkg/tracing/httptop/httptop_test.go @@ -0,0 +1,56 @@ +package httptop_test + +import ( + "testing" + "time" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/otlp/profiles" + "github.com/flanksource/telltale/pkg/tracing/httptop" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestHTTPTop(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "HTTP top") } + +var _ = Describe("HTTP profiles", func() { + It("decodes an authoritative empty snapshot to clear completed requests", func() { + start := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + data, err := profiles.Encode([]profiles.Snapshot{{Kind: httptop.Kind, CaptureID: "capture", Source: "jvm", ProfileID: "0123456789abcdef0123456789abcdef", Sequence: 2, StartedAt: start, CompletedAt: start.Add(time.Second), Limit: 50, Complete: true}}) + Expect(err).NotTo(HaveOccurred()) + blob := agentclient.Blob{Data: data} + blob.Info.ContentType, blob.Info.Session = profiles.ContentType, "capture" + decoded, err := httptop.Decode(blob, map[string]string{"telltale.source.instance": "jvm"}) + Expect(err).NotTo(HaveOccurred()) + Expect(decoded.Rows).To(BeEmpty()) + Expect(decoded.Frame).NotTo(BeNil()) + Expect(decoded.Frame.Source).To(Equal("jvm/capture")) + Expect(decoded.Frame.Tick).To(Equal(int64(2))) + }) + DescribeTable("rejects invalid sample controls", func(options httptop.Options) { Expect(options.Validate()).NotTo(Succeed()) }, Entry("zero poll", httptop.Options{Poll: "0s"}), Entry("negative limit", httptop.Options{Top: -1}), Entry("negative age", httptop.Options{MinElapsed: "-1s"})) + It("projects authoritative source and operation identity without rounding elapsed time", func() { + started := time.Date(2026, 1, 2, 3, 4, 5, 123, time.UTC) + snapshot := profiles.Snapshot{Kind: httptop.Kind, Source: "jvm", CaptureID: "capture", Resource: map[string]any{"service.name": "example"}, Rows: []profiles.Row{{OperationID: "request-7", ElapsedNS: 1234567, SampledAt: started.Add(time.Second), Attributes: map[string]any{"startedAt": started.Format(time.RFC3339Nano), "source": "untrusted", "requestId": "untrusted", "direction": "incoming", "method": "GET", "route": "/orders/{id}", "handler": "example.OrderController#get", "sampled": false}, TraceID: "0123456789abcdef0123456789abcdef", SpanID: "0123456789abcdef"}}} + rows, err := httptop.FromSnapshot(snapshot) + Expect(err).NotTo(HaveOccurred()) + Expect(rows).To(HaveLen(1)) + Expect(rows[0].Source).To(Equal("jvm/capture")) + Expect(rows[0].RequestID).To(Equal("request-7")) + Expect(rows[0].Handler).To(Equal("example.OrderController#get")) + Expect(rows[0].StartedAt).To(Equal(started)) + Expect(rows[0].ElapsedMs).To(Equal(1.234567)) + Expect(rows[0].Sampled).NotTo(BeNil()) + Expect(*rows[0].Sampled).To(BeFalse()) + Expect(rows[0].Resource).To(Equal(map[string]string{"service.name": "example"})) + }) + + DescribeTable("binds agent profiles to the armed capture and JVM source", func(capture, source string) { + start := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + data, err := profiles.Encode([]profiles.Snapshot{{Kind: httptop.Kind, CaptureID: "capture", Source: "jvm", ProfileID: "0123456789abcdef0123456789abcdef", Sequence: 1, StartedAt: start, CompletedAt: start.Add(time.Second), Limit: 50, Complete: true}}) + Expect(err).NotTo(HaveOccurred()) + blob := agentclient.Blob{Data: data} + blob.Info.ContentType, blob.Info.Session = profiles.ContentType, capture + _, err = httptop.Decode(blob, map[string]string{"telltale.source.instance": source}) + Expect(err).To(HaveOccurred()) + }, Entry("foreign capture", "foreign", "jvm"), Entry("foreign source", "capture", "foreign"), Entry("missing capture", "", "jvm"), Entry("missing source", "capture", "")) +}) diff --git a/pkg/tracing/httptop/kind.go b/pkg/tracing/httptop/kind.go new file mode 100644 index 0000000..7914ee7 --- /dev/null +++ b/pkg/tracing/httptop/kind.go @@ -0,0 +1,136 @@ +package httptop + +import ( + "encoding/json" + "fmt" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/tracing" + tracehttp "github.com/flanksource/telltale/pkg/tracing/http" + "github.com/flanksource/telltale/pkg/tracing/seam" + "github.com/flanksource/telltale/pkg/tracing/target" +) + +const Kind = "http_top" + +type Options struct { + tracehttp.Selection + Poll string `json:"poll,omitempty" clicky:"title=Poll,desc=Sampling interval such as 5s"` + Top int `json:"top,omitempty" clicky:"title=Top,desc=Maximum active requests per sample; default 50"` + MinElapsed string `json:"minElapsed,omitempty" clicky:"title=Minimum elapsed,desc=Only show requests running at least this long"` +} + +func (o Options) Validate() error { + if err := o.Selection.Validate(); err != nil { + return err + } + if o.Poll != "" { + if value, err := time.ParseDuration(o.Poll); err != nil || value <= 0 { + return fmt.Errorf("poll must be a positive duration") + } + } + if o.MinElapsed != "" { + if value, err := time.ParseDuration(o.MinElapsed); err != nil || value < 0 { + return fmt.Errorf("minElapsed must be a nonnegative duration") + } + } + if o.Top < 0 || o.Top > 10000 { + return fmt.Errorf("top must be between zero and 10000") + } + return nil +} + +type Params struct { + Options + Target target.Target `json:"target,omitzero"` +} + +type Sample struct { + tracehttp.Metadata + Source string `json:"source" filter:"terms"` + SampledAt time.Time `json:"sampledAt"` + StartedAt time.Time `json:"startedAt"` + Method string `json:"method" filter:"terms"` + Host string `json:"host" filter:"terms"` + URL string `json:"url" filter:"text"` + ElapsedMs float64 `json:"elapsedMs" pretty:"label=Elapsed,type=duration,unit=ms"` + Resource map[string]string `json:"resource,omitempty"` +} + +type handler struct { + agent seam.AgentSeam[Params, Sample] +} + +func (handler) Params() Params { + return Params{Options: Options{Poll: "5s", Top: 50, Selection: tracehttp.Selection{Direction: tracehttp.Both}}} +} +func (handler) Schema() recordresults.ResultType[Sample] { + return ResultType() +} + +func ResultType() recordresults.ResultType[Sample] { + return recordresults.ResultType[Sample]{Kind: Kind, Title: "Active HTTP requests", TimeColumn: "sampledAt", Follow: true, SearchColumns: []string{"url", "route", "thread"}} +} +func (h handler) AgentAvailable(probe string, t target.Target) (bool, error) { + return h.agent.Available(probe, t) +} +func (h handler) Prepare(ctx dbcontext.Context, p Params, out tracing.Emitter[Sample]) (dbcontext.Context, func() error, error) { + return h.agent.Prepare(ctx, p, out) +} +func (h handler) Handle(ctx dbcontext.Context, p Params, out tracing.Emitter[Sample], _ tracing.Records[Sample]) error { + return h.agent.Handle(ctx, p, out) +} +func (h handler) Checkpoint(ctx dbcontext.Context, _ Params, out tracing.Emitter[Sample]) error { + var params struct { + Reason string `json:"reason,omitempty"` + } + if _, err := tracing.DecodeCheckpointParams(ctx, ¶ms); err != nil { + return err + } + raw, err := json.Marshal(params) + if err != nil { + return err + } + return h.agent.Checkpoint(ctx, raw, out) +} + +func NewKind(registry *agentclient.Registry) *tracing.Handler[Params, Sample] { + agent := seam.AgentSeam[Params, Sample]{Probe: Kind, Registry: registry, Target: func(p Params) target.Target { return p.Target }, ArmParams: armParams, DecodeBlob: Decode} + return tracing.NewHandler[Params, Sample](handler{agent: agent}, tracing.Capabilities{Live: true, Follow: true}, target.Agent).WithTop(tracing.TopSpec{Identity: []string{"source", "requestId"}, Order: query.Order{{Column: "elapsedMs", Desc: true}, {Column: "requestId"}}, PerSource: true}) +} + +func armParams(p Params) (json.RawMessage, error) { + if err := p.Validate(); err != nil { + return nil, err + } + interval := 5 * time.Second + if p.Poll != "" { + var err error + interval, err = time.ParseDuration(p.Poll) + if err != nil { + return nil, err + } + } + limit := p.Top + if limit == 0 { + limit = 50 + } + var minimum time.Duration + if p.MinElapsed != "" { + var err error + minimum, err = time.ParseDuration(p.MinElapsed) + if err != nil { + return nil, err + } + } + return json.Marshal(struct { + Options + PollIntervalMs int64 `json:"pollIntervalMs"` + Limit int `json:"limit"` + MinElapsedNs int64 `json:"minElapsedNs,omitempty"` + }{Options: p.Options, PollIntervalMs: interval.Milliseconds(), Limit: limit, MinElapsedNs: int64(minimum)}) +} diff --git a/pkg/tracing/jfr/jfr.go b/pkg/tracing/jfr/jfr.go index abb1f77..1c35d58 100644 --- a/pkg/tracing/jfr/jfr.go +++ b/pkg/tracing/jfr/jfr.go @@ -14,6 +14,7 @@ import ( dbcontext "github.com/flanksource/commons-db/context" "github.com/flanksource/commons-db/query" "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/commons-db/recordstore/sampleframe" "github.com/flanksource/telltale/pkg/tracing" "github.com/flanksource/telltale/pkg/tracing/jfr/decode" ) @@ -54,12 +55,14 @@ type Recording interface { // the watermark after which its next dump's events are new, and the samples // decoded but not yet emitted. type running struct { - mu sync.Mutex - recording Recording - options RecordOptions - watermark time.Time - pending []Sample - report Report + mu sync.Mutex + recording Recording + options RecordOptions + watermark time.Time + intervalFrom time.Time + pending []Sample + pendingFrame *sampleframe.Frame + report Report } type runningKey struct{} @@ -114,7 +117,7 @@ func (p profiler[P]) Prepare(ctx dbcontext.Context, params P, records tracing.Em return nil } return ctx.WithValue(runningKey{}, &running{ - recording: recording, options: options, + recording: recording, options: options, intervalFrom: time.Now().UTC(), report: Report{Engine: options.Engine, Recording: handle, Events: map[string]int64{}}, }), release, nil } @@ -164,6 +167,12 @@ func (r *running) dumpEvery(ctx context.Context, records tracing.Emitter[Sample] func (r *running) dump(ctx context.Context, records tracing.Emitter[Sample]) error { r.mu.Lock() defer r.mu.Unlock() + if r.pendingFrame != nil { + if err := r.flush(ctx, records); err != nil { + return err + } + } + startedAt := r.intervalFrom data, err := r.recording.Dump(ctx) if err != nil { return errors.Join(fmt.Errorf("dump JFR recording %q: %w", r.report.Recording, err), r.flush(ctx, records)) @@ -182,6 +191,7 @@ func (r *running) dump(ctx context.Context, records tracing.Emitter[Sample]) err for _, sample := range profile.Samples { r.pending = append(r.pending, Sample{StackSample: sample}) } + r.pendingFrame = &sampleframe.Frame{Source: r.report.Recording, StartedAt: startedAt, CompletedAt: time.Now().UTC(), Observed: len(r.pending)} for event, samples := range profile.Counts { r.report.Events[event] += samples } @@ -193,13 +203,15 @@ func (r *running) dump(ctx context.Context, records tracing.Emitter[Sample]) err // flush emits the decoded samples not yet emitted. func (r *running) flush(ctx context.Context, records tracing.Emitter[Sample]) error { - for len(r.pending) > 0 { - if err := records.Emit(ctx, r.pending[0]); err != nil { - return err - } - r.pending = r.pending[1:] + if r.pendingFrame == nil { + return nil + } + if err := tracing.PublishFrame(ctx, records, *r.pendingFrame, r.pending); err != nil { + return err } + r.intervalFrom = r.pendingFrame.CompletedAt r.pending = nil + r.pendingFrame = nil return nil } @@ -207,5 +219,5 @@ func (r *running) flush(ctx context.Context, records tracing.Emitter[Sample]) er // stacks cut at 64KiB. func NewKindFor[P Recordable](recorder Recorder) *tracing.Handler[P, Sample] { return tracing.NewHandler[P, Sample](profiler[P]{recorder: recorder}, tracing.Capabilities{Live: true}, "process"). - WithTruncation(maxStackBytes) + WithTruncation(maxStackBytes).WithTop(topSpec()) } diff --git a/pkg/tracing/jfr/kind.go b/pkg/tracing/jfr/kind.go index c7f4131..a919ba4 100644 --- a/pkg/tracing/jfr/kind.go +++ b/pkg/tracing/jfr/kind.go @@ -9,6 +9,7 @@ import ( dbcontext "github.com/flanksource/commons-db/context" "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/commons-db/recordstore/sampleframe" "github.com/flanksource/telltale/pkg/agentclient" "github.com/flanksource/telltale/pkg/tracing" "github.com/flanksource/telltale/pkg/tracing/jfr/decode" @@ -31,24 +32,34 @@ func (b builtin) AgentAvailable(probe string, t target.Target) (bool, error) { func jfrAgent(p Params) seam.AgentSeam[Params, Sample] { var mu sync.Mutex watermarks := map[string]time.Time{} - return seam.AgentSeam[Params, Sample]{Probe: KindName, Target: func(p Params) target.Target { return p.Target }, ArmParams: func(p Params) (json.RawMessage, error) { return json.Marshal(p) }, DecodeBlob: func(blob agentclient.Blob, resource map[string]string) ([]Sample, error) { + pending := map[string]time.Time{} + return seam.AgentSeam[Params, Sample]{Probe: KindName, Target: func(p Params) target.Target { return p.Target }, ArmParams: func(p Params) (json.RawMessage, error) { return json.Marshal(p) }, DecodeBlob: func(blob agentclient.Blob, resource map[string]string) (seam.DecodedBlob[Sample], error) { + started := time.Now().UTC() if blob.Info.ContentType != protocol.ContentTypeJFR { - return nil, fmt.Errorf("JFR content type %q", blob.Info.ContentType) + return seam.DecodedBlob[Sample]{}, fmt.Errorf("JFR content type %q", blob.Info.ContentType) } mu.Lock() defer mu.Unlock() profile, err := decode.Decode(blob.Data, decode.DecodeOptions{After: watermarks[blob.Info.Session], Threads: p.Threads, MaxStackDepth: p.MaxStackDepth}) if err != nil { - return nil, err + return seam.DecodedBlob[Sample]{}, err } if profile.Latest.After(watermarks[blob.Info.Session]) { - watermarks[blob.Info.Session] = profile.Latest + pending[blob.Info.Session] = profile.Latest } rows := make([]Sample, len(profile.Samples)) for i, sample := range profile.Samples { rows[i] = Sample{StackSample: sample, Resource: maps.Clone(resource)} } - return rows, nil + return seam.DecodedBlob[Sample]{Rows: rows, Frame: &sampleframe.Frame{Source: resource["telltale.agent.id"] + "/" + resource["process.pid"] + "/" + blob.Info.Name, StartedAt: started, CompletedAt: time.Now().UTC(), Observed: len(rows)}}, nil + }, CommitBlob: func(blob agentclient.Blob) error { + mu.Lock() + defer mu.Unlock() + if latest, exists := pending[blob.Info.Session]; exists { + watermarks[blob.Info.Session] = latest + delete(pending, blob.Info.Session) + } + return nil }} } @@ -82,5 +93,5 @@ func (b builtin) Checkpoint(ctx dbcontext.Context, p Params, out tracing.Emitter return ctx.Value(agentKey{}).(seam.AgentSeam[Params, Sample]).Checkpoint(ctx, json.RawMessage(`{}`), out) } func Kind() *tracing.Handler[Params, Sample] { - return tracing.NewHandler[Params, Sample](builtin{native: profiler[Params]{recorder: NativeRecorder{}}}, tracing.Capabilities{Live: true}, target.Process, target.Agent).WithTruncation(maxStackBytes) + return tracing.NewHandler[Params, Sample](builtin{native: profiler[Params]{recorder: NativeRecorder{}}}, tracing.Capabilities{Live: true}, target.Process, target.Agent).WithTruncation(maxStackBytes).WithTop(topSpec()) } diff --git a/pkg/tracing/jfr/pretty.go b/pkg/tracing/jfr/pretty.go new file mode 100644 index 0000000..36cf850 --- /dev/null +++ b/pkg/tracing/jfr/pretty.go @@ -0,0 +1,18 @@ +package jfr + +import ( + "strings" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" + "github.com/flanksource/telltale/pkg/tracing" +) + +func (r Sample) Pretty() api.Text { + return clicky.Text(r.Event, "text-blue-500 font-bold").Space().Append(r.Thread, "text-muted"). + Space().Add(tracing.PrettySampleValue(r.Value, r.Unit)).Space().Append(r.Frame, "font-mono") +} + +func (r Sample) PrettyFull() api.Textable { + return r.Pretty().NewLine().Add(clicky.CodeBlock("text", strings.ReplaceAll(r.Stack, ";", "\n"))) +} diff --git a/pkg/tracing/jfr/summary.go b/pkg/tracing/jfr/summary.go new file mode 100644 index 0000000..a623e44 --- /dev/null +++ b/pkg/tracing/jfr/summary.go @@ -0,0 +1,30 @@ +package jfr + +import ( + "github.com/flanksource/telltale/pkg/tracing" + "iter" +) + +func (b builtin) Summarize(rows iter.Seq[Sample]) (map[string]any, error) { + return b.native.Summarize(rows) +} + +func (profiler[P]) Summarize(rows iter.Seq[Sample]) (map[string]any, error) { + events := map[string]int64{} + totals := map[string]int64{} + functions := map[string]map[string]int64{} + for row := range rows { + events[row.Event] += row.Samples + totals[row.Event+"/"+row.Unit] += row.Value + key := row.Event + "/" + row.Unit + if functions[key] == nil { + functions[key] = map[string]int64{} + } + functions[key][row.Frame] += row.Value + } + ranked := map[string]any{} + for key, values := range functions { + ranked[key] = tracing.RankedValues(values) + } + return map[string]any{"events": events, "totals": totals, "functions": ranked}, nil +} diff --git a/pkg/tracing/jfr/table.go b/pkg/tracing/jfr/table.go new file mode 100644 index 0000000..c06229e --- /dev/null +++ b/pkg/tracing/jfr/table.go @@ -0,0 +1,27 @@ +package jfr + +import ( + "github.com/flanksource/clicky/api" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/telltale/pkg/tracing" +) + +func (Sample) Columns() []api.ColumnDef { + return query.TableColumnsFor[Sample]("event", "value", "thread", "frame") +} +func (s Sample) Row() map[string]any { + row := query.TableRowFor(s) + row["value"] = api.TableCell{Value: tracing.PrettySampleValue(s.Value, s.Unit), FilterValue: s.Value} + return row +} +func (s Sample) RowDetail() api.Textable { return s.PrettyFull() } + +func topSpec() tracing.TopSpec { + return tracing.TopSpec{Stack: &tracing.TopStack{Column: "stack", MetricColumn: "event", Separator: ";"}, Identity: []string{"event", "thread", "stack"}, Order: query.Order{{Column: "value", Desc: true}}, Metrics: []tracing.TopMetric{ + {Key: "cpu", Label: "CPU samples", Unit: "samples", View: "flamegraph"}, + {Key: "alloc", Label: "Allocated bytes", Unit: "bytes", View: "treemap"}, + {Key: "wall", Label: "Wall samples", Unit: "samples", View: "flamegraph"}, + {Key: "lock", Label: "Lock time", Unit: "ns", View: "flamegraph"}, + {Key: "park", Label: "Park time", Unit: "ns", View: "flamegraph"}, + }} +} diff --git a/pkg/tracing/jms/checkpoint_test.go b/pkg/tracing/jms/checkpoint_test.go new file mode 100644 index 0000000..7d25fd6 --- /dev/null +++ b/pkg/tracing/jms/checkpoint_test.go @@ -0,0 +1,62 @@ +package jms_test + +import ( + "context" + "encoding/json" + + dbcontext "github.com/flanksource/commons-db/context" + _ "github.com/flanksource/commons-db/query/providers/sqlite" + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/jms" + "github.com/flanksource/telltale/pkg/tracing/tracingtest" + sdk "github.com/flanksource/telltale/sdk" + "github.com/flanksource/telltale/sdk/protocol" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type checkpointProbe struct{ received chan json.RawMessage } + +func (*checkpointProbe) Spec() protocol.ProbeSpec { + return protocol.ProbeSpec{Name: jms.Kind, Checkpoints: true} +} + +func (*checkpointProbe) Arm(context.Context, json.RawMessage, sdk.Sink) (func() error, error) { + return func() error { return nil }, nil +} + +func (p *checkpointProbe) Checkpoint(_ context.Context, params json.RawMessage, _ sdk.Sink) error { + p.received <- params + return nil +} + +var _ = Describe("JMS checkpoint parameters", func() { + It("forwards stop reasons through the runtime and rejects invalid parameters before contacting the agent", func() { + GinkgoT().Setenv("TELLTALE_ALLOW_MUTATION", "1") + probe := &checkpointProbe{received: make(chan json.RawMessage, 16)} + sdk.RegisterProbe(probe) + server := sdk.Loopback() + DeferCleanup(func() { Expect(server.Close()).To(Succeed()) }) + agent, err := agentclient.Connect(context.Background(), server.Connect(), "") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { Expect(agent.Close()).To(Succeed()) }) + registry := agentclient.NewRegistry() + registry.Dir = GinkgoT().TempDir() + Expect(registry.Add(agent)).To(Succeed()) + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{jms.Kind: jms.NewKind(registry)}) + params, err := json.Marshal(map[string]any{"target": map[string]any{"seam": "agent", "agent": agent.Hello.AgentID, "allowMutation": true}}) + Expect(err).NotTo(HaveOccurred()) + capture, err := env.Runtime.Start(dbcontext.New(), tracing.StartRequest{Kind: jms.Kind, Params: params}) + Expect(err).NotTo(HaveOccurred()) + tracingtest.Running(capture.Session()) + checkpoint := tracing.WithCheckpointParams(context.Background(), json.RawMessage(`{"reason":"stop"}`)) + Expect(env.Runtime.Checkpoint(checkpoint, capture)).To(Succeed()) + Eventually(probe.received).Should(Receive(MatchJSON(`{"reason":"stop"}`))) + for _, raw := range []string{`{"reason":"stop","unknown":true}`, `{"reason":7}`} { + err := env.Runtime.Checkpoint(tracing.WithCheckpointParams(context.Background(), json.RawMessage(raw)), capture) + Expect(err).To(MatchError(ContainSubstring("invalid checkpoint params"))) + Expect(probe.received).To(BeEmpty()) + } + }) +}) diff --git a/pkg/tracing/jms/jms.go b/pkg/tracing/jms/jms.go new file mode 100644 index 0000000..92e1d5c --- /dev/null +++ b/pkg/tracing/jms/jms.go @@ -0,0 +1,135 @@ +// Package jms records JMS operations observed inside a connected JVM. +package jms + +import ( + "encoding/json" + "fmt" + "maps" + "strings" + "time" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/target" +) + +const Kind = "jms" + +type Options struct { + Direction string `json:"direction,omitempty" clicky:"title=Direction,enum=both,incoming,outgoing"` + Destinations []string `json:"destinations,omitempty" clicky:"title=Destinations,desc=Exact queue or topic names to observe; empty observes all"` + Body bool `json:"body,omitempty" clicky:"title=Message bodies,desc=Capture bounded text bodies without consuming stream cursors or deserializing objects"` + MaxBodyBytes int `json:"maxBodyBytes,omitempty" clicky:"title=Body byte limit"` + Capacity int `json:"capacity,omitempty" clicky:"title=Buffer capacity"` +} + +func (o Options) Validate() error { + if o.Direction != "" && o.Direction != "both" && o.Direction != "incoming" && o.Direction != "outgoing" { + return fmt.Errorf("direction must be both, incoming or outgoing") + } + for _, destination := range o.Destinations { + if strings.TrimSpace(destination) == "" { + return fmt.Errorf("destinations must not contain blank names") + } + } + if o.MaxBodyBytes < 0 || o.MaxBodyBytes > 1<<20 || (!o.Body && o.MaxBodyBytes != 0) { + return fmt.Errorf("maxBodyBytes requires body capture and must be between zero and 1048576") + } + if o.Capacity < 0 || o.Capacity > 50000 { + return fmt.Errorf("capacity must be between zero and 50000") + } + return nil +} + +type Params struct { + Target target.Target `json:"target,omitzero"` + Options +} + +func (p Params) Validate() error { return p.Options.Validate() } + +type Row struct { + tracing.Truncation + StartedAt time.Time `json:"startedAt"` + DurationMs float64 `json:"durationMs"` + Operation string `json:"operation" filter:"terms"` + Direction string `json:"direction" filter:"terms"` + Outcome string `json:"outcome" filter:"terms"` + Destination string `json:"destination,omitempty" filter:"terms"` + DestinationType string `json:"destinationType,omitempty"` + MessageID string `json:"messageId,omitempty"` + DeliveryID string `json:"deliveryId,omitempty"` + CorrelationID string `json:"correlationId,omitempty"` + Redelivered *bool `json:"redelivered,omitempty"` + ReplyTo string `json:"replyTo,omitempty"` + Subscription string `json:"subscription,omitempty"` + MessageType string `json:"messageType,omitempty"` + Framework string `json:"framework,omitempty"` + Handler string `json:"handler,omitempty"` + Transacted *bool `json:"transacted,omitempty"` + AcknowledgeMode *int `json:"acknowledgeMode,omitempty"` + Body string `json:"body,omitempty"` + BodyTruncated bool `json:"bodyTruncated,omitempty"` + BodyUnsupported string `json:"bodyUnsupported,omitempty"` + Error string `json:"error,omitempty" filter:"text"` + ErrorType string `json:"errorType,omitempty"` + Phase string `json:"phase,omitempty" filter:"terms"` + MetadataErrors []string `json:"metadataErrors,omitempty"` + TraceID string `json:"traceId,omitempty"` + SpanID string `json:"spanId,omitempty"` + Resource map[string]string `json:"resource,omitempty"` +} + +func Decode(blob agentclient.Blob, resource map[string]string) ([]Row, error) { + if blob.Info.ContentType != "application/json" { + return nil, fmt.Errorf("JMS events require application/json, received %q", blob.Info.ContentType) + } + var raw []json.RawMessage + if err := json.Unmarshal(blob.Data, &raw); err != nil { + return nil, fmt.Errorf("decode JMS events: %w", err) + } + if raw == nil { + return nil, fmt.Errorf("JMS events must be an array, received null") + } + rows := make([]Row, 0, len(raw)) + for i, item := range raw { + row, err := DecodeRecord(item) + if err != nil { + return nil, fmt.Errorf("JMS event %d: %w", i, err) + } + merged := maps.Clone(resource) + if merged == nil { + merged = map[string]string{} + } + maps.Copy(merged, row.Resource) + row.Resource = merged + rows = append(rows, row) + } + return rows, nil +} + +func DecodeRecord(raw json.RawMessage) (Row, error) { + var row Row + if err := json.Unmarshal(raw, &row); err != nil { + return row, fmt.Errorf("decode JMS event: %w", err) + } + if row.StartedAt.IsZero() || row.DurationMs < 0 { + return row, fmt.Errorf("JMS event requires startedAt and nonnegative durationMs") + } + if row.Operation != "send" && row.Operation != "receive" && row.Operation != "process" { + return row, fmt.Errorf("unknown JMS operation %q", row.Operation) + } + if (row.Operation == "send" && row.Direction != "outgoing") || (row.Operation != "send" && row.Direction != "incoming") { + return row, fmt.Errorf("JMS %s has invalid direction %q", row.Operation, row.Direction) + } + if row.Outcome != "success" && row.Outcome != "error" && row.Outcome != "timeout" && row.Outcome != "incomplete" { + return row, fmt.Errorf("unknown JMS outcome %q", row.Outcome) + } + if row.Outcome == "error" && row.Error == "" { + return row, fmt.Errorf("failed JMS event requires error detail") + } + if row.Outcome == "timeout" && row.Operation != "receive" { + return row, fmt.Errorf("only JMS receive operations can time out") + } + return row, nil +} diff --git a/pkg/tracing/jms/jms_test.go b/pkg/tracing/jms/jms_test.go new file mode 100644 index 0000000..ef21f27 --- /dev/null +++ b/pkg/tracing/jms/jms_test.go @@ -0,0 +1,77 @@ +package jms_test + +import ( + "encoding/json" + "testing" + "time" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/tracing/jms" + "github.com/flanksource/telltale/sdk/protocol" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestJMS(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "JMS capture") } + +var _ = Describe("JMS capture", func() { + DescribeTable("rejects invalid capture options", func(options jms.Options) { + Expect(options.Validate()).To(HaveOccurred()) + }, + Entry("unknown direction", jms.Options{Direction: "sideways"}), + Entry("blank destination", jms.Options{Destinations: []string{"orders", " "}}), + Entry("negative body limit", jms.Options{Body: true, MaxBodyBytes: -1}), + Entry("oversized body limit", jms.Options{Body: true, MaxBodyBytes: 1<<20 + 1}), + Entry("body limit without body capture", jms.Options{MaxBodyBytes: 100}), + Entry("negative capacity", jms.Options{Capacity: -1}), + Entry("oversized capacity", jms.Options{Capacity: 50001}), + ) + + It("accepts defaults and explicit incoming or outgoing selection", func() { + for _, direction := range []string{"", "both", "incoming", "outgoing"} { + Expect((jms.Options{Direction: direction, Destinations: []string{"orders"}}).Validate()).To(Succeed()) + } + }) + + It("retains message metadata and error phase from a checkpoint", func() { + body := []byte(`[{"startedAt":"2026-11-14T10:00:00Z","durationMs":12.5,"operation":"process","direction":"incoming","outcome":"error","destination":"orders","redelivered":false,"transacted":true,"acknowledgeMode":0,"phase":"conversion","error":"invalid order","metadataErrors":["subscription unavailable"],"resource":{"tenant":"tenant-x"}}]`) + rows, err := jms.Decode(agentclient.Blob{Info: protocol.Blob{ContentType: "application/json"}, Data: body}, map[string]string{"service.name": "worker"}) + Expect(err).NotTo(HaveOccurred()) + Expect(rows).To(HaveLen(1)) + Expect(rows[0].StartedAt).To(Equal(time.Date(2026, 11, 14, 10, 0, 0, 0, time.UTC))) + Expect(rows[0].Resource).To(Equal(map[string]string{"service.name": "worker", "tenant": "tenant-x"})) + Expect(rows[0].Redelivered).To(HaveValue(BeFalse())) + Expect(rows[0].Transacted).To(HaveValue(BeTrue())) + Expect(rows[0].AcknowledgeMode).To(HaveValue(Equal(0))) + Expect(rows[0].Phase).To(Equal("conversion")) + Expect(rows[0].MetadataErrors).To(Equal([]string{"subscription unavailable"})) + }) + + DescribeTable("rejects malformed or inconsistent JVM records", func(raw string) { + _, err := jms.DecodeRecord(json.RawMessage(raw)) + Expect(err).To(HaveOccurred()) + }, + Entry("null", `null`), + Entry("missing timestamp", `{"operation":"send","direction":"outgoing","outcome":"success"}`), + Entry("negative duration", `{"startedAt":"2026-11-14T10:00:00Z","durationMs":-1,"operation":"send","direction":"outgoing","outcome":"success"}`), + Entry("send declared incoming", `{"startedAt":"2026-11-14T10:00:00Z","operation":"send","direction":"incoming","outcome":"success"}`), + Entry("unknown operation", `{"startedAt":"2026-11-14T10:00:00Z","operation":"acknowledge","direction":"incoming","outcome":"success"}`), + Entry("error without error detail", `{"startedAt":"2026-11-14T10:00:00Z","operation":"send","direction":"outgoing","outcome":"error"}`), + Entry("send misclassified as timeout", `{"startedAt":"2026-11-14T10:00:00Z","operation":"send","direction":"outgoing","outcome":"timeout"}`), + ) + + It("rejects null checkpoints while accepting an empty event array", func() { + blob := agentclient.Blob{Info: protocol.Blob{ContentType: "application/json"}, Data: []byte(`null`)} + _, err := jms.Decode(blob, nil) + Expect(err).To(MatchError(ContainSubstring("must be an array"))) + blob.Data = []byte(`[]`) + rows, err := jms.Decode(blob, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(rows).To(BeEmpty()) + }) + + It("refuses an unrelated content type", func() { + _, err := jms.Decode(agentclient.Blob{Info: protocol.Blob{ContentType: "application/x-jfr"}, Data: []byte(`[]`)}, nil) + Expect(err).To(MatchError(ContainSubstring("application/json"))) + }) +}) diff --git a/pkg/tracing/jms/pretty.go b/pkg/tracing/jms/pretty.go new file mode 100644 index 0000000..38fb4b6 --- /dev/null +++ b/pkg/tracing/jms/pretty.go @@ -0,0 +1,18 @@ +package jms + +import ( + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" + "time" +) + +func (r Row) Pretty() api.Text { + text := api.Text{}.Append("JMS "+r.Operation, "font-bold").Space().Append(r.Destination).Space().Append(r.Handler, "text-muted").Space().Add(clicky.Human(time.Duration(r.DurationMs * float64(time.Millisecond)))) + if r.Error != "" { + text = text.Space().Append(r.Error, "text-red-500") + } + return text +} +func (r Row) PrettyFull() api.Textable { + return r.Pretty().NewLine().Add(clicky.Map(map[string]any{"messageId": r.MessageID, "deliveryId": r.DeliveryID, "correlationId": r.CorrelationID, "outcome": r.Outcome, "body": r.Body})) +} diff --git a/pkg/tracing/jms/provider.go b/pkg/tracing/jms/provider.go new file mode 100644 index 0000000..f781126 --- /dev/null +++ b/pkg/tracing/jms/provider.go @@ -0,0 +1,60 @@ +package jms + +import ( + "encoding/json" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/seam" + "github.com/flanksource/telltale/pkg/tracing/target" +) + +type handler struct{ agent seam.AgentSeam[Params, Row] } + +func (handler) Params() Params { return Params{Options: Options{Direction: "both"}} } + +func (handler) Schema() recordresults.ResultType[Row] { + return recordresults.ResultType[Row]{Kind: Kind, Title: "JMS operations", TimeColumn: "startedAt", Follow: true, SearchColumns: []string{"destination", "operation", "error", "handler", "messageId", "correlationId"}} +} + +func (h handler) AgentAvailable(probe string, t target.Target) (bool, error) { + return h.agent.Available(probe, t) +} + +func (h handler) Prepare(ctx dbcontext.Context, p Params, out tracing.Emitter[Row]) (dbcontext.Context, func() error, error) { + return h.agent.Prepare(ctx, p, out) +} + +func (h handler) Handle(ctx dbcontext.Context, p Params, out tracing.Emitter[Row], _ tracing.Records[Row]) error { + return h.agent.Handle(ctx, p, out) +} + +func (h handler) Checkpoint(ctx dbcontext.Context, _ Params, out tracing.Emitter[Row]) error { + var params struct { + Reason string `json:"reason,omitempty"` + } + if _, err := tracing.DecodeCheckpointParams(ctx, ¶ms); err != nil { + return err + } + raw, err := json.Marshal(params) + if err != nil { + return err + } + return h.agent.Checkpoint(ctx, raw, out) +} + +func NewKind(registry *agentclient.Registry) *tracing.Handler[Params, Row] { + agent := seam.AgentSeam[Params, Row]{ + Probe: Kind, Registry: registry, Target: func(p Params) target.Target { return p.Target }, + ArmParams: func(p Params) (json.RawMessage, error) { return json.Marshal(p.Options) }, + DecodeRecord: DecodeRecord, + DecodeBlob: func(blob agentclient.Blob, resource map[string]string) (seam.DecodedBlob[Row], error) { + rows, err := Decode(blob, resource) + return seam.DecodedBlob[Row]{Rows: rows}, err + }, + } + return tracing.NewHandler[Params, Row](handler{agent}, tracing.Capabilities{Live: true}, target.Agent). + WithSecretMasking().WithTruncation(1 << 20) +} diff --git a/pkg/tracing/jmstop/pretty.go b/pkg/tracing/jmstop/pretty.go new file mode 100644 index 0000000..46131d6 --- /dev/null +++ b/pkg/tracing/jmstop/pretty.go @@ -0,0 +1,38 @@ +package jmstop + +import ( + "fmt" + "time" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" +) + +func (s Sample) Pretty() api.Text { + return api.Text{}.Append("JMS handler", "font-bold").Space().Append(s.Destination).Space().Append(s.Handler).Space().Append(s.State, "text-muted").Space().Append(fmt.Sprintf("%d active", s.ActiveCount)).Space().Add(clicky.Human(time.Duration(s.ElapsedMs * float64(time.Millisecond)))) +} + +func (s Sample) PrettyFull() api.Textable { + return s.Pretty().NewLine().Add(clicky.Map(map[string]any{"source": s.Source, "handlerId": s.HandlerID, "subscription": s.Subscription, "sampledAt": s.SampledAt, "resource": s.Resource})) +} + +func (s Sample) RowDetail() api.Textable { return s.PrettyFull() } + +func (Sample) Columns() []api.ColumnDef { + return []api.ColumnDef{ + api.Column("destination").Label("Destination").MaxWidth(16).Build(), + api.Column("handler").Label("Handler").MaxWidth(32).Build(), + api.Column("state").Label("State").Build(), + api.Column("activeCount").Label("Active").Format(api.FormatInteger).Build(), + api.Column("elapsedMs").Label("Elapsed").Type("duration").Unit("ms").Build(), + api.Column("subscription").Label("Subscription").DefaultHidden().Build(), + api.Column("source").Label("Source").DefaultHidden().Build(), + api.Column("handlerId").Label("Handler ID").DefaultHidden().Build(), + api.Column("sampledAt").Label("Sampled at").DefaultHidden().Build(), + api.Column("resource").Label("Resource").DefaultHidden().Build(), + } +} + +func (s Sample) Row() map[string]any { + return map[string]any{"destination": s.Destination, "handler": s.Handler, "state": s.State, "activeCount": api.Text{Content: fmt.Sprint(s.ActiveCount)}, "elapsedMs": s.ElapsedMs, "subscription": s.Subscription, "source": s.Source, "handlerId": s.HandlerID, "sampledAt": s.SampledAt, "resource": s.Resource} +} diff --git a/pkg/tracing/jmstop/top.go b/pkg/tracing/jmstop/top.go new file mode 100644 index 0000000..347e689 --- /dev/null +++ b/pkg/tracing/jmstop/top.go @@ -0,0 +1,137 @@ +package jmstop + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/otlp/profiles" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/seam" + "github.com/flanksource/telltale/pkg/tracing/target" +) + +const Kind = "jms_top" + +type Params struct { + Target target.Target `json:"target,omitzero"` + Destinations []string `json:"destinations,omitempty" clicky:"title=Destinations,desc=Only map handlers for these JMS destinations"` + Poll string `json:"poll,omitempty" clicky:"title=Poll,desc=Sampling interval such as 5s"` + Top int `json:"top,omitempty" clicky:"title=Top,desc=Maximum handler mappings per sample; default 50"` +} + +func (p Params) Validate() error { + for _, destination := range p.Destinations { + if strings.TrimSpace(destination) == "" { + return fmt.Errorf("destinations must contain nonblank names") + } + } + if p.Top < 0 || p.Top > 10000 { + return fmt.Errorf("top must be between zero and 10000") + } + if p.Poll != "" { + if d, err := time.ParseDuration(p.Poll); err != nil || d < time.Millisecond { + return fmt.Errorf("poll must be at least 1ms") + } + } + return nil +} + +type Sample struct { + Source string `json:"source" filter:"terms"` + HandlerID string `json:"handlerId"` + Destination string `json:"destination" filter:"terms"` + Handler string `json:"handler" filter:"text"` + Subscription string `json:"subscription,omitempty" filter:"terms"` + State string `json:"state" filter:"terms"` + ActiveCount int `json:"activeCount"` + ElapsedMs float64 `json:"elapsedMs" pretty:"type=duration,unit=ms"` + SampledAt time.Time `json:"sampledAt"` + Resource map[string]string `json:"resource,omitempty"` +} + +func ResultType() recordresults.ResultType[Sample] { + return recordresults.ResultType[Sample]{Kind: Kind, Title: "JMS receive handlers", TimeColumn: "sampledAt", Follow: true, SearchColumns: []string{"destination", "handler", "subscription"}} +} +func FromSnapshot(s profiles.Snapshot) ([]Sample, error) { + if s.Kind != Kind { + return nil, fmt.Errorf("JMS top cannot decode %q", s.Kind) + } + rows := make([]Sample, 0, len(s.Rows)) + for _, r := range s.Rows { + data, err := json.Marshal(r.Attributes) + if err != nil { + return nil, err + } + var row Sample + if err := json.Unmarshal(data, &row); err != nil { + return nil, err + } + if row.Handler == "" || row.ActiveCount < 0 || (row.State != "idle" && row.State != "busy") { + return nil, fmt.Errorf("invalid JMS handler mapping %q", r.OperationID) + } + row.Source, row.HandlerID, row.SampledAt, row.ElapsedMs = s.Frame().Source, r.OperationID, r.SampledAt, float64(r.ElapsedNS)/1e6 + row.Resource = map[string]string{} + for k, v := range s.Resource { + row.Resource[k] = fmt.Sprint(v) + } + rows = append(rows, row) + } + return rows, nil +} +func decode(b agentclient.Blob, resource map[string]string) (seam.DecodedBlob[Sample], error) { + if b.Info.ContentType != profiles.ContentType || b.Info.Session == "" || resource["telltale.source.instance"] == "" { + return seam.DecodedBlob[Sample]{}, fmt.Errorf("JMS top requires authenticated OTLP Profiles and source identity") + } + snapshots, err := profiles.Decode(b.Data, profiles.Options{ExpectedKind: Kind, ExpectedCaptureID: b.Info.Session, ExpectedSource: resource["telltale.source.instance"]}) + if err != nil { + return seam.DecodedBlob[Sample]{}, err + } + if len(snapshots) != 1 { + return seam.DecodedBlob[Sample]{}, fmt.Errorf("JMS top requires one snapshot") + } + rows, err := FromSnapshot(snapshots[0]) + frame := snapshots[0].Frame() + return seam.DecodedBlob[Sample]{Rows: rows, Frame: &frame}, err +} + +type handler struct { + agent seam.AgentSeam[Params, Sample] +} + +func (handler) Params() Params { return Params{Poll: "5s", Top: 50} } +func (handler) Schema() recordresults.ResultType[Sample] { return ResultType() } +func (h handler) AgentAvailable(p string, t target.Target) (bool, error) { + return h.agent.Available(p, t) +} +func (h handler) Prepare(c dbcontext.Context, p Params, e tracing.Emitter[Sample]) (dbcontext.Context, func() error, error) { + return h.agent.Prepare(c, p, e) +} +func (h handler) Handle(c dbcontext.Context, p Params, e tracing.Emitter[Sample], _ tracing.Records[Sample]) error { + return h.agent.Handle(c, p, e) +} +func (h handler) Checkpoint(c dbcontext.Context, _ Params, e tracing.Emitter[Sample]) error { + return h.agent.Checkpoint(c, json.RawMessage(`{}`), e) +} +func NewKind(registry *agentclient.Registry) *tracing.Handler[Params, Sample] { + agent := seam.AgentSeam[Params, Sample]{Probe: Kind, Registry: registry, Target: func(p Params) target.Target { return p.Target }, DecodeBlob: decode, ArmParams: func(p Params) (json.RawMessage, error) { + if err := p.Validate(); err != nil { + return nil, err + } + d := 5 * time.Second + if p.Poll != "" { + d, _ = time.ParseDuration(p.Poll) + } + limit := p.Top + if limit == 0 { + limit = 50 + } + return json.Marshal(map[string]any{"pollIntervalMs": d.Milliseconds(), "limit": limit, "destinations": p.Destinations}) + }} + return tracing.NewHandler[Params, Sample](handler{agent}, tracing.Capabilities{Live: true, Follow: true}, target.Agent).WithTop(tracing.TopSpec{Identity: []string{"source", "handlerId"}, Order: query.Order{{Column: "activeCount", Desc: true}, {Column: "elapsedMs", Desc: true}, {Column: "handler"}}, PerSource: true}) +} diff --git a/pkg/tracing/jmstop/top_test.go b/pkg/tracing/jmstop/top_test.go new file mode 100644 index 0000000..1a49f39 --- /dev/null +++ b/pkg/tracing/jmstop/top_test.go @@ -0,0 +1,31 @@ +package jmstop + +import ( + "github.com/flanksource/telltale/pkg/otlp/profiles" + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "testing" + "time" +) + +func TestJMSTop(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "JMS Top") } + +var _ = Describe("handler profiles", func() { + It("rejects blank destination names before arming", func() { + Expect((Params{Destinations: []string{"orders", " "}}).Validate()).To(MatchError(ContainSubstring("nonblank"))) + }) + It("round trips an idle mapping without losing its identity", func() { + now := time.Now() + snapshot := profiles.Snapshot{Kind: Kind, CaptureID: "capture", Source: "instance", ProfileID: uuid.NewString(), Sequence: 1, Complete: true, Observed: 1, Limit: 50, StartedAt: now, CompletedAt: now, + Rows: []profiles.Row{{OperationID: "handler-1", SampledAt: now, Attributes: map[string]any{"destination": "orders", "handler": "OrderListener.onMessage", "state": "idle", "activeCount": 0}}}} + data, err := profiles.Encode([]profiles.Snapshot{snapshot}) + Expect(err).NotTo(HaveOccurred()) + decoded, err := profiles.Decode(data, profiles.Options{ExpectedKind: Kind}) + Expect(err).NotTo(HaveOccurred()) + rows, err := FromSnapshot(decoded[0]) + Expect(err).NotTo(HaveOccurred()) + Expect(rows).To(ConsistOf(And(HaveField("HandlerID", "handler-1"), HaveField("State", "idle"), HaveField("ActiveCount", 0)))) + Expect(rows[0].Pretty().String()).To(ContainSubstring("0 active")) + }) +}) diff --git a/pkg/tracing/jvm/agentops/agentops.go b/pkg/tracing/jvm/agentops/agentops.go new file mode 100644 index 0000000..f15fd79 --- /dev/null +++ b/pkg/tracing/jvm/agentops/agentops.go @@ -0,0 +1,78 @@ +// Package agentops holds the typed results of the Java agent's probe, +// evaluation, extension and catalog operations. +package agentops + +import ( + "encoding/json" + "fmt" + "strings" + "time" +) + +// Results decode this package's operation results into their typed models. +var Results = map[string]func(json.RawMessage) (any, error){ + InstrumentOp: func(raw json.RawMessage) (any, error) { return DecodeInstrument(raw) }, + ProbesListOp: func(raw json.RawMessage) (any, error) { return DecodeProbes(raw) }, + ProbesReadOp: func(raw json.RawMessage) (any, error) { return DecodeProbeRead(raw) }, + ProbesFreezeOp: func(raw json.RawMessage) (any, error) { return DecodeFreeze(raw) }, + GroovyEvalOp: func(raw json.RawMessage) (any, error) { return DecodeGroovy(raw) }, + OgnlEvalOp: func(raw json.RawMessage) (any, error) { return DecodeOgnl(raw) }, + ExtensionsListOp: func(raw json.RawMessage) (any, error) { return DecodeExtensions(raw) }, + ExtensionsInstallOp: func(raw json.RawMessage) (any, error) { return DecodeExtensionInstall(raw, false) }, + ExtensionsReplaceOp: func(raw json.RawMessage) (any, error) { return DecodeExtensionInstall(raw, true) }, + ExtensionsUnloadOp: func(raw json.RawMessage) (any, error) { return DecodeExtensionUnload(raw) }, + CatalogOp: func(raw json.RawMessage) (any, error) { return DecodeCatalog(raw) }, +} + +// decode unmarshals op's result, refusing it unless every key in required is present. +func decode(op string, raw json.RawMessage, into any, required ...string) error { + if err := json.Unmarshal(raw, into); err != nil { + return fmt.Errorf("decode %s result: %w", op, err) + } + if len(required) == 0 { + return nil + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil { + return fmt.Errorf("decode %s result: %w", op, err) + } + for _, key := range required { + if _, ok := fields[key]; !ok { + return fmt.Errorf("%s returned no %s: %s", op, key, raw) + } + } + return nil +} + +func shortType(name string) string { return name[strings.LastIndex(name, ".")+1:] } + +// shortError names each exception class in text by its simple name. +func shortError(text string) string { + parts := strings.Split(text, ": ") + for i, part := range parts { + simple := shortType(part) + if strings.Contains(part, ".") && !strings.Contains(part, " ") && simple != "" && simple[0] >= 'A' && simple[0] <= 'Z' { + parts[i] = simple + } + } + return strings.Join(parts, ": ") +} + +func clock(at time.Time) string { return at.Local().Format("15:04:05.000") } + +func formatDuration(d time.Duration) string { + switch { + case d < time.Millisecond: + return fmt.Sprintf("%dµs", (d+500*time.Nanosecond)/time.Microsecond) + case d < time.Second: + return fmt.Sprintf("%.1fms", float64(d)/float64(time.Millisecond)) + } + return fmt.Sprintf("%.2fs", d.Seconds()) +} + +func plural(n int64, one, many string) string { + if n == 1 { + return "1 " + one + } + return fmt.Sprintf("%d %s", n, many) +} diff --git a/pkg/tracing/jvm/agentops/agentops_test.go b/pkg/tracing/jvm/agentops/agentops_test.go new file mode 100644 index 0000000..a866c43 --- /dev/null +++ b/pkg/tracing/jvm/agentops/agentops_test.go @@ -0,0 +1,42 @@ +package agentops_test + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/flanksource/clicky/api" + "github.com/flanksource/telltale/pkg/tracing/jvm/agentops" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestAgentOps(t *testing.T) { + time.Local = time.UTC + RegisterFailHandler(Fail) + RunSpecs(t, "JVM agent operation results") +} + +func plain(pretty interface{ Pretty() api.Text }) []string { + return strings.Split(pretty.Pretty().String(), "\n") +} + +func rows[T api.TableProvider](items []T) []map[string]any { + result := make([]map[string]any, len(items)) + for i, item := range items { + result[i] = item.Row() + } + return result +} + +var _ = Describe("result registry", func() { + It("decodes every operation of this slice and refuses a result that is not its own", func() { + Expect(agentops.Results).To(HaveLen(11)) + for op, decode := range agentops.Results { + _, err := decode(json.RawMessage(`{}`)) + Expect(err).To(HaveOccurred(), op) + Expect(err.Error()).To(ContainSubstring(op)) + } + }) +}) diff --git a/pkg/tracing/jvm/agentops/catalog.go b/pkg/tracing/jvm/agentops/catalog.go new file mode 100644 index 0000000..c4e082c --- /dev/null +++ b/pkg/tracing/jvm/agentops/catalog.go @@ -0,0 +1,81 @@ +package agentops + +import ( + "encoding/json" + "fmt" + + "github.com/flanksource/clicky/api" +) + +// CatalogOp describes every operation the agent and its extensions serve. +const CatalogOp = "jvm.catalog" + +// Catalog is the agent's operations, built-in first, then each extension's. +type Catalog []Operation + +// Operation is one catalog descriptor. Extension names the extension that +// serves it; it is empty for a built-in operation. +type Operation struct { + Name string `json:"name"` + Title string `json:"title"` + Description string `json:"description"` + Command string `json:"command,omitempty"` + ParamsSchema json.RawMessage `json:"paramsSchema"` + ResultSchema json.RawMessage `json:"resultSchema"` + Mutates bool `json:"mutates"` + Scope string `json:"scope"` + Extension string `json:"extension,omitempty"` +} + +// DecodeCatalog decodes the agent's catalog, refusing an operation without a name. +func DecodeCatalog(raw json.RawMessage) (Catalog, error) { + var result Catalog + if err := decode(CatalogOp, raw, &result); err != nil { + return result, err + } + if result == nil { + return result, fmt.Errorf("%s returned no operations: %s", CatalogOp, raw) + } + for i, operation := range result { + if operation.Name == "" { + return result, fmt.Errorf("%s returned operation %d without a name: %s", CatalogOp, i, raw) + } + } + return result, nil +} + +func (Operation) Columns() []api.ColumnDef { + return []api.ColumnDef{ + api.Column("name").Label("Operation").Build(), + api.Column("command").Label("Command").Build(), + api.Column("scope").Label("Scope").Build(), + api.Column("mutates").Label("").Build(), + api.Column("description").Label("Description").MaxWidth(80).Build(), + } +} + +func (o Operation) Row() map[string]any { + mutates := api.Text{} + if o.Mutates { + mutates = mutates.Append("mutates", "text-yellow-600") + } + return map[string]any{ + "name": o.Name, + "command": o.Command, + "scope": o.Scope, + "mutates": mutates, + "description": o.Description, + } +} + +// Pretty is the operations under a heading counting those that mutate. +func (c Catalog) Pretty() api.Text { + mutating := 0 + for _, operation := range c { + if operation.Mutates { + mutating++ + } + } + return api.Text{}.Append("Catalog", "font-bold").Append(fmt.Sprintf(" %d operations · %d mutates", len(c), mutating), "text-muted"). + NewLine().Add(api.NewTableFrom(c)) +} diff --git a/pkg/tracing/jvm/agentops/eval.go b/pkg/tracing/jvm/agentops/eval.go new file mode 100644 index 0000000..1250808 --- /dev/null +++ b/pkg/tracing/jvm/agentops/eval.go @@ -0,0 +1,87 @@ +package agentops + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/flanksource/clicky/api" +) + +const ( + // GroovyEvalOp runs a Groovy script in the application's class loader. + GroovyEvalOp = "jvm.groovy.eval" + // OgnlEvalOp evaluates an OGNL expression in the application's class loader. + OgnlEvalOp = "jvm.ognl.eval" +) + +// Evaluation is what an expression returned: Value for a null, string, number +// or boolean (as JSON), Graph for any other object. +type Evaluation struct { + Value json.RawMessage `json:"value,omitempty"` + Graph *Node `json:"graph,omitempty"` +} + +// GroovyResult is a script's evaluation with what it printed and how long it ran. +type GroovyResult struct { + Evaluation + Stdout string `json:"stdout"` + Stderr string `json:"stderr"` + ElapsedMs int64 `json:"elapsedMs"` +} + +// DecodeGroovy decodes a script's result, refusing one without a value or graph. +func DecodeGroovy(raw json.RawMessage) (GroovyResult, error) { + var result GroovyResult + if err := decode(GroovyEvalOp, raw, &result, "stdout", "stderr", "elapsedMs"); err != nil { + return result, err + } + return result, requireEvaluated(GroovyEvalOp, raw, result.Evaluation) +} + +// DecodeOgnl decodes an expression's result, refusing one without a value or graph. +func DecodeOgnl(raw json.RawMessage) (Evaluation, error) { + var result Evaluation + if err := decode(OgnlEvalOp, raw, &result); err != nil { + return result, err + } + return result, requireEvaluated(OgnlEvalOp, raw, result) +} + +func requireEvaluated(op string, raw json.RawMessage, e Evaluation) error { + if len(e.Value) == 0 && e.Graph == nil { + return fmt.Errorf("%s returned neither a value nor a graph: %s", op, raw) + } + return nil +} + +// Pretty is "= value" on one line, or the graph's type over its member tree. +func (e Evaluation) Pretty() api.Text { return e.summary("") } + +func (e Evaluation) summary(elapsed string) api.Text { + line := api.Text{}.Append("= ") + if e.Graph != nil { + line = line.Append(shortType(e.Graph.Type), "font-bold") + } else { + line = line.Append(string(e.Value), "font-bold") + } + if elapsed != "" { + line = line.Append(" · "+elapsed, "text-muted") + } + if e.Graph != nil { + line = line.Add(e.Graph.tree("")) + } + return line +} + +// Pretty is the evaluation, then the script's stdout and, in yellow, its stderr. +func (g GroovyResult) Pretty() api.Text { + text := g.summary(fmt.Sprintf("%dms", g.ElapsedMs)) + if out := strings.TrimRight(g.Stdout, "\n"); out != "" { + text = text.NewLine().Append(out) + } + if errs := strings.TrimRight(g.Stderr, "\n"); errs != "" { + text = text.NewLine().Append(errs, "text-yellow-600") + } + return text +} diff --git a/pkg/tracing/jvm/agentops/eval_test.go b/pkg/tracing/jvm/agentops/eval_test.go new file mode 100644 index 0000000..b5cc085 --- /dev/null +++ b/pkg/tracing/jvm/agentops/eval_test.go @@ -0,0 +1,81 @@ +package agentops_test + +import ( + "encoding/json" + + "github.com/flanksource/telltale/pkg/tracing/jvm/agentops" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const agentGroovyMap = `{"graph":{"id":null,"label":"result","kind":"map","path":null,"expandable":false,"type":"java.util.LinkedHashMap","value":null,"raw":null,"children":[` + + `{"id":null,"label":"name","kind":"scalar","path":null,"expandable":false,"type":"java.lang.String","value":"idle","raw":null,"children":null},` + + `{"id":null,"label":"sizes [2]","kind":"list","path":null,"expandable":false,"type":"java.util.ArrayList","value":null,"raw":null,"children":[` + + `{"id":null,"label":"0","kind":"scalar","path":null,"expandable":false,"type":"java.lang.Integer","value":"1","raw":null,"children":null},` + + `{"id":null,"label":"1","kind":"scalar","path":null,"expandable":false,"type":"java.lang.Integer","value":"2","raw":null,"children":null}]},` + + `{"id":null,"label":"nested","kind":"map","path":null,"expandable":false,"type":"java.util.LinkedHashMap","value":null,"raw":null,"children":[` + + `{"id":null,"label":"a","kind":"scalar","path":null,"expandable":false,"type":null,"value":"null","raw":null,"children":null},` + + `{"id":null,"label":"b","kind":"scalar","path":null,"expandable":false,"type":"java.lang.Boolean","value":"true","raw":null,"children":null}]},` + + `{"id":null,"label":"when","kind":"scalar","path":null,"expandable":false,"type":"java.util.Date","value":"Thu Jan 01 02:00:00 IST 1970","raw":null,"children":null}]},` + + `"stdout":"","stderr":"","elapsedMs":173}` + +const agentOgnlThread = `{"graph":{"id":null,"label":"result","kind":"bean","path":null,"expandable":false,"type":"java.lang.Thread","value":null,"raw":null,"children":[` + + `{"id":null,"label":"name","kind":"error","path":null,"expandable":false,"type":"java.lang.String","value":null,"raw":"unreadable: java.lang.reflect.InaccessibleObjectException","children":null},` + + `{"id":null,"label":"tid","kind":"error","path":null,"expandable":false,"type":"long","value":null,"raw":"unreadable: java.lang.reflect.InaccessibleObjectException","children":null}]}}` + +var _ = Describe("groovy and ognl evaluation", func() { + It("prints a scalar groovy result on one line with the script's output below it", func() { + result, err := agentops.DecodeGroovy(json.RawMessage(`{"value":3,"stdout":"hi\n","stderr":"","elapsedMs":211}`)) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(agentops.GroovyResult{Evaluation: agentops.Evaluation{Value: json.RawMessage(`3`)}, Stdout: "hi\n", ElapsedMs: 211})) + Expect(plain(result)).To(Equal([]string{"= 3 · 211ms", "hi"})) + }) + + It("keeps a null groovy result and its stderr", func() { + result, err := agentops.DecodeGroovy(json.RawMessage(`{"value":null,"stdout":"","stderr":"careful\n","elapsedMs":168}`)) + Expect(err).NotTo(HaveOccurred()) + Expect(plain(result)).To(Equal([]string{"= null · 168ms", "careful"})) + }) + + It("draws a groovy object graph as an indented tree", func() { + result, err := agentops.DecodeGroovy(json.RawMessage(agentGroovyMap)) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Graph.Children[1].Children[0]).To(Equal(agentops.Node{Label: "0", Kind: "scalar", Type: "java.lang.Integer", Value: "1"})) + Expect(plain(result)).To(Equal([]string{ + "= LinkedHashMap · 173ms", + "├─ name: String = \"idle\"", + "├─ sizes [2]: ArrayList", + "│ ├─ 0: Integer = 1", + "│ └─ 1: Integer = 2", + "├─ nested: LinkedHashMap", + "│ ├─ a = null", + "│ └─ b: Boolean = true", + "└─ when: Date = Thu Jan 01 02:00:00 IST 1970", + })) + }) + + It("prints a scalar ognl result on one line", func() { + result, err := agentops.DecodeOgnl(json.RawMessage(`{"value":"idle"}`)) + Expect(err).NotTo(HaveOccurred()) + Expect(plain(result)).To(Equal([]string{`= "idle"`})) + }) + + It("marks the unreadable members of an ognl graph", func() { + result, err := agentops.DecodeOgnl(json.RawMessage(agentOgnlThread)) + Expect(err).NotTo(HaveOccurred()) + Expect(plain(result)).To(Equal([]string{ + "= Thread", + "├─ name: String ✗ unreadable: InaccessibleObjectException", + "└─ tid: long ✗ unreadable: InaccessibleObjectException", + })) + }) + + It("refuses an evaluation that carries neither a value nor a graph", func() { + for _, raw := range []string{`{"stdout":"","stderr":"","elapsedMs":1}`, `[]`} { + _, err := agentops.DecodeGroovy(json.RawMessage(raw)) + Expect(err).To(HaveOccurred(), raw) + } + _, err := agentops.DecodeGroovy(json.RawMessage(`{"value":1}`)) + Expect(err).To(HaveOccurred(), "a groovy result always reports its elapsed time") + }) +}) diff --git a/pkg/tracing/jvm/agentops/extensions.go b/pkg/tracing/jvm/agentops/extensions.go new file mode 100644 index 0000000..6b13d4e --- /dev/null +++ b/pkg/tracing/jvm/agentops/extensions.go @@ -0,0 +1,142 @@ +package agentops + +import ( + "encoding/json" + "fmt" + + "github.com/flanksource/clicky/api" +) + +const ( + // ExtensionsListOp lists the diagnostic extensions loaded into the agent. + ExtensionsListOp = "jvm.extensions.list" + // ExtensionsInstallOp loads a new diagnostic extension JAR. + ExtensionsInstallOp = "jvm.extensions.install" + // ExtensionsReplaceOp swaps a loaded extension for a new JAR of the same id. + ExtensionsReplaceOp = "jvm.extensions.replace" + // ExtensionsUnloadOp unloads a diagnostic extension. + ExtensionsUnloadOp = "jvm.extensions.unload" +) + +// Extensions are the loaded diagnostic extensions. +type Extensions []Extension + +// Extension is one loaded JAR: Digest is its SHA-256, Active its in-flight +// calls and scripts, Status "active" or "failed" after a cleanup failure. +type Extension struct { + ID string `json:"id"` + Version string `json:"version"` + Digest string `json:"digest"` + Active int `json:"active"` + Status string `json:"status"` + Operations []string `json:"operations"` +} + +// DecodeExtensions decodes the agent's extension list. +func DecodeExtensions(raw json.RawMessage) (Extensions, error) { + var result Extensions + if err := decode(ExtensionsListOp, raw, &result); err != nil { + return result, err + } + if result == nil { + return result, fmt.Errorf("%s returned no extension list: %s", ExtensionsListOp, raw) + } + for _, extension := range result { + if extension.ID == "" { + return result, fmt.Errorf("%s returned an extension without an id: %s", ExtensionsListOp, raw) + } + } + return result, nil +} + +func (Extension) Columns() []api.ColumnDef { + return []api.ColumnDef{ + api.Column("id").Label("ID").Build(), + api.Column("version").Label("Version").Build(), + api.Column("status").Label("Status").Build(), + api.Column("active").Label("Active").Build(), + api.Column("operations").Label("Operations").Build(), + api.Column("digest").Label("Digest").Build(), + } +} + +func (e Extension) Row() map[string]any { + status := api.Text{}.Append(e.Status, "text-green-600") + if e.Status != "active" { + status = api.Text{}.Append(e.Status, "text-red-600") + } + digest := e.Digest + if len(digest) > 12 { + digest = digest[:12] + } + return map[string]any{ + "id": e.ID, + "version": e.Version, + "status": status, + "active": e.Active, + "operations": len(e.Operations), + "digest": digest, + } +} + +// Pretty is the extensions under a heading, or one muted line when none is loaded. +func (e Extensions) Pretty() api.Text { + if len(e) == 0 { + return api.Text{}.Append("no extensions loaded", "text-muted") + } + return api.Text{}.Append("Extensions", "font-bold").Append(fmt.Sprintf(" %d loaded", len(e)), "text-muted"). + NewLine().Add(api.NewTableFrom(e)) +} + +// ExtensionInstalled is the id an install or replace loaded. +type ExtensionInstalled struct { + Installed string `json:"installed"` + replaced bool +} + +// DecodeExtensionInstall decodes an install's result, or a replace's when replaced. +func DecodeExtensionInstall(raw json.RawMessage, replaced bool) (ExtensionInstalled, error) { + op := ExtensionsInstallOp + if replaced { + op = ExtensionsReplaceOp + } + result := ExtensionInstalled{replaced: replaced} + if err := decode(op, raw, &result); err != nil { + return result, err + } + if result.Installed == "" { + return result, fmt.Errorf("%s returned no installed extension: %s", op, raw) + } + return result, nil +} + +// Pretty is the install acknowledged on one line. +func (e ExtensionInstalled) Pretty() api.Text { + verb := "installed" + if e.replaced { + verb = "replaced" + } + return api.Text{}.Append("✓ "+verb+" extension "+e.Installed, "text-green-600") +} + +// ExtensionUnloaded is the id an unload removed. +type ExtensionUnloaded struct { + Unloaded string `json:"unloaded"` +} + +// DecodeExtensionUnload decodes an unload's result. +func DecodeExtensionUnload(raw json.RawMessage) (ExtensionUnloaded, error) { + var result ExtensionUnloaded + if err := decode(ExtensionsUnloadOp, raw, &result); err != nil { + return result, err + } + if result.Unloaded == "" { + return result, fmt.Errorf("%s returned no unloaded extension: %s", ExtensionsUnloadOp, raw) + } + return result, nil +} + +// Pretty is the unload acknowledged on one line. +func (e ExtensionUnloaded) Pretty() api.Text { + return api.Text{}.Append("✓ unloaded extension "+e.Unloaded, "text-green-600") +} diff --git a/pkg/tracing/jvm/agentops/extensions_test.go b/pkg/tracing/jvm/agentops/extensions_test.go new file mode 100644 index 0000000..41b63bb --- /dev/null +++ b/pkg/tracing/jvm/agentops/extensions_test.go @@ -0,0 +1,67 @@ +package agentops_test + +import ( + "encoding/json" + + "github.com/flanksource/clicky/api" + "github.com/flanksource/telltale/pkg/tracing/jvm/agentops" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// Built from Extensions.list(): the demo JVM has no extensionsDir, so nothing can be installed in it. +const sourceExtensions = `[{"id":"acme-cache","version":"1.2.0","digest":"3f5a9c0e1b2d4f6a8c0e2b4d6f8a0c2e4b6d8f0a2c4e6b8d0f2a4c6e8b0d2f4a","active":0,"status":"active","operations":["extension.acme-cache.stats","extension.acme-cache.evict"]}]` + +const agentCatalog = `[{"name":"jvm.probes.list","title":"jvm.probes.list","description":"List method probe keys","command":"jvm trace probes","paramsSchema":{"type":"object","properties":{},"required":[],"additionalProperties":false},"resultSchema":{"type":"object"},"mutates":false,"scope":"jvm:read"},` + + `{"name":"jvm.probes.freeze","title":"jvm.probes.freeze","description":"Stop a method probe from recording further events","command":"jvm trace freeze","paramsSchema":{"type":"object","properties":{"key":{"type":"string","title":"key"}},"required":["key"],"additionalProperties":false},"resultSchema":{"type":"object"},"mutates":true,"scope":"jvm:trace"}]` + +var _ = Describe("extensions and the catalog", func() { + It("tabulates the loaded extensions", func() { + result, err := agentops.DecodeExtensions(json.RawMessage(sourceExtensions)) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(agentops.Extensions{{ + ID: "acme-cache", Version: "1.2.0", Status: "active", + Digest: "3f5a9c0e1b2d4f6a8c0e2b4d6f8a0c2e4b6d8f0a2c4e6b8d0f2a4c6e8b0d2f4a", + Operations: []string{"extension.acme-cache.stats", "extension.acme-cache.evict"}, + }})) + Expect(rows(result)).To(Equal([]map[string]any{{ + "id": "acme-cache", "version": "1.2.0", "status": api.Text{}.Append("active", "text-green-600"), + "active": 0, "operations": 2, "digest": "3f5a9c0e1b2d", + }})) + Expect(plain(result)[0]).To(Equal("Extensions 1 loaded")) + }) + + It("says no extension is loaded on one line", func() { + result, err := agentops.DecodeExtensions(json.RawMessage(`[]`)) + Expect(err).NotTo(HaveOccurred()) + Expect(plain(result)).To(Equal([]string{"no extensions loaded"})) + }) + + It("acknowledges an install, a replace and an unload on one line each", func() { + installed, err := agentops.Results[agentops.ExtensionsInstallOp](json.RawMessage(`{"installed":"acme-cache"}`)) + Expect(err).NotTo(HaveOccurred()) + replaced, err := agentops.Results[agentops.ExtensionsReplaceOp](json.RawMessage(`{"installed":"acme-cache"}`)) + Expect(err).NotTo(HaveOccurred()) + unloaded, err := agentops.DecodeExtensionUnload(json.RawMessage(`{"unloaded":"acme-cache"}`)) + Expect(err).NotTo(HaveOccurred()) + Expect([][]string{ + plain(installed.(interface{ Pretty() api.Text })), plain(replaced.(interface{ Pretty() api.Text })), plain(unloaded), + }).To(Equal([][]string{{"✓ installed extension acme-cache"}, {"✓ replaced extension acme-cache"}, {"✓ unloaded extension acme-cache"}})) + }) + + It("tabulates the catalog's operations, marking those that mutate", func() { + result, err := agentops.DecodeCatalog(json.RawMessage(agentCatalog)) + Expect(err).NotTo(HaveOccurred()) + Expect(string(result[1].ParamsSchema)).To(Equal(`{"type":"object","properties":{"key":{"type":"string","title":"key"}},"required":["key"],"additionalProperties":false}`)) + Expect(rows(result)).To(Equal([]map[string]any{ + {"name": "jvm.probes.list", "command": "jvm trace probes", "scope": "jvm:read", "mutates": api.Text{}, "description": "List method probe keys"}, + {"name": "jvm.probes.freeze", "command": "jvm trace freeze", "scope": "jvm:trace", "mutates": api.Text{}.Append("mutates", "text-yellow-600"), "description": "Stop a method probe from recording further events"}, + })) + Expect(plain(result)[0]).To(Equal("Catalog 2 operations · 1 mutates")) + }) + + It("refuses a catalog entry without a name", func() { + _, err := agentops.DecodeCatalog(json.RawMessage(`[{"title":"x"}]`)) + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/pkg/tracing/jvm/agentops/graph.go b/pkg/tracing/jvm/agentops/graph.go new file mode 100644 index 0000000..1788a58 --- /dev/null +++ b/pkg/tracing/jvm/agentops/graph.go @@ -0,0 +1,70 @@ +package agentops + +import ( + "fmt" + + "github.com/flanksource/clicky/api" +) + +// Node is one Java value as the agent's object graph renders it: a scalar, +// map, list, bean, identity, opaque or error. ID and Path are unset in a probe +// snapshot; Raw carries an error node's reason. +type Node struct { + ID string `json:"id,omitempty"` + Label string `json:"label,omitempty"` + Kind string `json:"kind"` + Path string `json:"path,omitempty"` + Expandable bool `json:"expandable,omitempty"` + Type string `json:"type,omitempty"` + Value string `json:"value,omitempty"` + Raw string `json:"raw,omitempty"` + Children []Node `json:"children,omitempty"` +} + +// compact is a scalar as itself, a string quoted, and anything else by its +// short type with its value or member count. +func (n Node) compact() string { + switch { + case n.Kind == "error": + return "✗ " + shortError(n.Raw) + case n.Kind == "scalar" && n.Type == "java.lang.String": + return fmt.Sprintf("%q", n.Value) + case n.Kind == "scalar" && n.Value == "": + return "null" + case n.Kind == "scalar": + return n.Value + case n.Value != "": + return shortType(n.Type) + "{" + n.Value + "}" + } + return fmt.Sprintf("%s{%d}", shortType(n.Type), len(n.Children)) +} + +// tree is the node's children below it, one "label: Type = value" line each, +// nested members indented under their parent. +func (n Node) tree(indent string) api.Text { + var lines api.Text + for i, child := range n.Children { + mark, next := "├─ ", "│ " + if i == len(n.Children)-1 { + mark, next = "└─ ", " " + } + lines = lines.NewLine().Append(indent+mark, "text-muted").Append(child.line()).Add(child.tree(indent + next)) + } + return lines +} + +func (n Node) line() api.Text { + line := api.Text{}.Append(n.Label, "font-bold") + if n.Type != "" { + line = line.Append(": " + shortType(n.Type)) + } + switch { + case n.Kind == "error": + return line.Append(" ✗ "+shortError(n.Raw), "text-red-600") + case n.Kind == "scalar": + return line.Append(" = " + n.compact()) + case n.Value != "": + return line.Append(" = "+n.Value, "text-muted") + } + return line +} diff --git a/pkg/tracing/jvm/agentops/probe_read.go b/pkg/tracing/jvm/agentops/probe_read.go new file mode 100644 index 0000000..6ce2de9 --- /dev/null +++ b/pkg/tracing/jvm/agentops/probe_read.go @@ -0,0 +1,165 @@ +package agentops + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/flanksource/clicky/api" +) + +// ProbesReadOp reads a probe's buffered events forward from a sequence number. +const ProbesReadOp = "jvm.probes.read" + +// ProbeRead is one read of a probe's ring. A closed probe carries its final +// counters and no events. GC is the span's {count, pauseMs} when measured. +type ProbeRead struct { + Key string `json:"key"` + Generation int64 `json:"generation"` + Events []ProbeEvent `json:"events"` + NextSeq int64 `json:"nextSeq,omitempty"` + OldestSeq int64 `json:"oldestSeq,omitempty"` + WriteSeq int64 `json:"writeSeq"` + Dropped int64 `json:"dropped,omitempty"` + Truncated bool `json:"truncated,omitempty"` + Suppressed int64 `json:"suppressed,omitempty"` + Filtered int64 `json:"filtered"` + Omitted int64 `json:"omitted"` + Degraded bool `json:"degraded,omitempty"` + Capacity int `json:"capacity"` + GC json.RawMessage `json:"gc,omitempty"` + Active bool `json:"active"` + Closed bool `json:"closed"` + Complete bool `json:"complete,omitempty"` + MaxEvents int `json:"maxEvents,omitempty"` + Owners []string `json:"owners,omitempty"` +} + +// ProbeEvent is one event as the JVM rendered it. TS is epoch nanoseconds; +// exit, return and throw events name their method only through Frame. +type ProbeEvent struct { + Seq int64 `json:"seq"` + TS int64 `json:"ts"` + Kind string `json:"kind"` + Key string `json:"key"` + ThreadID int64 `json:"threadId"` + Thread string `json:"thread,omitempty"` + Depth int `json:"depth"` + Frame int64 `json:"frame"` + Parent int64 `json:"parent,omitempty"` + Class string `json:"class,omitempty"` + Method string `json:"method,omitempty"` + Descriptor string `json:"descriptor,omitempty"` + Line int `json:"line,omitempty"` + Args []Node `json:"args,omitempty"` + Value *Node `json:"value,omitempty"` + Error string `json:"error,omitempty"` + DurationNs int64 `json:"durationNs,omitempty"` + AllocBytes *int64 `json:"allocBytes,omitempty"` + GCCount *int64 `json:"gcCount,omitempty"` + GCPauseMs *int64 `json:"gcPauseMs,omitempty"` + Unwound int `json:"unwound,omitempty"` + Truncated bool `json:"truncated,omitempty"` + Omitted int `json:"omitted,omitempty"` +} + +// DecodeProbeRead decodes a read, parsing each event line the JVM rendered, +// and fails when the agent reports the probe is not open. +func DecodeProbeRead(raw json.RawMessage) (ProbeRead, error) { + var wire struct { + ProbeRead + Events []string `json:"events"` + Error string `json:"error"` + } + if err := decode(ProbesReadOp, raw, &wire, "key"); err != nil { + return ProbeRead{}, err + } + if wire.Error != "" { + return ProbeRead{}, fmt.Errorf("%s: %s", ProbesReadOp, wire.Error) + } + if wire.Events == nil { + return ProbeRead{}, fmt.Errorf("%s returned no events: %s", ProbesReadOp, raw) + } + read := wire.ProbeRead + read.Events = make([]ProbeEvent, len(wire.Events)) + for i, line := range wire.Events { + if err := json.Unmarshal([]byte(line), &read.Events[i]); err != nil { + return ProbeRead{}, fmt.Errorf("decode %s event %d %q: %w", ProbesReadOp, i, line, err) + } + } + return read, nil +} + +// Pretty is a heading with the read's position and state over one line per event. +func (r ProbeRead) Pretty() api.Text { + heading := api.Text{}.Append("Probe "+r.Key, "font-bold") + if r.Closed { + return heading.Append(" closed · "+plural(r.WriteSeq, "event", "events")+" written", "text-muted") + } + facts := []string{plural(int64(len(r.Events)), "event", "events")} + if len(r.Events) > 0 { + facts[0] += fmt.Sprintf(", seq %d–%d of %d", r.Events[0].Seq, r.Events[len(r.Events)-1].Seq, r.WriteSeq) + } + facts = append(facts, r.state()) + if r.Truncated { + facts = append(facts, fmt.Sprintf("more from --since %d", r.NextSeq)) + } + text := heading.Append(" "+strings.Join(facts, " · "), "text-muted") + if r.Dropped > 0 { + text = text.Append(fmt.Sprintf(" · ⚠ %d evicted before this read", r.Dropped), "text-yellow-600") + } + names := map[frameKey]string{} + for _, event := range r.Events { + text = text.NewLine().Add(event.line(names)) + } + return text +} + +func (r ProbeRead) state() string { + switch { + case r.Complete: + return "complete" + case r.Active: + return "recording" + } + return "frozen" +} + +type frameKey struct{ thread, frame int64 } + +// line is the event as "time thread kind Class.method…", naming the method of +// an exit, return or throw from the enter of the same frame in names. +func (e ProbeEvent) line(names map[frameKey]string) api.Text { + key := frameKey{e.ThreadID, e.Frame} + name := names[key] + if e.Method != "" { + name = shortType(e.Class) + "." + e.Method + names[key] = name + } + if name == "" { + name = fmt.Sprintf("frame %d", e.Frame) + } + thread := e.Thread + if thread == "" { + thread = fmt.Sprintf("thread-%d", e.ThreadID) + } + line := api.Text{}.Append(clock(time.Unix(0, e.TS))+" "+thread+" "+e.Kind+" ", "text-muted").Append(name, "font-bold") + switch e.Kind { + case "enter": + args := make([]string, len(e.Args)) + for i, arg := range e.Args { + args[i] = arg.compact() + } + line = line.Append("(" + strings.Join(args, ", ") + ")") + case "return": + if e.Value != nil { + line = line.Append(" → " + e.Value.compact()) + } + case "throw", "error": + line = line.Append(" ✗ "+shortError(e.Error), "text-red-600") + case "exit": + line = line.Append(" "+formatDuration(time.Duration(e.DurationNs)), "text-muted") + } + return line +} diff --git a/pkg/tracing/jvm/agentops/probes.go b/pkg/tracing/jvm/agentops/probes.go new file mode 100644 index 0000000..3d1b0de --- /dev/null +++ b/pkg/tracing/jvm/agentops/probes.go @@ -0,0 +1,212 @@ +package agentops + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/flanksource/clicky/api" +) + +const ( + // InstrumentOp weaves a method probe into a class under a restore lease. + InstrumentOp = "jvm.classes.instrument" + // ProbesListOp lists the JVM's open method probes. + ProbesListOp = "jvm.probes.list" + // ProbesFreezeOp stops a probe recording while keeping its buffer readable. + ProbesFreezeOp = "jvm.probes.freeze" +) + +// Instrumented is the class a probe was woven into and the methods it matched. +// LeaseExpiresAt is epoch milliseconds, 0 when the class holds no lease. +type Instrumented struct { + Class string `json:"class"` + Loader string `json:"loader"` + Digest string `json:"digest"` + OriginalDigest string `json:"originalDigest"` + HandedDigest string `json:"handedDigest"` + Patched bool `json:"patched"` + LeaseExpiresAt int64 `json:"leaseExpiresAt"` + Methods []InstrumentedMethod `json:"methods"` + ProbeKey string `json:"probeKey"` + Generation int64 `json:"generation"` + Capacity int `json:"capacity"` + Alloc bool `json:"alloc"` + GC bool `json:"gc"` + Instrumented bool `json:"instrumented"` +} + +// InstrumentedMethod is one method the selector matched; Skipped says why one +// could not be instrumented. +type InstrumentedMethod struct { + Name string `json:"name"` + Descriptor string `json:"descriptor"` + Instrumented bool `json:"instrumented"` + Skipped string `json:"skipped,omitempty"` +} + +// DecodeInstrument decodes the agent's instrument result. +func DecodeInstrument(raw json.RawMessage) (Instrumented, error) { + var result Instrumented + if err := decode(InstrumentOp, raw, &result, "class", "probeKey", "methods", "instrumented"); err != nil { + return result, err + } + return result, nil +} + +// Pretty is the instrument acknowledged on one line. +// Failure is why no method was instrumented, for a command that must exit non-zero. +func (r Instrumented) Failure() error { + if r.Instrumented { + return nil + } + reasons := []string{} + for _, method := range r.Methods { + if method.Skipped != "" { + reasons = append(reasons, method.Name+method.Descriptor+" skipped: "+method.Skipped) + } + } + if len(reasons) == 0 { + return fmt.Errorf("%s was not instrumented", r.Class) + } + return fmt.Errorf("%s was not instrumented: %s", r.Class, strings.Join(reasons, "; ")) +} + +func (r Instrumented) Pretty() api.Text { + var armed, skipped []string + for _, method := range r.Methods { + name := shortType(r.Class) + "." + method.Name + method.Descriptor + if method.Instrumented { + armed = append(armed, name) + } else { + skipped = append(skipped, name+": "+method.Skipped) + } + } + if !r.Instrumented { + return api.Text{}.Append(fmt.Sprintf("✗ %s was not instrumented: the rewrite changed nothing · probe %s", r.Class, r.ProbeKey), "text-red-600") + } + facts := []string{"probe " + r.ProbeKey} + if r.LeaseExpiresAt > 0 { + facts = append(facts, "lease until "+time.UnixMilli(r.LeaseExpiresAt).Local().Format("15:04:05")) + } + facts = append(facts, fmt.Sprintf("ring %d", r.Capacity)) + if r.Alloc { + facts = append(facts, "alloc") + } + if r.GC { + facts = append(facts, "gc") + } + line := api.Text{}.Append("✓ instrumented ", "text-green-600").Append(strings.Join(armed, ", "), "font-bold"). + Append(" · "+strings.Join(facts, " · "), "text-muted") + if len(skipped) > 0 { + line = line.Append(" · skipped "+strings.Join(skipped, ", "), "text-yellow-600") + } + return line +} + +// Probes are the JVM's open method probes. +type Probes struct { + Keys []ProbeKey `json:"keys"` +} + +// ProbeKey is one open probe. Owners are "loader|class"; WriteSeq-OldestSeq +// events are buffered. Armings are the options each owner armed it with. +type ProbeKey struct { + Key string `json:"key"` + Owners []string `json:"owners"` + Generation int64 `json:"generation"` + Capacity int `json:"capacity"` + WriteSeq int64 `json:"writeSeq"` + OldestSeq int64 `json:"oldestSeq"` + Degraded bool `json:"degraded"` + Armings []map[string]any `json:"armings,omitempty"` +} + +// DecodeProbes decodes the agent's probe list. +func DecodeProbes(raw json.RawMessage) (Probes, error) { + var result Probes + if err := decode(ProbesListOp, raw, &result); err != nil { + return result, err + } + if result.Keys == nil { + return result, fmt.Errorf("%s returned no keys: %s", ProbesListOp, raw) + } + return result, nil +} + +func (ProbeKey) Columns() []api.ColumnDef { + return []api.ColumnDef{ + api.Column("key").Label("Key").Build(), + api.Column("class").Label("Class").Build(), + api.Column("method").Label("Method").Build(), + api.Column("events").Label("Events").Build(), + api.Column("capacity").Label("Ring").Build(), + api.Column("degraded").Label("").Build(), + } +} + +func (p ProbeKey) Row() map[string]any { + var classes, methods []string + for _, owner := range p.Owners { + _, class, _ := strings.Cut(owner, "|") + classes = append(classes, shortType(class)) + } + for _, arming := range p.Armings { + if method, ok := arming["method"].(string); ok { + methods = append(methods, method) + } + } + degraded := "" + if p.Degraded { + degraded = "degraded" + } + return map[string]any{ + "key": p.Key, + "class": strings.Join(classes, ", "), + "method": strings.Join(methods, ", "), + "events": p.WriteSeq - p.OldestSeq, + "capacity": p.Capacity, + "degraded": degraded, + } +} + +// Pretty is the open probes under a heading, or one muted line when none is open. +func (p Probes) Pretty() api.Text { + if len(p.Keys) == 0 { + return api.Text{}.Append("no probes", "text-muted") + } + return api.Text{}.Append("Probes", "font-bold").Append(fmt.Sprintf(" %d open", len(p.Keys)), "text-muted"). + NewLine().Add(api.NewTableFrom(p.Keys)) +} + +// Frozen is a probe that stopped recording; Changed is false when it already had. +type Frozen struct { + Key string `json:"key"` + Frozen bool `json:"frozen"` + Changed bool `json:"changed"` + Generation int64 `json:"generation"` + WriteSeq int64 `json:"writeSeq"` + Capacity int `json:"capacity"` +} + +// DecodeFreeze decodes the agent's freeze result. +func DecodeFreeze(raw json.RawMessage) (Frozen, error) { + var result Frozen + if err := decode(ProbesFreezeOp, raw, &result, "key", "frozen", "changed"); err != nil { + return result, err + } + if !result.Frozen { + return result, fmt.Errorf("%s did not freeze probe %s: %s", ProbesFreezeOp, result.Key, raw) + } + return result, nil +} + +// Pretty is the freeze acknowledged on one line. +func (f Frozen) Pretty() api.Text { + head := "✓ froze probe " + f.Key + if !f.Changed { + head = "✓ probe " + f.Key + " was already frozen" + } + return api.Text{}.Append(head, "text-green-600").Append(" · "+plural(f.WriteSeq, "event", "events")+" written", "text-muted") +} diff --git a/pkg/tracing/jvm/agentops/probes_test.go b/pkg/tracing/jvm/agentops/probes_test.go new file mode 100644 index 0000000..f43a081 --- /dev/null +++ b/pkg/tracing/jvm/agentops/probes_test.go @@ -0,0 +1,109 @@ +package agentops_test + +import ( + "encoding/json" + + "github.com/flanksource/telltale/pkg/tracing/jvm/agentops" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const agentInstrumented = `{"class":"Idle","loader":"18a70f16","digest":"sha256:6d8883aba2914eab63e4f3ef18dc10a084b3945211f2917643ff2294e049accf","originalDigest":"sha256:8ee7b08b76749dc3475e5bcad8a829a7f261d1d70d9abc5d4b7945e6356d84d2","handedDigest":"sha256:6d8883aba2914eab63e4f3ef18dc10a084b3945211f2917643ff2294e049accf","patched":true,"leaseExpiresAt":1794805019014,"methods":[{"name":"step","descriptor":"(J)J","instrumented":true}],"probeKey":"agentops-values","generation":1794804899014,"capacity":4096,"alloc":false,"gc":false,"instrumented":true}` + +const agentProbes = `{"keys":[{"key":"agentops-args","owners":["18a70f16|Idle"],"generation":1794804880129,"capacity":4096,"writeSeq":228,"oldestSeq":0,"degraded":false,"armings":[{"owner":"18a70f16|Idle","method":"work","calls":false}]},{"key":"agentops-values","owners":["18a70f16|Idle"],"generation":1794804899014,"capacity":4096,"writeSeq":126,"oldestSeq":0,"degraded":false,"armings":[{"owner":"18a70f16|Idle","method":"step","calls":false}]}]}` + +const agentProbeRead = `{"key":"agentops-values","generation":1794804899014,"events":[` + + `"{\"seq\":0,\"ts\":1794804899088676209,\"kind\":\"enter\",\"key\":\"agentops-values\",\"threadId\":3,\"thread\":\"main\",\"depth\":1,\"frame\":155,\"parent\":154,\"class\":\"Idle\",\"method\":\"step\",\"descriptor\":\"(J)J\",\"args\":[{\"label\":\"arg0\",\"kind\":\"scalar\",\"type\":\"java.lang.Long\",\"value\":\"101626\"}]}",` + + `"{\"seq\":1,\"ts\":1794804899089983750,\"kind\":\"return\",\"key\":\"agentops-values\",\"threadId\":3,\"thread\":\"main\",\"depth\":2,\"frame\":155,\"value\":{\"label\":\"return\",\"kind\":\"scalar\",\"type\":\"java.lang.Long\",\"value\":\"304878\"}}",` + + `"{\"seq\":2,\"ts\":1794804899089991917,\"kind\":\"exit\",\"key\":\"agentops-values\",\"threadId\":3,\"thread\":\"main\",\"depth\":1,\"frame\":155,\"durationNs\":1316791}",` + + `"{\"seq\":3,\"ts\":1794804899090000417,\"kind\":\"enter\",\"key\":\"agentops-values\",\"threadId\":3,\"thread\":\"main\",\"depth\":1,\"frame\":156,\"parent\":154,\"class\":\"Idle\",\"method\":\"step\",\"descriptor\":\"(J)J\",\"args\":[{\"label\":\"arg0\",\"kind\":\"scalar\",\"type\":\"java.lang.Long\",\"value\":\"50813\"}]}"` + + `],"nextSeq":4,"oldestSeq":0,"writeSeq":6,"dropped":0,"truncated":true,"suppressed":0,"filtered":0,"omitted":0,"degraded":false,"capacity":4096,"active":true,"closed":false,"complete":false,"maxEvents":1000,"owners":["18a70f16|Idle"]}` + +const agentProbeClosed = `{"key":"agentops-values","closed":true,"generation":1794804899014,"writeSeq":570,"capacity":4096,"filtered":0,"omitted":0,"active":false,"events":[]}` + +const agentFrozen = `{"key":"agentops-fixture","frozen":true,"changed":true,"generation":1794804857514,"writeSeq":120,"capacity":4096}` + +var _ = Describe("method probes", func() { + It("decodes an instrument result and acknowledges it on one line", func() { + result, err := agentops.DecodeInstrument(json.RawMessage(agentInstrumented)) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(agentops.Instrumented{ + Class: "Idle", Loader: "18a70f16", Patched: true, Instrumented: true, + Digest: "sha256:6d8883aba2914eab63e4f3ef18dc10a084b3945211f2917643ff2294e049accf", + OriginalDigest: "sha256:8ee7b08b76749dc3475e5bcad8a829a7f261d1d70d9abc5d4b7945e6356d84d2", + HandedDigest: "sha256:6d8883aba2914eab63e4f3ef18dc10a084b3945211f2917643ff2294e049accf", + LeaseExpiresAt: 1794805019014, ProbeKey: "agentops-values", Generation: 1794804899014, Capacity: 4096, + Methods: []agentops.InstrumentedMethod{{Name: "step", Descriptor: "(J)J", Instrumented: true}}, + })) + Expect(plain(result)).To(Equal([]string{"✓ instrumented Idle.step(J)J · probe agentops-values · lease until 04:56:59 · ring 4096"})) + }) + + It("names the methods an instrument skipped and why", func() { + result := agentops.Instrumented{Class: "Idle", ProbeKey: "p", Capacity: 4096, Instrumented: true, Methods: []agentops.InstrumentedMethod{ + {Name: "work", Descriptor: "(J)J", Instrumented: true}, + {Name: "", Descriptor: "()V", Skipped: "constructor"}, + }} + Expect(plain(result)).To(Equal([]string{"✓ instrumented Idle.work(J)J · probe p · ring 4096 · skipped Idle.()V: constructor"})) + }) + + It("reports an instrument that armed no method as a failure, so the command exits non-zero", func() { + armed := agentops.Instrumented{Class: "Idle", ProbeKey: "p", Instrumented: true} + Expect(armed.Failure()).To(Succeed()) + skipped := agentops.Instrumented{Class: "Idle", ProbeKey: "p", Methods: []agentops.InstrumentedMethod{{Name: "", Descriptor: "()V", Skipped: "constructor"}}} + Expect(skipped.Failure()).To(MatchError("Idle was not instrumented: ()V skipped: constructor")) + }) + + It("tabulates each open probe with its class, method and buffered events", func() { + result, err := agentops.DecodeProbes(json.RawMessage(agentProbes)) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Keys[0].Armings).To(Equal([]map[string]any{{"owner": "18a70f16|Idle", "method": "work", "calls": false}})) + Expect(rows(result.Keys)).To(Equal([]map[string]any{ + {"key": "agentops-args", "class": "Idle", "method": "work", "events": int64(228), "capacity": 4096, "degraded": ""}, + {"key": "agentops-values", "class": "Idle", "method": "step", "events": int64(126), "capacity": 4096, "degraded": ""}, + })) + Expect(plain(result)[0]).To(Equal("Probes 2 open")) + }) + + It("says there are no probes on one line", func() { + result, err := agentops.DecodeProbes(json.RawMessage(`{"keys":[]}`)) + Expect(err).NotTo(HaveOccurred()) + Expect(plain(result)).To(Equal([]string{"no probes"})) + }) + + It("prints a probe's events one per line, naming the method an exit closes", func() { + result, err := agentops.DecodeProbeRead(json.RawMessage(agentProbeRead)) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Events).To(HaveLen(4)) + Expect(result.Events[1]).To(Equal(agentops.ProbeEvent{ + Seq: 1, TS: 1794804899089983750, Kind: "return", Key: "agentops-values", ThreadID: 3, Thread: "main", Depth: 2, Frame: 155, + Value: &agentops.Node{Label: "return", Kind: "scalar", Type: "java.lang.Long", Value: "304878"}, + })) + Expect(plain(result)).To(Equal([]string{ + "Probe agentops-values 4 events, seq 0–3 of 6 · recording · more from --since 4", + "04:54:59.088 main enter Idle.step(101626)", + "04:54:59.089 main return Idle.step → 304878", + "04:54:59.089 main exit Idle.step 1.3ms", + "04:54:59.090 main enter Idle.step(50813)", + })) + }) + + It("reports a closed probe's end on one line", func() { + result, err := agentops.DecodeProbeRead(json.RawMessage(agentProbeClosed)) + Expect(err).NotTo(HaveOccurred()) + Expect(plain(result)).To(Equal([]string{"Probe agentops-values closed · 570 events written"})) + }) + + It("fails on a read of a probe that is not open rather than printing no events", func() { + _, err := agentops.DecodeProbeRead(json.RawMessage(`{"key":"no-such-key","active":false,"closed":false,"error":"no probe is open for key no-such-key"}`)) + Expect(err).To(MatchError(ContainSubstring("no probe is open for key no-such-key"))) + }) + + It("acknowledges a freeze on one line", func() { + result, err := agentops.DecodeFreeze(json.RawMessage(agentFrozen)) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(agentops.Frozen{Key: "agentops-fixture", Frozen: true, Changed: true, Generation: 1794804857514, WriteSeq: 120, Capacity: 4096})) + Expect(plain(result)).To(Equal([]string{"✓ froze probe agentops-fixture · 120 events written"})) + result.Changed = false + Expect(plain(result)).To(Equal([]string{"✓ probe agentops-fixture was already frozen · 120 events written"})) + }) +}) diff --git a/pkg/tracing/jvm/classes/classes.go b/pkg/tracing/jvm/classes/classes.go new file mode 100644 index 0000000..6385962 --- /dev/null +++ b/pkg/tracing/jvm/classes/classes.go @@ -0,0 +1,170 @@ +// Package classes holds the typed results of the Java agent's class browsing, +// decompiling and hot-patching operations. +package classes + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/flanksource/clicky/api" +) + +const ( + ClassesOp = "jvm.classes" + LoadersOp = "jvm.loaders" + SourceOp = "jvm.classes.source" + BytecodeOp = "jvm.classes.bytecode" + CompileOp = "jvm.classes.compile" + PatchOp = "jvm.classes.patch" + RestoreOp = "jvm.classes.restore" + LeaseOp = "jvm.classes.lease" + PatchesOp = "jvm.classes.patches" +) + +// Results decode this package's operation results into their typed models. +var Results = map[string]func(json.RawMessage) (any, error){ + ClassesOp: func(raw json.RawMessage) (any, error) { return DecodeClassPage(raw) }, + LoadersOp: func(raw json.RawMessage) (any, error) { return DecodeLoaders(raw) }, + SourceOp: func(raw json.RawMessage) (any, error) { return DecodeSource(raw) }, + BytecodeOp: func(raw json.RawMessage) (any, error) { return DecodeBytecode(raw) }, + CompileOp: func(raw json.RawMessage) (any, error) { return DecodeCompile(raw) }, + PatchOp: func(raw json.RawMessage) (any, error) { return DecodePatch(raw) }, + RestoreOp: func(raw json.RawMessage) (any, error) { return DecodeRestore(raw) }, + LeaseOp: func(raw json.RawMessage) (any, error) { return DecodeLease(raw) }, + PatchesOp: func(raw json.RawMessage) (any, error) { return DecodePatches(raw) }, +} + +// decodeObject decodes an object result into out, refusing it unless every +// named field is present and not null. +func decodeObject(op string, raw json.RawMessage, out any, fields ...string) error { + var present map[string]json.RawMessage + if err := json.Unmarshal(raw, &present); err != nil || present == nil { + return fmt.Errorf("%s returned %s, not its result object", op, raw) + } + for _, field := range fields { + if value, ok := present[field]; !ok || bytes.Equal(value, []byte("null")) { + return fmt.Errorf("%s result has no %q: %s", op, field, raw) + } + } + if err := json.Unmarshal(raw, out); err != nil { + return fmt.Errorf("decode %s result: %w", op, err) + } + return nil +} + +// decodeRows decodes an array result, refusing null and any row valid rejects. +func decodeRows[T any](op string, raw json.RawMessage, valid func(T) bool) ([]T, error) { + var rows []T + if err := json.Unmarshal(raw, &rows); err != nil { + return nil, fmt.Errorf("decode %s result: %w", op, err) + } + if rows == nil { + return nil, fmt.Errorf("%s returned %s, not a list", op, raw) + } + for i, row := range rows { + if !valid(row) { + return nil, fmt.Errorf("%s row %d is incomplete: %s", op, i, raw) + } + } + return rows, nil +} + +// ClassPage is one page of the loaded classes matching a query. Total counts +// every match; PlatformExcluded is how many JDK/EE classes the listing hid. +type ClassPage struct { + Total int `json:"total"` + Offset int `json:"offset"` + Limit int `json:"limit"` + PlatformExcluded int `json:"platformExcluded"` + Classes []Class `json:"classes"` +} + +// Class is one loaded class and the hash of the loader that defined it. +type Class struct { + Name string `json:"name"` + Loader string `json:"loader"` +} + +func DecodeClassPage(raw json.RawMessage) (ClassPage, error) { + var page ClassPage + err := decodeObject(ClassesOp, raw, &page, "total", "offset", "limit", "platformExcluded", "classes") + return page, err +} + +func (Class) Columns() []api.ColumnDef { + return []api.ColumnDef{ + api.Column("name").Label("Class").MaxWidth(100).Build(), + api.Column("loader").Label("Loader").Build(), + } +} + +func (c Class) Row() map[string]any { + return map[string]any{"name": c.Name, "loader": c.Loader} +} + +func (p ClassPage) Pretty() api.Text { + summary := fmt.Sprintf(" %d-%d of %d", p.Offset+1, p.Offset+len(p.Classes), p.Total) + if len(p.Classes) == 0 { + summary = fmt.Sprintf(" none at offset %d of %d", p.Offset, p.Total) + } + if p.PlatformExcluded > 0 { + summary += fmt.Sprintf(" · %d platform classes hidden", p.PlatformExcluded) + } + text := api.Text{}.Append("Classes", "font-bold").Append(summary, "text-muted") + if len(p.Classes) == 0 { + return text + } + return text.NewLine().Add(api.NewTableFrom(p.Classes)) +} + +// Loaders are the JVM's class loaders, application code first. A virtual row +// stands for a whole family of LoaderCount loaders. +type Loaders []Loader + +type Loader struct { + Hash string `json:"hash"` + ClassName string `json:"className"` + Label string `json:"label"` + Description string `json:"description"` + Kind string `json:"kind"` + ClassCount int `json:"classCount"` + Virtual bool `json:"virtual,omitempty"` + LoaderCount int `json:"loaderCount,omitempty"` +} + +func DecodeLoaders(raw json.RawMessage) (Loaders, error) { + return decodeRows(LoadersOp, raw, func(l Loader) bool { return l.Hash != "" && l.Kind != "" }) +} + +func (Loader) Columns() []api.ColumnDef { + return []api.ColumnDef{ + api.Column("hash").Label("Loader").Build(), + api.Column("label").Label("Label").Build(), + api.Column("kind").Label("Kind").Build(), + api.Column("classCount").Label("Classes").Build(), + api.Column("className").Label("Class").MaxWidth(60).Build(), + api.Column("description").Label("Packages").MaxWidth(60).Build(), + } +} + +func (l Loader) Row() map[string]any { + return map[string]any{ + "hash": l.Hash, + "label": l.Label, + "kind": l.Kind, + "classCount": l.ClassCount, + "className": l.ClassName, + "description": l.Description, + } +} + +func (l Loaders) Pretty() api.Text { + total := 0 + for _, loader := range l { + total += loader.ClassCount + } + return api.Text{}.Append("Class loaders", "font-bold"). + Append(fmt.Sprintf(" %d loaders · %d classes", len(l), total), "text-muted"). + NewLine().Add(api.NewTableFrom(l)) +} diff --git a/pkg/tracing/jvm/classes/classes_test.go b/pkg/tracing/jvm/classes/classes_test.go new file mode 100644 index 0000000..68ebdbd --- /dev/null +++ b/pkg/tracing/jvm/classes/classes_test.go @@ -0,0 +1,207 @@ +package classes_test + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/flanksource/telltale/pkg/tracing/jvm/classes" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestClasses(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "JVM class operation results") } + +const ( + idleLoader = "18a70f16" + appLoader = "14dad5dc" + + agentClasses = `{"total":679,"offset":0,"limit":3,"platformExcluded":3549,"classes":[{"name":"Idle","loader":"18a70f16"},{"name":"com.flanksource.telltale.Agent","loader":"14dad5dc"},{"name":"com.flanksource.telltale.Agent$$Lambda/0x00000e00011c9f40","loader":"14dad5dc"}]}` + + agentLoaders = `[{"hash":"14dad5dc","className":"jdk.internal.loader.ClassLoaders$AppClassLoader","label":"JVM classpath + java agents","description":"com.sun, Jackson, com.flanksource, telltale-diag","kind":"container","classCount":1937},{"hash":"18a70f16","className":"com.sun.tools.javac.launcher.MemoryClassLoader","label":"MemoryClassLoader","description":"Idle","kind":"unknown","classCount":1},{"hash":"0","className":"","label":"JDK bootstrap","description":"java.lang, java.util, jdk.internal, sun.nio","kind":"jdk","classCount":2241},{"hash":"~compiled","className":"jdk.internal.reflect.DelegatingClassLoader","label":"Reflection accessors (12 loaders)","description":"jdk.internal.reflect","kind":"jdk","classCount":12,"virtual":true,"loaderCount":12}]` + + idleSource = "public class Idle {\n static void check(long l) {\n if (l < 0L) {\n throw new IllegalStateException(\"negative \" + l);\n }\n }\n}\n" + + agentSource = `{"class":"Idle","loader":"18a70f16","source":"public class Idle {\n static void check(long l) {\n if (l < 0L) {\n throw new IllegalStateException(\"negative \" + l);\n }\n }\n}\n","methods":[{"name":"check","descriptor":"(J)V","line":3,"endLine":4}],"sourceLineNumbers":[0,0,14,14,0,0,0,0],"lineRows":{"14":3},"digest":"sha256:8ee7b08b76749dc3475e5bcad8a829a7f261d1d70d9abc5d4b7945e6356d84d2","decompiler":"cfr-0.152","durationMs":147,"cached":false}` + + // The first 12 bytes of Idle's live class file: magic, version 0.69 (Java 25), then the constant pool count. + agentBytecode = `{"bytecode":"yv66vgAAAEUARQoA"}` + + agentCompiled = `{"success":true,"classes":["Idle"],"diagnostics":[],"classpathEntries":2}` + agentCompileError = `{"success":false,"classes":[],"diagnostics":[{"kind":"ERROR","line":3,"column":57,"code":"compiler.err.expected","message":"';' expected"},{"kind":"ERROR","line":-1,"column":-1,"code":"compiler.err.cant.resolve","message":"in the generated stand-in for Idle: cannot find symbol"}],"classpathEntries":2}` + + originalDigest = "sha256:8ee7b08b76749dc3475e5bcad8a829a7f261d1d70d9abc5d4b7945e6356d84d2" + patchedDigest = "sha256:1c0e9b85c1656afd46b860a0e2bcfb1ae676eed8993f49fcc861862203f0c963" + leaseExpiresAt = int64(1794804977512) + leaseUntil = "2026-11-16T04:56:17Z" + + agentPatched = `{"success":true,"classes":["Idle"],"diagnostics":[],"classpathEntries":2,"method":"(J)V","class":"Idle","loader":"18a70f16","digest":"` + patchedDigest + `","originalDigest":"` + originalDigest + `","handedDigest":"` + patchedDigest + `","patched":true,"leaseExpiresAt":1794804977512,"applied":true}` + agentPatchRejected = `{"success":false,"classes":[],"diagnostics":[{"kind":"ERROR","line":3,"column":57,"code":"compiler.err.expected","message":"';' expected"}],"classpathEntries":2,"method":"(J)V","applied":false}` + + agentLease = `{"class":"Idle","loader":"18a70f16","patched":true,"leaseExpiresAt":1794804977512}` + agentRestored = `{"class":"Idle","loader":"18a70f16","patched":false,"restored":["Idle"],"digest":"` + originalDigest + `","probesClosed":[{"key":"agentops-args","closed":true,"generation":1794804880129,"writeSeq":144,"capacity":4096}]}` + agentReleased = `{"class":"Idle","loader":"18a70f16","digest":"` + patchedDigest + `","originalDigest":"` + originalDigest + `","patched":true,"leaseExpiresAt":1794804977512,"restored":[],"released":"agentops-args","heldBy":["agentops-fixture"],"probesClosed":[]}` + + agentPatches = `[{"class":"Idle","loader":"18a70f16","digest":"` + patchedDigest + `","originalDigest":"` + originalDigest + `","patched":true,"leaseExpiresAt":1794804977512},{"class":"example.Gone","loader":"5e1f00d","patched":false,"error":"no longer loaded: class not loaded by loader 5e1f00d: example.Gone"}]` +) + +func decode(op, raw string) any { + GinkgoHelper() + value, err := classes.Results[op](json.RawMessage(raw)) + Expect(err).NotTo(HaveOccurred()) + return value +} + +func lines(text interface{ String() string }) []string { + return strings.Split(strings.TrimRight(text.String(), "\n"), "\n") +} + +var _ = Describe("class operation results", func() { + It("registers a decoder for every class operation", func() { + Expect(classes.Results).To(HaveLen(9)) + for _, op := range []string{classes.ClassesOp, classes.LoadersOp, classes.SourceOp, classes.BytecodeOp, classes.CompileOp, classes.PatchOp, classes.RestoreOp, classes.LeaseOp, classes.PatchesOp} { + Expect(classes.Results).To(HaveKey(op)) + } + }) + + It("refuses what is not the operation's result rather than reading it as empty", func() { + refused := map[string][]string{ + classes.ClassesOp: {`{}`, `[]`, `{"total":1,"offset":0,"limit":3,"platformExcluded":0,"classes":null}`}, + classes.LoadersOp: {`{}`, `null`, `[{"label":"no hash"}]`}, + classes.SourceOp: {`{}`, `{"class":"Idle","loader":"18a70f16","source":"","decompiler":"cfr-0.152"}`}, + classes.BytecodeOp: {`{}`, `{"bytecode":"AAAA"}`, `{"bytecode":"not base64"}`}, + classes.CompileOp: {`{}`, `{"success":true}`}, + classes.PatchOp: {`{}`, `{"success":true,"classes":[],"diagnostics":[],"applied":true}`}, + classes.RestoreOp: {`{}`, `{"class":"Idle","loader":"18a70f16","patched":false}`}, + classes.LeaseOp: {`{}`, `{"class":"Idle","loader":"18a70f16"}`}, + classes.PatchesOp: {`{}`, `null`, `[{"patched":true}]`}, + } + for op, raws := range refused { + for _, raw := range raws { + _, err := classes.Results[op](json.RawMessage(raw)) + Expect(err).To(MatchError(ContainSubstring(op)), op+" "+raw) + } + } + }) + + It("pages loaded classes under a heading that says what the page leaves out", func() { + page := decode(classes.ClassesOp, agentClasses).(classes.ClassPage) + Expect(page).To(Equal(classes.ClassPage{Total: 679, Limit: 3, PlatformExcluded: 3549, Classes: []classes.Class{ + {Name: "Idle", Loader: idleLoader}, + {Name: "com.flanksource.telltale.Agent", Loader: appLoader}, + {Name: "com.flanksource.telltale.Agent$$Lambda/0x00000e00011c9f40", Loader: appLoader}, + }})) + Expect(lines(page.Pretty())[0]).To(Equal("Classes 1-3 of 679 · 3549 platform classes hidden")) + Expect(page.Classes[0].Row()).To(Equal(map[string]any{"name": "Idle", "loader": idleLoader})) + }) + + It("tabulates loaders, keeping a merged family's loader count", func() { + loaders := decode(classes.LoadersOp, agentLoaders).(classes.Loaders) + Expect(loaders).To(HaveLen(4)) + Expect(loaders[3]).To(Equal(classes.Loader{Hash: "~compiled", ClassName: "jdk.internal.reflect.DelegatingClassLoader", Label: "Reflection accessors (12 loaders)", Description: "jdk.internal.reflect", Kind: "jdk", ClassCount: 12, Virtual: true, LoaderCount: 12})) + Expect(lines(loaders.Pretty())[0]).To(Equal("Class loaders 4 loaders · 4191 classes")) + Expect(loaders[1].Row()).To(Equal(map[string]any{"hash": idleLoader, "label": "MemoryClassLoader", "kind": "unknown", "classCount": 1, "className": "com.sun.tools.javac.launcher.MemoryClassLoader", "description": "Idle"})) + }) + + It("prints decompiled source under one header line", func() { + source := decode(classes.SourceOp, agentSource).(classes.Source) + Expect(source).To(Equal(classes.Source{ + Class: "Idle", Loader: idleLoader, Source: idleSource, + Methods: []classes.SourceMethod{{Name: "check", Descriptor: "(J)V", Line: 3, EndLine: 4}}, + SourceLineNumbers: []int{0, 0, 14, 14, 0, 0, 0, 0}, LineRows: map[int]int{14: 3}, + Digest: originalDigest, Decompiler: "cfr-0.152", DurationMs: 147, + })) + Expect(source.Pretty().String()).To(Equal("Idle · loader 18a70f16 · cfr-0.152 · 147ms · 1 method\n" + idleSource)) + }) + + It("dumps bytecode in hex under its class file version", func() { + bytecode := decode(classes.BytecodeOp, agentBytecode).(classes.Bytecode) + Expect(bytecode.Bytecode).To(Equal([]byte{0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 0x45, 0, 0x45, 0x0a, 0})) + Expect(lines(bytecode.Pretty())).To(Equal([]string{ + "Bytecode 12B · class file 69.0 (Java 25)", + "00000000 ca fe ba be 00 00 00 45 00 45 0a 00 |.......E.E..|", + })) + }) + + It("acknowledges a clean compile in one line and lists each diagnostic on its own", func() { + compiled := decode(classes.CompileOp, agentCompiled).(classes.Compile) + Expect(compiled).To(Equal(classes.Compile{Success: true, Classes: []string{"Idle"}, Diagnostics: []classes.Diagnostic{}, ClasspathEntries: 2})) + Expect(compiled.Pretty().String()).To(Equal("✓ compiled Idle · 2 classpath entries")) + + failed := decode(classes.CompileOp, agentCompileError).(classes.Compile) + Expect(lines(failed.Pretty())).To(Equal([]string{ + "✗ compile failed · 2 errors · 2 classpath entries", + "ERROR 3:57 ';' expected (compiler.err.expected)", + "ERROR in the generated stand-in for Idle: cannot find symbol (compiler.err.cant.resolve)", + })) + }) + + It("acknowledges an applied patch with its method, loader and lease in one line", func() { + patch := decode(classes.PatchOp, agentPatched).(classes.Patch) + Expect(patch).To(Equal(classes.Patch{ + Compile: classes.Compile{Success: true, Classes: []string{"Idle"}, Diagnostics: []classes.Diagnostic{}, ClasspathEntries: 2}, + Method: "(J)V", Applied: true, + PatchStatus: classes.PatchStatus{Class: "Idle", Loader: idleLoader, Digest: patchedDigest, OriginalDigest: originalDigest, HandedDigest: patchedDigest, Patched: true, LeaseExpiresAt: leaseExpiresAt}, + })) + Expect(patch.Pretty().String()).To(Equal("✓ patched Idle (J)V · loader 18a70f16 · lease until " + leaseUntil + " · sha256:1c0e9b85")) + + rejected := decode(classes.PatchOp, agentPatchRejected).(classes.Patch) + Expect(lines(rejected.Pretty())).To(Equal([]string{ + "✗ not patched (J)V: compile failed · 1 error · 2 classpath entries", + "ERROR 3:57 ';' expected (compiler.err.expected)", + })) + }) + + It("reports a failed compile or a patch that was not applied as a failure, so the command exits non-zero", func() { + failures := func(results ...interface{ Failure() error }) []string { + var messages []string + for _, result := range results { + if err := result.Failure(); err != nil { + messages = append(messages, err.Error()) + } + } + return messages + } + Expect(failures( + decode(classes.CompileOp, agentCompiled).(classes.Compile), + decode(classes.CompileOp, agentCompileError).(classes.Compile), + decode(classes.PatchOp, agentPatched).(classes.Patch), + decode(classes.PatchOp, agentPatchRejected).(classes.Patch), + classes.Patch{Applied: true, PatchStatus: classes.PatchStatus{Class: "Idle", Error: "the patched bytes are identical to the loaded class"}}, + )).To(Equal([]string{ + "compile failed with 2 errors", + "(J)V was not patched: compile failed with 1 error", + "Idle: the patched bytes are identical to the loaded class", + })) + }) + + It("acknowledges a lease renewal in one line", func() { + lease := decode(classes.LeaseOp, agentLease).(classes.Lease) + Expect(lease).To(Equal(classes.Lease{Class: "Idle", Loader: idleLoader, Patched: true, LeaseExpiresAt: leaseExpiresAt})) + Expect(lease.Pretty().String()).To(Equal("✓ Idle lease until " + leaseUntil + " · loader 18a70f16")) + }) + + It("acknowledges a restore, or says which probes still hold the patch", func() { + restored := decode(classes.RestoreOp, agentRestored).(classes.Restore) + Expect(restored).To(Equal(classes.Restore{ + PatchStatus: classes.PatchStatus{Class: "Idle", Loader: idleLoader, Digest: originalDigest}, + Restored: []string{"Idle"}, + ProbesClosed: []map[string]any{{"key": "agentops-args", "closed": true, "generation": float64(1794804880129), "writeSeq": float64(144), "capacity": float64(4096)}}, + })) + Expect(restored.Pretty().String()).To(Equal("✓ restored Idle · loader 18a70f16 · closed probe agentops-args")) + + held := decode(classes.RestoreOp, agentReleased).(classes.Restore) + Expect(held.HeldBy).To(Equal([]string{"agentops-fixture"})) + Expect(held.Pretty().String()).To(Equal("✓ released probe agentops-args · Idle still patched, held by agentops-fixture")) + }) + + It("tabulates patches, or says there are none", func() { + patches := decode(classes.PatchesOp, agentPatches).(classes.Patches) + Expect(patches[1]).To(Equal(classes.PatchStatus{Class: "example.Gone", Loader: "5e1f00d", Error: "no longer loaded: class not loaded by loader 5e1f00d: example.Gone"})) + Expect(lines(patches.Pretty())[0]).To(Equal("Patches 2 classes · 1 patched")) + Expect(classes.PatchRow(patches[0]).Row()).To(Equal(map[string]any{"class": "Idle", "loader": idleLoader, "patched": "patched", "lease": leaseUntil, "digest": "sha256:1c0e9b85", "error": ""})) + + none := decode(classes.PatchesOp, `[]`).(classes.Patches) + Expect(none.Pretty().String()).To(Equal("no active patches")) + }) +}) diff --git a/pkg/tracing/jvm/classes/patch.go b/pkg/tracing/jvm/classes/patch.go new file mode 100644 index 0000000..dd55a7a --- /dev/null +++ b/pkg/tracing/jvm/classes/patch.go @@ -0,0 +1,299 @@ +package classes + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/flanksource/clicky/api" +) + +// Compile is a javac run over replacement source, applied or not. Classes are +// every class file it produced; diagnostics are positioned in the source sent. +type Compile struct { + Success bool `json:"success"` + Classes []string `json:"classes"` + Diagnostics []Diagnostic `json:"diagnostics"` + ClasspathEntries int `json:"classpathEntries"` +} + +// Diagnostic is one javac message. Line and Column are -1 when it fell in the +// generated stand-in rather than the method the caller sent. +type Diagnostic struct { + Kind string `json:"kind"` + Line int64 `json:"line"` + Column int64 `json:"column"` + Code string `json:"code"` + Message string `json:"message"` +} + +func DecodeCompile(raw json.RawMessage) (Compile, error) { + var compile Compile + err := decodeObject(CompileOp, raw, &compile, "success", "classes", "diagnostics") + return compile, err +} + +func (c Compile) Pretty() api.Text { + if !c.Success { + return c.failure("✗ ") + } + text := api.Text{}.Append("✓ compiled "+strings.Join(c.Classes, ", "), "text-green-600"). + Append(" · "+classpath(c.ClasspathEntries), "text-muted") + return c.withDiagnostics(text) +} + +// Failure is why the compile failed, for a command that must exit non-zero. +func (c Compile) Failure() error { + if c.Success { + return nil + } + return fmt.Errorf("compile failed with %s", plural(c.errors(), "error")) +} + +func (c Compile) errors() int { + errors := 0 + for _, d := range c.Diagnostics { + if d.Kind == "ERROR" { + errors++ + } + } + return errors +} + +func (c Compile) failure(prefix string) api.Text { + text := api.Text{}.Append(fmt.Sprintf("%scompile failed · %s", prefix, plural(c.errors(), "error")), "text-red-600"). + Append(" · "+classpath(c.ClasspathEntries), "text-muted") + return c.withDiagnostics(text) +} + +func (c Compile) withDiagnostics(text api.Text) api.Text { + for _, d := range c.Diagnostics { + text = text.NewLine().Append(d.String(), d.class()) + } + return text +} + +func (d Diagnostic) String() string { + line := d.Kind + " " + if d.Line > 0 { + line += fmt.Sprintf("%d:%d ", d.Line, d.Column) + } + line += d.Message + if d.Code != "" { + line += " (" + d.Code + ")" + } + return line +} + +func (d Diagnostic) class() string { + switch d.Kind { + case "ERROR": + return "text-red-600" + case "WARNING", "MANDATORY_WARNING": + return "text-yellow-600" + } + return "text-muted" +} + +// PatchStatus is one class's live bytes against the original held for it. +// LeaseExpiresAt is epoch milliseconds, 0 when nothing will restore it on its +// own; Error says why the class could not be read or was left unchanged. +type PatchStatus struct { + Class string `json:"class"` + Loader string `json:"loader"` + Digest string `json:"digest,omitempty"` + OriginalDigest string `json:"originalDigest,omitempty"` + HandedDigest string `json:"handedDigest,omitempty"` + Patched bool `json:"patched"` + LeaseExpiresAt int64 `json:"leaseExpiresAt,omitempty"` + Error string `json:"error,omitempty"` +} + +// Patch is a compile and, when it succeeded, the redefinition it applied. +// Method is the resolved descriptor of a single replaced method. +type Patch struct { + Compile + PatchStatus + Method string `json:"method,omitempty"` + Applied bool `json:"applied"` +} + +func DecodePatch(raw json.RawMessage) (Patch, error) { + var patch Patch + if err := decodeObject(PatchOp, raw, &patch, "success", "classes", "diagnostics", "applied"); err != nil { + return patch, err + } + if patch.Applied && (patch.Class == "" || patch.Digest == "") { + return patch, fmt.Errorf("%s reported applied without the class it patched: %s", PatchOp, raw) + } + return patch, nil +} + +// Failure is why nothing was patched, for a command that must exit non-zero. +func (p Patch) Failure() error { + target := strings.TrimSpace(p.Class + " " + p.Method) + switch { + case !p.Applied && !p.Success: + return fmt.Errorf("%s was not patched: %w", target, p.Compile.Failure()) + case !p.Applied: + return fmt.Errorf("%s was not patched", target) + case p.Error != "": + return fmt.Errorf("%s: %s", target, p.Error) + } + return nil +} + +func (p Patch) Pretty() api.Text { + target := strings.TrimSpace(p.Class + " " + p.Method) + if !p.Applied { + return p.failure(strings.TrimSpace("✗ not patched "+target) + ": ") + } + if p.Error != "" { + return api.Text{}.Append("✗ "+p.Error, "text-red-600") + } + return api.Text{}.Append("✓ patched "+target, "text-green-600"). + Append(" · loader "+p.Loader+" · ", "text-muted"). + Add(lease(p.LeaseExpiresAt)). + Append(" · "+shortDigest(p.Digest), "text-muted") +} + +// Lease is a patched class's restore deadline after a renewal. +type Lease struct { + Class string `json:"class"` + Loader string `json:"loader"` + Patched bool `json:"patched"` + LeaseExpiresAt int64 `json:"leaseExpiresAt"` +} + +func DecodeLease(raw json.RawMessage) (Lease, error) { + var l Lease + err := decodeObject(LeaseOp, raw, &l, "class", "loader", "patched", "leaseExpiresAt") + return l, err +} + +func (l Lease) Pretty() api.Text { + if !l.Patched { + return api.Text{}.Append(l.Class+" is not patched", "text-yellow-600"). + Append(" · loader "+l.Loader, "text-muted") + } + return api.Text{}.Append("✓ "+l.Class+" ", "text-green-600").Add(lease(l.LeaseExpiresAt)). + Append(" · loader "+l.Loader, "text-muted") +} + +// Restore is a class put back to its original bytes, or, when Released names +// a probe other probes still need (HeldBy), that probe's claim let go of. +type Restore struct { + PatchStatus + Restored []string `json:"restored"` + Released string `json:"released,omitempty"` + HeldBy []string `json:"heldBy,omitempty"` + ProbesClosed []map[string]any `json:"probesClosed"` +} + +func DecodeRestore(raw json.RawMessage) (Restore, error) { + var r Restore + err := decodeObject(RestoreOp, raw, &r, "class", "loader", "patched", "restored") + return r, err +} + +func (r Restore) Pretty() api.Text { + if len(r.HeldBy) > 0 { + return api.Text{}.Append("✓ released probe "+r.Released, "text-green-600"). + Append(" · "+r.Class+" still patched, held by "+strings.Join(r.HeldBy, ", "), "text-muted") + } + detail := " · loader " + r.Loader + if r.Released != "" { + detail += " · released probe " + r.Released + } + var closed []string + for _, probe := range r.ProbesClosed { + closed = append(closed, fmt.Sprint(probe["key"])) + } + switch len(closed) { + case 0: + case 1: + detail += " · closed probe " + closed[0] + default: + detail += " · closed probes " + strings.Join(closed, ", ") + } + return api.Text{}.Append("✓ restored "+strings.Join(r.Restored, ", "), "text-green-600").Append(detail, "text-muted") +} + +// Patches are the classes holding stashed original bytes, by class name. +type Patches []PatchStatus + +// PatchRow is a PatchStatus as a row of the patches table. +type PatchRow PatchStatus + +func DecodePatches(raw json.RawMessage) (Patches, error) { + return decodeRows(PatchesOp, raw, func(p PatchStatus) bool { return p.Class != "" && p.Loader != "" }) +} + +func (PatchRow) Columns() []api.ColumnDef { + return []api.ColumnDef{ + api.Column("class").Label("Class").MaxWidth(100).Build(), + api.Column("loader").Label("Loader").Build(), + api.Column("patched").Label("State").Build(), + api.Column("lease").Label("Lease until").Build(), + api.Column("digest").Label("Digest").Build(), + api.Column("error").Label("Error").MaxWidth(80).Build(), + } +} + +func (p PatchRow) Row() map[string]any { + state, until := "original", "none" + if p.Patched { + state = "patched" + } + if p.Error != "" { + state = "error" + } + if p.LeaseExpiresAt > 0 { + until = leaseTime(p.LeaseExpiresAt) + } + return map[string]any{"class": p.Class, "loader": p.Loader, "patched": state, "lease": until, "digest": shortDigest(p.Digest), "error": p.Error} +} + +func (p Patches) Pretty() api.Text { + if len(p) == 0 { + return api.Text{}.Append("no active patches", "text-muted") + } + rows := make([]PatchRow, len(p)) + patched := 0 + for i, status := range p { + rows[i] = PatchRow(status) + if status.Patched { + patched++ + } + } + return api.Text{}.Append("Patches", "font-bold"). + Append(fmt.Sprintf(" %s · %d patched", plural(len(p), "class"), patched), "text-muted"). + NewLine().Add(api.NewTableFrom(rows)) +} + +func lease(expiresAt int64) api.Text { + if expiresAt <= 0 { + return api.Text{}.Append("no restore lease", "text-yellow-600") + } + return api.Text{}.Append("lease until "+leaseTime(expiresAt), "text-muted") +} + +func leaseTime(expiresAt int64) string { + return time.UnixMilli(expiresAt).UTC().Format(time.RFC3339) +} + +func shortDigest(digest string) string { + algorithm, sum, found := strings.Cut(digest, ":") + if !found || len(sum) <= 8 { + return digest + } + return algorithm + ":" + sum[:8] +} + +func classpath(entries int) string { + if entries == 1 { + return "1 classpath entry" + } + return fmt.Sprintf("%d classpath entries", entries) +} diff --git a/pkg/tracing/jvm/classes/source.go b/pkg/tracing/jvm/classes/source.go new file mode 100644 index 0000000..b724f3b --- /dev/null +++ b/pkg/tracing/jvm/classes/source.go @@ -0,0 +1,93 @@ +package classes + +import ( + "bytes" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "strings" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" +) + +// Source is a class decompiled from the bytes it is running. SourceLineNumbers +// gives the original line behind each decompiled row (0 where none maps); +// LineRows is the reverse, original line to its first decompiled row. +type Source struct { + Class string `json:"class"` + Loader string `json:"loader"` + Source string `json:"source"` + Methods []SourceMethod `json:"methods"` + SourceLineNumbers []int `json:"sourceLineNumbers"` + LineRows map[int]int `json:"lineRows"` + Digest string `json:"digest"` + Decompiler string `json:"decompiler"` + DurationMs int64 `json:"durationMs"` + Cached bool `json:"cached"` +} + +// SourceMethod is where one method's body sits in the decompiled source, 1-based. +type SourceMethod struct { + Name string `json:"name"` + Descriptor string `json:"descriptor"` + Line int `json:"line"` + EndLine int `json:"endLine"` +} + +func DecodeSource(raw json.RawMessage) (Source, error) { + var source Source + if err := decodeObject(SourceOp, raw, &source, "class", "loader", "source", "decompiler"); err != nil { + return source, err + } + if source.Source == "" { + return source, fmt.Errorf("%s returned no source for %s", SourceOp, source.Class) + } + return source, nil +} + +func (s Source) Pretty() api.Text { + detail := fmt.Sprintf(" · loader %s · %s · %dms · %s", s.Loader, s.Decompiler, s.DurationMs, plural(len(s.Methods), "method")) + if s.Cached { + detail += " · cached" + } + return api.Text{}.Append(s.Class, "font-bold").Append(detail, "text-muted"). + NewLine().Add(clicky.CodeBlock("java", s.Source)) +} + +// Bytecode is the class file a class is running; it travels as base64. +type Bytecode struct { + Bytecode []byte `json:"bytecode"` +} + +var classMagic = []byte{0xca, 0xfe, 0xba, 0xbe} + +func DecodeBytecode(raw json.RawMessage) (Bytecode, error) { + var bytecode Bytecode + if err := decodeObject(BytecodeOp, raw, &bytecode, "bytecode"); err != nil { + return bytecode, err + } + if len(bytecode.Bytecode) < 8 || !bytes.HasPrefix(bytecode.Bytecode, classMagic) { + return bytecode, fmt.Errorf("%s returned %d bytes that are not a class file", BytecodeOp, len(bytecode.Bytecode)) + } + return bytecode, nil +} + +func (b Bytecode) Pretty() api.Text { + minor := binary.BigEndian.Uint16(b.Bytecode[4:6]) + major := binary.BigEndian.Uint16(b.Bytecode[6:8]) + return api.Text{}.Append("Bytecode", "font-bold"). + Append(fmt.Sprintf(" %s · class file %d.%d (Java %d)", api.HumanizeBytes(int64(len(b.Bytecode))).String(), major, minor, int(major)-44), "text-muted"). + NewLine().Add(clicky.CodeBlock("hexdump", hex.Dump(b.Bytecode))) +} + +func plural(n int, noun string) string { + if n == 1 { + return fmt.Sprintf("1 %s", noun) + } + if strings.HasSuffix(noun, "s") { + return fmt.Sprintf("%d %ses", n, noun) + } + return fmt.Sprintf("%d %ss", n, noun) +} diff --git a/pkg/tracing/jvm/heap.go b/pkg/tracing/jvm/heap.go new file mode 100644 index 0000000..17b22f7 --- /dev/null +++ b/pkg/tracing/jvm/heap.go @@ -0,0 +1,96 @@ +// Package jvm holds the typed results of Java agent diagnostic operations. +package jvm + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/flanksource/clicky/api" +) + +// HeapHistogramOp reads the heap's class histogram (GC.class_histogram). It +// changes nothing unless Live is set, which forces a full GC first. +const HeapHistogramOp = "jvm.heap.histogram" + +// HeapHistogramParams are the agent's jvm.heap.histogram parameters. +type HeapHistogramParams struct { + Top int `json:"top,omitempty"` + ClassPattern string `json:"classPattern,omitempty"` + Live bool `json:"live,omitempty"` +} + +// Validate refuses what the agent would refuse; zero Top is the agent's default of 100. +func (p HeapHistogramParams) Validate() error { + if p.Top < 0 { + return fmt.Errorf("top must be a positive number of rows, got %d", p.Top) + } + return nil +} + +// HeapHistogram is the heap's classes by retained bytes, largest first. The +// totals are the whole heap's; Matched is how many classes ClassPattern kept +// before the list was cut to Top. +type HeapHistogram struct { + CollectedAt time.Time `json:"collectedAt"` + Live bool `json:"live"` + TotalInstances int64 `json:"totalInstances"` + TotalBytes int64 `json:"totalBytes"` + Matched int `json:"matched"` + Rows []HeapClass `json:"rows"` +} + +// HeapClass is one histogram row. Rank is the JDK's position in the full, +// unfiltered histogram; Loader is the module or loader the JDK printed beside +// the class, when it printed one. +type HeapClass struct { + Rank int `json:"rank"` + ClassName string `json:"className"` + Instances int64 `json:"instances"` + Bytes int64 `json:"bytes"` + Loader string `json:"loader,omitempty"` +} + +// DecodeHeapHistogram decodes the agent's result, refusing anything that is +// not a histogram rather than reading it as an empty heap. +func DecodeHeapHistogram(raw json.RawMessage) (HeapHistogram, error) { + var histogram HeapHistogram + if err := json.Unmarshal(raw, &histogram); err != nil { + return histogram, fmt.Errorf("decode %s result: %w", HeapHistogramOp, err) + } + if histogram.CollectedAt.IsZero() || histogram.Rows == nil || histogram.TotalBytes <= 0 { + return histogram, fmt.Errorf("%s returned no histogram: %s", HeapHistogramOp, raw) + } + return histogram, nil +} + +func (HeapClass) Columns() []api.ColumnDef { + return []api.ColumnDef{ + api.Column("rank").Label("#").Build(), + api.Column("className").Label("Class").MaxWidth(100).Build(), + api.Column("instances").Label("Instances").Build(), + api.Column("bytes").Label("Bytes").Build(), + api.Column("loader").Label("Loader").Build(), + } +} + +func (c HeapClass) Row() map[string]any { + return map[string]any{ + "rank": c.Rank, + "className": c.ClassName, + "instances": api.HumanNumber(c.Instances), + "bytes": api.HumanizeBytes(c.Bytes), + "loader": c.Loader, + } +} + +// Pretty is the histogram's heading over its HeapClass table. +func (h HeapHistogram) Pretty() api.Text { + scope := "all objects" + if h.Live { + scope = "live objects" + } + return api.Text{}.Append("Heap histogram", "font-bold"). + Append(fmt.Sprintf(" %s · %s in %s instances · %d classes matched · %s", scope, api.HumanizeBytes(h.TotalBytes).String(), api.HumanNumber(h.TotalInstances).String(), h.Matched, h.CollectedAt.Format(time.RFC3339)), "text-muted"). + NewLine().Add(api.NewTableFrom(h.Rows)) +} diff --git a/pkg/tracing/jvm/heap_test.go b/pkg/tracing/jvm/heap_test.go new file mode 100644 index 0000000..9cbdaa1 --- /dev/null +++ b/pkg/tracing/jvm/heap_test.go @@ -0,0 +1,53 @@ +package jvm_test + +import ( + "encoding/json" + "testing" + "time" + + "github.com/flanksource/telltale/pkg/tracing/jvm" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestJVM(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "JVM operation results") } + +const agentHistogram = `{"collectedAt":"2026-11-14T10:00:00Z","live":false,"totalInstances":14340,"totalBytes":1751776,"matched":2, +"rows":[{"rank":3,"className":"example.Cache","instances":40,"bytes":1048576},{"rank":4,"className":"example.Order","instances":300,"bytes":7200,"loader":"app"}]}` + +var _ = Describe("heap histogram", func() { + It("decodes the agent's typed histogram", func() { + histogram, err := jvm.DecodeHeapHistogram(json.RawMessage(agentHistogram)) + Expect(err).NotTo(HaveOccurred()) + Expect(histogram).To(Equal(jvm.HeapHistogram{ + CollectedAt: time.Date(2026, 11, 14, 10, 0, 0, 0, time.UTC), TotalInstances: 14340, TotalBytes: 1751776, Matched: 2, + Rows: []jvm.HeapClass{ + {Rank: 3, ClassName: "example.Cache", Instances: 40, Bytes: 1048576}, + {Rank: 4, ClassName: "example.Order", Instances: 300, Bytes: 7200, Loader: "app"}, + }, + })) + }) + + It("refuses a result that is not a histogram rather than reporting an empty heap", func() { + for _, raw := range []string{`{}`, `{"rows":[],"totalBytes":1}`, `{"collectedAt":"2026-11-14T10:00:00Z","rows":null,"totalBytes":1}`, `[]`} { + _, err := jvm.DecodeHeapHistogram(json.RawMessage(raw)) + Expect(err).To(HaveOccurred(), raw) + } + }) + + It("tabulates each class with its rank, counts and loader", func() { + row := jvm.HeapClass{Rank: 4, ClassName: "example.Order", Instances: 300, Bytes: 7200, Loader: "app"} + var keys []string + for _, column := range row.Columns() { + keys = append(keys, column.Name) + } + Expect(keys).To(Equal([]string{"rank", "className", "instances", "bytes", "loader"})) + Expect(row.Row()).To(HaveKeyWithValue("className", "example.Order")) + Expect(row.Row()).To(HaveKeyWithValue("loader", "app")) + }) + + It("refuses a negative top before calling the agent", func() { + Expect(jvm.HeapHistogramParams{Top: -1}.Validate()).To(HaveOccurred()) + Expect(jvm.HeapHistogramParams{Top: 20, ClassPattern: "com.example.*", Live: true}.Validate()).To(Succeed()) + }) +}) diff --git a/pkg/tracing/jvm/jmx/attributes.go b/pkg/tracing/jvm/jmx/attributes.go new file mode 100644 index 0000000..e02b348 --- /dev/null +++ b/pkg/tracing/jvm/jmx/attributes.go @@ -0,0 +1,197 @@ +package jmx + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/flanksource/clicky/api" +) + +// MBeanAttributes is every readable attribute of one MBean, as jvm.jmx.read returns it +// when no attribute is named. +type MBeanAttributes struct { + ObjectName string `json:"objectName"` + Attributes []AttributeValue `json:"attributes"` +} + +type AttributeValue struct { + Name string `json:"name"` + Value Value `json:"value"` +} + +// DecodeRead decodes either shape jvm.jmx.read returns: an MBean's attributes when no +// attribute was named, otherwise the named attribute's value verbatim. +func DecodeRead(raw json.RawMessage) (any, error) { + value, err := decodeValue(ReadOp, raw) + if err != nil { + return nil, err + } + var shape map[string]json.RawMessage + if json.Unmarshal(raw, &shape) != nil || shape["objectName"] == nil || shape["attributes"] == nil { + return value, nil + } + var all MBeanAttributes + if err := json.Unmarshal(raw, &all); err != nil { + return nil, fmt.Errorf("decode %s result: %w", ReadOp, err) + } + if all.ObjectName == "" || all.Attributes == nil { + return nil, fmt.Errorf("%s returned no attributes: %s", ReadOp, raw) + } + return all, nil +} + +func (a MBeanAttributes) Pretty() api.Text { + text := api.Text{}.Append(a.ObjectName, "font-bold").Append(fmt.Sprintf(" %d attributes", len(a.Attributes)), "text-muted") + for _, attribute := range a.Attributes { + text = text.NewLine().Append(attribute.Name, "font-bold").Append(" = " + attribute.Value.String()) + } + return text +} + +// Invocation is what an MBean operation returned; a void operation returns +// null. Requested names the MBean and operation, which the agent does not echo. +type Invocation struct { + Result Value + ObjectName string + Operation string +} + +func (i Invocation) MarshalJSON() ([]byte, error) { return i.Result.MarshalJSON() } + +func DecodeInvoke(raw json.RawMessage) (Invocation, error) { + value, err := decodeValue(InvokeOp, raw) + return Invocation{Result: value}, err +} + +func (i Invocation) Requested(params json.RawMessage) (any, error) { + var request struct { + ObjectName string `json:"objectName"` + Operation string `json:"operation"` + } + if err := requiredParams(InvokeOp, params, &request, "objectName", "operation"); err != nil { + return nil, err + } + i.ObjectName, i.Operation = request.ObjectName, request.Operation + return i, nil +} + +func (i Invocation) Pretty() api.Text { + text := api.Text{}.Append("✓ invoked", "text-green-600") + if i.Operation != "" { + text = text.Append(" "+i.ObjectName, "text-muted").Append(" "+i.Operation, "font-bold") + } + if i.Result.IsNull() { + return text + } + return text.Append(" → " + i.Result.String()) +} + +// Applied is the agent's acknowledgement of a change it made. +type Applied struct { + Applied bool `json:"applied"` +} + +// AttributeWritten acknowledges jvm.jmx.write; Requested names the attribute +// and the value written, which the agent does not echo. +type AttributeWritten struct { + Applied + ObjectName string `json:"-"` + Attribute string `json:"-"` + Value json.RawMessage `json:"-"` +} + +// LevelSet acknowledges jvm.logging.set; Requested names the logger and its +// new level, null for one that inherits. +type LevelSet struct { + Applied + Name *string `json:"-"` + Level *string `json:"-"` +} + +func DecodeWrite(raw json.RawMessage) (AttributeWritten, error) { + applied, err := decodeApplied(WriteOp, raw) + return AttributeWritten{Applied: applied}, err +} + +func DecodeLevelSet(raw json.RawMessage) (LevelSet, error) { + applied, err := decodeApplied(LoggingSetOp, raw) + return LevelSet{Applied: applied}, err +} + +func (w AttributeWritten) Requested(params json.RawMessage) (any, error) { + var request struct { + ObjectName string `json:"objectName"` + Attribute string `json:"attribute"` + Value json.RawMessage `json:"value"` + } + if err := requiredParams(WriteOp, params, &request, "objectName", "attribute"); err != nil { + return nil, err + } + w.ObjectName, w.Attribute, w.Value = request.ObjectName, request.Attribute, request.Value + return w, nil +} + +func (s LevelSet) Requested(params json.RawMessage) (any, error) { + var request struct { + Name *string `json:"name"` + Level *string `json:"level"` + } + if err := requiredParams(LoggingSetOp, params, &request, "name"); err != nil { + return nil, err + } + s.Name, s.Level = request.Name, request.Level + return s, nil +} + +// requiredParams decodes a request's params, refusing one without the named keys. +func requiredParams(op string, params json.RawMessage, into any, keys ...string) error { + var present map[string]json.RawMessage + if err := json.Unmarshal(params, &present); err != nil { + return fmt.Errorf("%s params: %w", op, err) + } + for _, key := range keys { + if _, ok := present[key]; !ok { + return fmt.Errorf("%s params have no %s", op, key) + } + } + return json.Unmarshal(params, into) +} + +func decodeApplied(op string, raw json.RawMessage) (Applied, error) { + var applied Applied + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&applied); err != nil { + return applied, fmt.Errorf("decode %s result: %w", op, err) + } + if !applied.Applied { + return applied, fmt.Errorf("%s was not applied: %s", op, raw) + } + return applied, nil +} + +func (w AttributeWritten) Pretty() api.Text { + if w.Attribute == "" { + return api.Text{}.Append("✓ attribute written", "text-green-600") + } + value := string(w.Value) + if len(w.Value) == 0 { + value = "null" + } + return api.Text{}.Append("✓ ", "text-green-600").Append(w.ObjectName+" ", "text-muted").Append(w.Attribute, "font-bold").Append(" = " + value) +} + +func (s LevelSet) Pretty() api.Text { + if s.Name == nil { + return api.Text{}.Append("✓ logger level set", "text-green-600") + } + name, level := *s.Name, "inherit" + if name == "" { + name = "(root)" + } + if s.Level != nil { + level = *s.Level + } + return api.Text{}.Append("✓ logger ", "text-green-600").Append(name, "font-bold").Append(" → " + level) +} diff --git a/pkg/tracing/jvm/jmx/jmx.go b/pkg/tracing/jvm/jmx/jmx.go new file mode 100644 index 0000000..f68d2bc --- /dev/null +++ b/pkg/tracing/jvm/jmx/jmx.go @@ -0,0 +1,83 @@ +// Package jmx holds the typed results of the Java agent's JMX, notification and logging operations. +package jmx + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + + "github.com/flanksource/clicky/api" +) + +const ( + SearchOp = "jvm.jmx.search" + InfoOp = "jvm.jmx.info" + ReadOp = "jvm.jmx.read" + WriteOp = "jvm.jmx.write" + InvokeOp = "jvm.jmx.invoke" + NotificationsListOp = "jvm.notifications.list" + LoggingListOp = "jvm.logging.list" + LoggingSetOp = "jvm.logging.set" +) + +// Results decode this package's operation results into their typed models. +var Results = map[string]func(json.RawMessage) (any, error){ + SearchOp: func(raw json.RawMessage) (any, error) { return DecodeSearch(raw) }, + InfoOp: func(raw json.RawMessage) (any, error) { return DecodeInfo(raw) }, + ReadOp: func(raw json.RawMessage) (any, error) { return DecodeRead(raw) }, + WriteOp: func(raw json.RawMessage) (any, error) { return DecodeWrite(raw) }, + InvokeOp: func(raw json.RawMessage) (any, error) { return DecodeInvoke(raw) }, + NotificationsListOp: func(raw json.RawMessage) (any, error) { return DecodeSubscriptions(raw) }, + LoggingListOp: func(raw json.RawMessage) (any, error) { return DecodeLoggers(raw) }, + LoggingSetOp: func(raw json.RawMessage) (any, error) { return DecodeLevelSet(raw) }, +} + +// Value is a JMX value as the agent lowered it to JSON, kept verbatim. +type Value json.RawMessage + +func (v Value) MarshalJSON() ([]byte, error) { return json.RawMessage(v).MarshalJSON() } + +func (v *Value) UnmarshalJSON(raw []byte) error { + *v = append((*v)[:0], raw...) + return nil +} + +func (v Value) IsNull() bool { return string(bytes.TrimSpace(v)) == "null" } + +// String is the value on one line: strings unquoted, everything else as compact JSON. +func (v Value) String() string { + var text string + if json.Unmarshal(v, &text) == nil { + return text + } + var compact bytes.Buffer + if err := json.Compact(&compact, v); err != nil { + return string(v) + } + return compact.String() +} + +func (v Value) Pretty() api.Text { return api.Text{}.Append(v.String()) } + +func decodeValue(op string, raw json.RawMessage) (Value, error) { + if !json.Valid(raw) { + return nil, fmt.Errorf("%s returned invalid JSON: %q", op, raw) + } + return Value(bytes.TrimSpace(raw)), nil +} + +var primitiveTypes = map[byte]string{'Z': "boolean", 'B': "byte", 'C': "char", 'S': "short", 'I': "int", 'J': "long", 'F': "float", 'D': "double"} + +// javaType reads a JVM type name (including descriptors such as [J) as a simple Java type. +func javaType(name string) string { + base := strings.TrimLeft(name, "[") + dims := len(name) - len(base) + switch { + case dims > 0 && len(base) == 1 && primitiveTypes[base[0]] != "": + base = primitiveTypes[base[0]] + case dims > 0 && strings.HasPrefix(base, "L") && strings.HasSuffix(base, ";"): + base = base[1 : len(base)-1] + } + return base[strings.LastIndex(base, ".")+1:] + strings.Repeat("[]", dims) +} diff --git a/pkg/tracing/jvm/jmx/jmx_test.go b/pkg/tracing/jvm/jmx/jmx_test.go new file mode 100644 index 0000000..da50e03 --- /dev/null +++ b/pkg/tracing/jvm/jmx/jmx_test.go @@ -0,0 +1,210 @@ +package jmx_test + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/flanksource/clicky/api" + "github.com/flanksource/telltale/pkg/tracing/jvm/jmx" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestJMX(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "JMX, notification and logging results") +} + +const ( + agentSearch = `["java.lang:type=Runtime","java.lang:name=G1 Old Gen,type=MemoryPool","java.lang:type=Memory"]` + + agentMemoryInfo = `{"objectName":"java.lang:type=Memory","className":"sun.management.MemoryImpl","description":"Information on the management interface of the MBean", +"attributes":[{"name":"HeapMemoryUsage","description":"HeapMemoryUsage","type":"javax.management.openmbean.CompositeData","readable":true,"writable":false,"is":false},{"name":"Verbose","description":"Verbose","type":"boolean","readable":true,"writable":true,"is":true}], +"operations":[{"name":"gc","description":"gc","returnType":"void","impact":3,"signature":[]}], +"notifications":[{"name":"javax.management.Notification","description":"Memory Notification","types":["java.management.memory.threshold.exceeded","java.management.memory.collection.threshold.exceeded"]}]}` + + agentThreadingInfo = `{"objectName":"java.lang:type=Threading","className":"com.sun.management.internal.HotSpotThreadImpl","description":"Information on the management interface of the MBean", +"attributes":[{"name":"ThreadCount","description":"ThreadCount","type":"int","readable":true,"writable":false,"is":false},{"name":"ThreadCpuTimeEnabled","description":"ThreadCpuTimeEnabled","type":"boolean","readable":true,"writable":true,"is":true},{"name":"AllThreadIds","description":"AllThreadIds","type":"[J","readable":true,"writable":false,"is":false}], +"operations":[{"name":"getThreadCpuTime","description":"getThreadCpuTime","returnType":"[J","impact":3,"signature":[{"name":"p0","description":"p0","type":"[J"}]},{"name":"getThreadInfo","description":"getThreadInfo","returnType":"[Ljavax.management.openmbean.CompositeData;","impact":3,"signature":[{"name":"p0","description":"p0","type":"[J"},{"name":"p1","description":"p1","type":"int"}]},{"name":"resetPeakThreadCount","description":"resetPeakThreadCount","returnType":"void","impact":3,"signature":[]}], +"notifications":[]}` + + agentReadAll = `{"objectName":"java.lang:type=Memory","attributes":[{"name":"ObjectPendingFinalizationCount","value":0},{"name":"HeapMemoryUsage","value":{"committed":50331648,"init":536870912,"max":8589934592,"used":13813496}},{"name":"NonHeapMemoryUsage","value":{"committed":40173568,"init":7667712,"max":-1,"used":37357104}},{"name":"Verbose","value":false},{"name":"ObjectName","value":"java.lang:type=Memory"}]}` + + agentReadOne = `{"committed":50331648,"init":536870912,"max":8589934592,"used":13813496}` + + agentSubscriptions = `[{"id":"6f1c2a9e-4b7d-4c1e-9a53-0d2b8e7f1a40","mbean":"java.lang:type=GarbageCollector,*","filters":["com.sun.management.gc.notification"],"capacity":500,"queued":3,"dropped":0,"active":true}, +{"id":"0b9e5d12-77aa-4f0c-8c11-5e6f7a8b9c0d","mbean":"java.lang:type=Memory","filters":null,"capacity":50,"queued":0,"dropped":12,"active":false}]` + + agentLoggers = `[{"name":"","level":"INFO","parent":null},{"name":"global","level":null,"parent":""},{"name":"org.benf.cfr.reader.entities.constantpool.ConstantPool","level":"WARNING","parent":""}]` +) + +func decode(op, raw string) any { + result, err := jmx.Results[op](json.RawMessage(raw)) + Expect(err).NotTo(HaveOccurred(), op) + return result +} + +func pretty(result any) string { + return result.(interface{ Pretty() api.Text }).Pretty().String() +} + +func ptr(s string) *string { return &s } + +var _ = Describe("jmx search", func() { + It("lists every matched MBean in name order under its count", func() { + Expect(pretty(decode(jmx.SearchOp, agentSearch))).To(Equal(strings.Join([]string{ + "MBeans 3 matched", + "java.lang:name=G1 Old Gen,type=MemoryPool", + "java.lang:type=Memory", + "java.lang:type=Runtime", + }, "\n"))) + }) + + It("says so in one line when nothing matched", func() { + Expect(pretty(decode(jmx.SearchOp, `[]`))).To(Equal("no MBeans matched")) + }) +}) + +var _ = Describe("jmx info", func() { + It("decodes the MBean's attributes, operations and notifications", func() { + Expect(decode(jmx.InfoOp, agentMemoryInfo)).To(Equal(jmx.MBeanInfo{ + ObjectName: "java.lang:type=Memory", ClassName: "sun.management.MemoryImpl", Description: "Information on the management interface of the MBean", + Attributes: []jmx.MBeanAttribute{ + {Name: "HeapMemoryUsage", Description: "HeapMemoryUsage", Type: "javax.management.openmbean.CompositeData", Readable: true}, + {Name: "Verbose", Description: "Verbose", Type: "boolean", Readable: true, Writable: true, Is: true}, + }, + Operations: []jmx.MBeanOperation{{Name: "gc", Description: "gc", ReturnType: "void", Impact: 3, Signature: []jmx.MBeanParameter{}}}, + Notifications: []jmx.MBeanNotification{{Name: "javax.management.Notification", Description: "Memory Notification", + Types: []string{"java.management.memory.threshold.exceeded", "java.management.memory.collection.threshold.exceeded"}}}, + })) + }) + + It("renders operations as signatures with readable Java types", func() { + Expect(pretty(decode(jmx.InfoOp, agentThreadingInfo))).To(Equal(strings.Join([]string{ + "java.lang:type=Threading com.sun.management.internal.HotSpotThreadImpl · Information on the management interface of the MBean", + "╭────────────────────┬───────┬──────────╮", + "│Attribute │Type │Access │", + "├────────────────────┼───────┼──────────┤", + "│ThreadCount │int │read-only │", + "│ThreadCpuTimeEnabled│boolean│read-write│", + "│AllThreadIds │long[] │read-only │", + "╰────────────────────┴───────┴──────────╯", + "Operations", + "getThreadCpuTime(long[]) → long[]", + "getThreadInfo(long[], int) → CompositeData[]", + "resetPeakThreadCount() → void", + }, "\n"))) + }) + + It("tabulates attributes with their simple type and access", func() { + row := jmx.MBeanAttribute{Name: "AllThreadIds", Description: "AllThreadIds", Type: "[J", Readable: true} + Expect(row.Row()).To(Equal(map[string]any{"name": "AllThreadIds", "type": "long[]", "access": "read-only", "description": ""})) + }) +}) + +var _ = Describe("jmx read", func() { + It("prints every attribute as one compact line", func() { + Expect(pretty(decode(jmx.ReadOp, agentReadAll))).To(Equal(strings.Join([]string{ + "java.lang:type=Memory 5 attributes", + "ObjectPendingFinalizationCount = 0", + `HeapMemoryUsage = {"committed":50331648,"init":536870912,"max":8589934592,"used":13813496}`, + `NonHeapMemoryUsage = {"committed":40173568,"init":7667712,"max":-1,"used":37357104}`, + "Verbose = false", + "ObjectName = java.lang:type=Memory", + }, "\n"))) + }) + + It("prints a single attribute's value as one line and keeps it verbatim as JSON", func() { + result := decode(jmx.ReadOp, agentReadOne) + Expect(pretty(result)).To(Equal(agentReadOne)) + Expect(json.Marshal(result)).To(MatchJSON(agentReadOne)) + }) +}) + +var _ = Describe("mutation acknowledgements", func() { + It("acknowledges a write, an invocation and a level change in one line", func() { + Expect(pretty(decode(jmx.WriteOp, `{"applied":true}`))).To(Equal("✓ attribute written")) + Expect(pretty(decode(jmx.LoggingSetOp, `{"applied":true}`))).To(Equal("✓ logger level set")) + Expect(pretty(decode(jmx.InvokeOp, `null`))).To(Equal("✓ invoked")) + Expect(pretty(decode(jmx.InvokeOp, `-1`))).To(Equal("✓ invoked → -1")) + }) + + It("names what the request changed, since the agent echoes only that it applied it", func() { + requested := func(op, params, raw string) string { + result, err := decode(op, raw).(interface { + Requested(json.RawMessage) (any, error) + }).Requested(json.RawMessage(params)) + Expect(err).NotTo(HaveOccurred()) + return pretty(result) + } + Expect(requested(jmx.WriteOp, `{"objectName":"java.lang:type=Memory","attribute":"Verbose","value":false}`, `{"applied":true}`)).To(Equal("✓ java.lang:type=Memory Verbose = false")) + Expect(requested(jmx.LoggingSetOp, `{"name":"global","level":"FINE"}`, `{"applied":true}`)).To(Equal("✓ logger global → FINE")) + Expect(requested(jmx.LoggingSetOp, `{"name":"","level":null}`, `{"applied":true}`)).To(Equal("✓ logger (root) → inherit")) + Expect(requested(jmx.InvokeOp, `{"objectName":"java.lang:type=Memory","operation":"gc"}`, `null`)).To(Equal("✓ invoked java.lang:type=Memory gc")) + Expect(requested(jmx.InvokeOp, `{"objectName":"java.lang:type=Threading","operation":"getThreadCpuTime","args":[[1]]}`, `-1`)).To(Equal("✓ invoked java.lang:type=Threading getThreadCpuTime → -1")) + _, err := decode(jmx.WriteOp, `{"applied":true}`).(interface { + Requested(json.RawMessage) (any, error) + }).Requested(json.RawMessage(`{"attribute":"Verbose"}`)) + Expect(err).To(MatchError(ContainSubstring("objectName"))) + }) +}) + +var _ = Describe("notification subscriptions", func() { + It("decodes every subscriber the agent buffers for", func() { + Expect(decode(jmx.NotificationsListOp, agentSubscriptions)).To(Equal(jmx.Subscriptions{ + {ID: "6f1c2a9e-4b7d-4c1e-9a53-0d2b8e7f1a40", MBean: "java.lang:type=GarbageCollector,*", Filters: []string{"com.sun.management.gc.notification"}, Capacity: 500, Queued: 3, Active: true}, + {ID: "0b9e5d12-77aa-4f0c-8c11-5e6f7a8b9c0d", MBean: "java.lang:type=Memory", Capacity: 50, Dropped: 12}, + })) + }) + + It("heads the table with queued and dropped totals", func() { + Expect(strings.SplitN(pretty(decode(jmx.NotificationsListOp, agentSubscriptions)), "\n", 2)[0]). + To(Equal("Notification subscriptions 2 · 3 queued · 12 dropped")) + Expect(pretty(decode(jmx.NotificationsListOp, `[]`))).To(Equal("no notification subscriptions")) + }) +}) + +var _ = Describe("logging list", func() { + It("decodes loggers keeping the root's absent parent distinct from the root as parent", func() { + Expect(decode(jmx.LoggingListOp, agentLoggers)).To(Equal(jmx.Loggers{ + {Name: "", Level: ptr("INFO")}, + {Name: "global", Parent: ptr("")}, + {Name: "org.benf.cfr.reader.entities.constantpool.ConstantPool", Level: ptr("WARNING"), Parent: ptr("")}, + })) + }) + + It("resolves each logger's effective level through its parents", func() { + loggers := decode(jmx.LoggingListOp, agentLoggers).(jmx.Loggers) + Expect(loggers.Rows()).To(Equal([]jmx.LoggerRow{ + {Logger: "(root)", Level: "INFO", Effective: "INFO"}, + {Logger: "global", Effective: "INFO"}, + {Logger: "org.benf.cfr.reader.entities.constantpool.ConstantPool", Level: "WARNING", Effective: "WARNING"}, + })) + Expect(strings.SplitN(pretty(loggers), "\n", 2)[0]).To(Equal("Loggers 3 · 2 with explicit levels")) + }) +}) + +var _ = Describe("strict decoding", func() { + DescribeTable("refuses what is not the operation's result", + func(op, raw string) { + _, err := jmx.Results[op](json.RawMessage(raw)) + Expect(err).To(HaveOccurred()) + }, + Entry("search object", jmx.SearchOp, `{}`), + Entry("search null", jmx.SearchOp, `null`), + Entry("info without class", jmx.InfoOp, `{"objectName":"java.lang:type=Memory"}`), + Entry("info array", jmx.InfoOp, `[]`), + Entry("read invalid json", jmx.ReadOp, `{`), + Entry("read attributes without object name", jmx.ReadOp, `{"objectName":"","attributes":[]}`), + Entry("write not applied", jmx.WriteOp, `{"applied":false}`), + Entry("write empty", jmx.WriteOp, `{}`), + Entry("logging set null", jmx.LoggingSetOp, `null`), + Entry("invoke empty", jmx.InvokeOp, ``), + Entry("subscriptions null", jmx.NotificationsListOp, `null`), + Entry("subscription without id", jmx.NotificationsListOp, `[{"mbean":"java.lang:type=Memory"}]`), + Entry("logger without name", jmx.LoggingListOp, `[{"level":"INFO","parent":null}]`), + Entry("loggers without root", jmx.LoggingListOp, `[{"name":"global","level":null,"parent":""}]`), + Entry("loggers object", jmx.LoggingListOp, `{}`), + ) +}) diff --git a/pkg/tracing/jvm/jmx/logging.go b/pkg/tracing/jvm/jmx/logging.go new file mode 100644 index 0000000..9e014e8 --- /dev/null +++ b/pkg/tracing/jvm/jmx/logging.go @@ -0,0 +1,126 @@ +package jmx + +import ( + "encoding/json" + "fmt" + + "github.com/flanksource/clicky/api" +) + +// Loggers are the java.util.logging loggers in name order. The root logger is named "" +// and has no Parent; a nil Level inherits the parent's. +type Loggers []Logger + +type Logger struct { + Name string `json:"name"` + Level *string `json:"level"` + Parent *string `json:"parent"` +} + +func (l *Logger) UnmarshalJSON(raw []byte) error { + var wire struct { + Name *string `json:"name"` + Level *string `json:"level"` + Parent *string `json:"parent"` + } + if err := json.Unmarshal(raw, &wire); err != nil { + return err + } + if wire.Name == nil { + return fmt.Errorf("logger without a name: %s", raw) + } + *l = Logger{Name: *wire.Name, Level: wire.Level, Parent: wire.Parent} + return nil +} + +func DecodeLoggers(raw json.RawMessage) (Loggers, error) { + var loggers Loggers + if err := json.Unmarshal(raw, &loggers); err != nil { + return nil, fmt.Errorf("decode %s result: %w", LoggingListOp, err) + } + for _, logger := range loggers { + if logger.Name == "" && logger.Parent == nil { + return loggers, nil + } + } + return nil, fmt.Errorf("%s returned no root logger: %s", LoggingListOp, raw) +} + +// LoggerRow is one logger with the level it actually logs at; Level is empty when inherited. +type LoggerRow struct { + Logger string `json:"logger"` + Level string `json:"level"` + Effective string `json:"effective"` +} + +// Rows resolve each logger's effective level through its listed parents. +func (l Loggers) Rows() []LoggerRow { + byName := make(map[string]Logger, len(l)) + for _, logger := range l { + byName[logger.Name] = logger + } + rows := make([]LoggerRow, len(l)) + for i, logger := range l { + rows[i] = LoggerRow{Logger: logger.Name, Level: deref(logger.Level), Effective: effectiveLevel(byName, logger)} + if logger.Name == "" { + rows[i].Logger = "(root)" + } + } + return rows +} + +func effectiveLevel(byName map[string]Logger, logger Logger) string { + for range len(byName) { + if logger.Level != nil { + return *logger.Level + } + if logger.Parent == nil { + return "" + } + parent, listed := byName[*logger.Parent] + if !listed { + return "" + } + logger = parent + } + return "" +} + +func deref(s *string) string { + if s == nil { + return "" + } + return *s +} + +func (LoggerRow) Columns() []api.ColumnDef { + return []api.ColumnDef{ + api.Column("logger").Label("Logger").MaxWidth(100).Build(), + api.Column("level").Label("Level").Build(), + api.Column("effective").Label("Effective").Build(), + } +} + +func (r LoggerRow) Row() map[string]any { + level := api.Text{}.Append(r.Level) + if r.Level == "" { + level = api.Text{}.Append("inherit", "text-muted") + } + effective := api.Text{}.Append(r.Effective) + if r.Effective == "" { + effective = api.Text{}.Append("unknown", "text-yellow-600") + } + return map[string]any{"logger": r.Logger, "level": level, "effective": effective} +} + +func (l Loggers) Pretty() api.Text { + explicit := 0 + for _, logger := range l { + if logger.Level != nil { + explicit++ + } + } + return api.Text{}.Append("Loggers", "font-bold"). + Append(fmt.Sprintf(" %d · %d with explicit levels", len(l), explicit), "text-muted"). + NewLine().Add(api.NewTableFrom(l.Rows())) +} diff --git a/pkg/tracing/jvm/jmx/mbeans.go b/pkg/tracing/jvm/jmx/mbeans.go new file mode 100644 index 0000000..5bf9410 --- /dev/null +++ b/pkg/tracing/jvm/jmx/mbeans.go @@ -0,0 +1,138 @@ +package jmx + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + + "github.com/flanksource/clicky/api" +) + +// MBeanNames are the canonical object names jvm.jmx.search matched, in the agent's order. +type MBeanNames []string + +func DecodeSearch(raw json.RawMessage) (MBeanNames, error) { + var names MBeanNames + if err := json.Unmarshal(raw, &names); err != nil { + return nil, fmt.Errorf("decode %s result: %w", SearchOp, err) + } + if names == nil { + return nil, fmt.Errorf("%s returned no MBean list: %s", SearchOp, raw) + } + return names, nil +} + +func (n MBeanNames) Pretty() api.Text { + if len(n) == 0 { + return api.Text{}.Append("no MBeans matched", "text-muted") + } + text := api.Text{}.Append("MBeans", "font-bold").Append(fmt.Sprintf(" %d matched", len(n)), "text-muted") + for _, name := range slices.Sorted(slices.Values(n)) { + domain, properties, _ := strings.Cut(name, ":") + text = text.NewLine().Append(domain, "font-bold").Append(":" + properties) + } + return text +} + +// MBeanInfo is an MBean's management interface as jvm.jmx.info lowers it. +type MBeanInfo struct { + ObjectName string `json:"objectName"` + ClassName string `json:"className"` + Description string `json:"description"` + Attributes []MBeanAttribute `json:"attributes"` + Operations []MBeanOperation `json:"operations"` + Notifications []MBeanNotification `json:"notifications"` +} + +type MBeanAttribute struct { + Name string `json:"name"` + Description string `json:"description"` + Type string `json:"type"` + Readable bool `json:"readable"` + Writable bool `json:"writable"` + Is bool `json:"is"` +} + +// MBeanOperation's Impact is the JMX constant: 0 INFO, 1 ACTION, 2 ACTION_INFO, 3 UNKNOWN. +type MBeanOperation struct { + Name string `json:"name"` + Description string `json:"description"` + ReturnType string `json:"returnType"` + Impact int `json:"impact"` + Signature []MBeanParameter `json:"signature"` +} + +type MBeanParameter struct { + Name string `json:"name"` + Description string `json:"description"` + Type string `json:"type"` +} + +type MBeanNotification struct { + Name string `json:"name"` + Description string `json:"description"` + Types []string `json:"types"` +} + +func DecodeInfo(raw json.RawMessage) (MBeanInfo, error) { + var info MBeanInfo + if err := json.Unmarshal(raw, &info); err != nil { + return info, fmt.Errorf("decode %s result: %w", InfoOp, err) + } + if info.ObjectName == "" || info.ClassName == "" || info.Attributes == nil || info.Operations == nil || info.Notifications == nil { + return info, fmt.Errorf("%s returned no MBean info: %s", InfoOp, raw) + } + return info, nil +} + +func (MBeanAttribute) Columns() []api.ColumnDef { + return []api.ColumnDef{ + api.Column("name").Label("Attribute").Build(), + api.Column("type").Label("Type").Build(), + api.Column("access").Label("Access").Build(), + api.Column("description").Label("Description").MaxWidth(80).Build(), + } +} + +// Row leaves out a description that only repeats the attribute's name, as the platform MBeans' do. +func (a MBeanAttribute) Row() map[string]any { + access := map[[2]bool]string{{true, true}: "read-write", {true, false}: "read-only", {false, true}: "write-only", {false, false}: "none"} + description := a.Description + if description == a.Name { + description = "" + } + return map[string]any{"name": a.Name, "type": javaType(a.Type), "access": access[[2]bool{a.Readable, a.Writable}], "description": description} +} + +func (o MBeanOperation) signature() string { + types := make([]string, len(o.Signature)) + for i, parameter := range o.Signature { + types[i] = javaType(parameter.Type) + } + return fmt.Sprintf("%s(%s) → %s", o.Name, strings.Join(types, ", "), javaType(o.ReturnType)) +} + +func (i MBeanInfo) Pretty() api.Text { + text := api.Text{}.Append(i.ObjectName, "font-bold").Append(" "+strings.Join(nonEmpty(i.ClassName, i.Description), " · "), "text-muted") + if len(i.Attributes) > 0 { + text = text.NewLine().Add(api.NewTableFrom(i.Attributes)) + } + if len(i.Operations) > 0 { + text = text.NewLine().Append("Operations", "font-bold") + for _, operation := range i.Operations { + text = text.NewLine().Append(operation.signature()) + } + } + if len(i.Notifications) > 0 { + text = text.NewLine().Append("Notifications", "font-bold") + for _, notification := range i.Notifications { + text = text.NewLine().Append(notification.Description).Append(" "+strings.Join(notification.Types, ", "), "text-muted") + } + } + return text +} + +func nonEmpty(values ...string) []string { + return slices.DeleteFunc(values, func(value string) bool { return value == "" }) +} diff --git a/pkg/tracing/jvm/jmx/notifications.go b/pkg/tracing/jvm/jmx/notifications.go new file mode 100644 index 0000000..207174a --- /dev/null +++ b/pkg/tracing/jvm/jmx/notifications.go @@ -0,0 +1,84 @@ +package jmx + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/flanksource/clicky/api" +) + +// Subscriptions are the notification subscribers the agent buffers for. MBean is the +// object name or pattern subscribed to; nil Filters accept every notification type. +type Subscriptions []Subscription + +type Subscription struct { + ID string `json:"id"` + MBean string `json:"mbean"` + Filters []string `json:"filters"` + Capacity int `json:"capacity"` + Queued int `json:"queued"` + Dropped int64 `json:"dropped"` + Active bool `json:"active"` +} + +func DecodeSubscriptions(raw json.RawMessage) (Subscriptions, error) { + var subscriptions Subscriptions + if err := json.Unmarshal(raw, &subscriptions); err != nil { + return nil, fmt.Errorf("decode %s result: %w", NotificationsListOp, err) + } + if subscriptions == nil { + return nil, fmt.Errorf("%s returned no subscription list: %s", NotificationsListOp, raw) + } + for _, subscription := range subscriptions { + if subscription.ID == "" || subscription.MBean == "" { + return nil, fmt.Errorf("%s returned a subscription without an id or MBean: %s", NotificationsListOp, raw) + } + } + return subscriptions, nil +} + +func (Subscription) Columns() []api.ColumnDef { + return []api.ColumnDef{ + api.Column("id").Label("Subscription").Build(), + api.Column("mbean").Label("MBean").MaxWidth(80).Build(), + api.Column("filters").Label("Types").MaxWidth(60).Build(), + api.Column("queued").Label("Queued").Build(), + api.Column("dropped").Label("Dropped").Build(), + api.Column("capacity").Label("Capacity").Build(), + api.Column("state").Label("State").Build(), + } +} + +func (s Subscription) Row() map[string]any { + filters := "all" + if s.Filters != nil { + filters = strings.Join(s.Filters, ", ") + } + dropped := api.Text{}.Append(api.HumanNumber(s.Dropped).String()) + if s.Dropped > 0 { + dropped = api.Text{}.Append(api.HumanNumber(s.Dropped).String(), "text-yellow-600") + } + state := api.Text{}.Append("frozen", "text-muted") + if s.Active { + state = api.Text{}.Append("active", "text-green-600") + } + return map[string]any{ + "id": s.ID, "mbean": s.MBean, "filters": filters, "queued": api.HumanNumber(int64(s.Queued)), + "dropped": dropped, "capacity": api.HumanNumber(int64(s.Capacity)), "state": state, + } +} + +func (s Subscriptions) Pretty() api.Text { + if len(s) == 0 { + return api.Text{}.Append("no notification subscriptions", "text-muted") + } + var queued, dropped int64 + for _, subscription := range s { + queued += int64(subscription.Queued) + dropped += subscription.Dropped + } + return api.Text{}.Append("Notification subscriptions", "font-bold"). + Append(fmt.Sprintf(" %d · %s queued · %s dropped", len(s), api.HumanNumber(queued).String(), api.HumanNumber(dropped).String()), "text-muted"). + NewLine().Add(api.NewTableFrom(s)) +} diff --git a/pkg/tracing/jvm/jvmruntime/dump.go b/pkg/tracing/jvm/jvmruntime/dump.go new file mode 100644 index 0000000..063ecf6 --- /dev/null +++ b/pkg/tracing/jvm/jvmruntime/dump.go @@ -0,0 +1,170 @@ +package jvmruntime + +import ( + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/flanksource/clicky/api" +) + +// ThreadDumpOp dumps every thread's stack as jstack text, then any deadlocks. +const ThreadDumpOp = "jvm.threads.dump" + +// ThreadDump is the agent's jstack text read back into threads. Deadlocks is +// the jstack deadlock report's lines, verbatim, when there is one. +type ThreadDump struct { + Threads []DumpedThread `json:"threads"` + Deadlocks []string `json:"deadlocks,omitempty"` +} + +// DumpedThread is one jstack block. Stack is its frame ("at ...") and lock +// ("- ...") lines in order; Priority is 0 when the thread ended mid-dump. +type DumpedThread struct { + Name string `json:"name"` + ID int64 `json:"id"` + Daemon bool `json:"daemon"` + Priority int `json:"priority,omitempty"` + State string `json:"state"` + Stack []string `json:"stack"` + Synchronizers []string `json:"synchronizers,omitempty"` +} + +var ( + threadHeader = regexp.MustCompile(`^"(.*)" #(\d+)( daemon)?(?: prio=(\d+))?$`) + deadlockHeader = regexp.MustCompile(`^Found \d+ deadlocks?:$`) +) + +const ( + stateLine = " java.lang.Thread.State: " + synchronizerLine = " Locked ownable synchronizers:" +) + +// DecodeThreadDump decodes the agent's dump, a JSON string of jstack text, +// refusing any line it does not recognise rather than dropping it. +func DecodeThreadDump(raw json.RawMessage) (ThreadDump, error) { + var text string + if err := json.Unmarshal(raw, &text); err != nil { + return ThreadDump{}, fmt.Errorf("decode %s result: %w", ThreadDumpOp, err) + } + return parseThreadDump(ThreadDumpOp, text) +} + +func parseThreadDump(op, text string) (ThreadDump, error) { + var dump ThreadDump + synchronizers := false + for number, line := range strings.Split(text, "\n") { + if dump.Deadlocks != nil || deadlockHeader.MatchString(line) { + if line != "" { + dump.Deadlocks = append(dump.Deadlocks, line) + } + continue + } + if match := threadHeader.FindStringSubmatch(line); match != nil { + thread, err := newDumpedThread(match) + if err != nil { + return dump, fmt.Errorf("%s line %d: %w", op, number+1, err) + } + dump.Threads = append(dump.Threads, thread) + synchronizers = false + continue + } + if line == "" { + continue + } + if len(dump.Threads) == 0 { + return dump, fmt.Errorf("%s line %d is outside any thread: %q", op, number+1, line) + } + var err error + if synchronizers, err = dump.Threads[len(dump.Threads)-1].parseLine(line, synchronizers); err != nil { + return dump, fmt.Errorf("%s line %d: %w", op, number+1, err) + } + } + for _, thread := range dump.Threads { + if thread.State == "" { + return dump, fmt.Errorf("%s thread %q #%d has no state", op, thread.Name, thread.ID) + } + } + if len(dump.Threads) == 0 { + return dump, fmt.Errorf("%s returned no threads: %q", op, text) + } + return dump, nil +} + +func newDumpedThread(header []string) (DumpedThread, error) { + thread := DumpedThread{Name: header[1], Daemon: header[3] != "", Stack: []string{}} + var err error + if thread.ID, err = strconv.ParseInt(header[2], 10, 64); err != nil { + return thread, fmt.Errorf("thread %q id: %w", header[1], err) + } + if header[4] != "" { + if thread.Priority, err = strconv.Atoi(header[4]); err != nil { + return thread, fmt.Errorf("thread %q priority: %w", header[1], err) + } + } + return thread, nil +} + +// parseLine files one line of a thread block, reporting whether the +// synchronizer list has started. +func (t *DumpedThread) parseLine(line string, synchronizers bool) (bool, error) { + switch { + case strings.HasPrefix(line, stateLine): + t.State = strings.TrimPrefix(line, stateLine) + case line == synchronizerLine: + return true, nil + case synchronizers && line == "\t- None": + case synchronizers && strings.HasPrefix(line, "\t- "): + t.Synchronizers = append(t.Synchronizers, strings.TrimPrefix(line, "\t- ")) + case !synchronizers && (strings.HasPrefix(line, "\tat ") || strings.HasPrefix(line, "\t- ")): + t.Stack = append(t.Stack, strings.TrimPrefix(line, "\t")) + default: + return synchronizers, fmt.Errorf("unexpected line in thread %q #%d: %q", t.Name, t.ID, line) + } + return synchronizers, nil +} + +// Pretty is a heading over each thread, then the deadlock report in red. +func (d ThreadDump) Pretty() api.Text { + text := api.Text{}.Append("Thread dump", "font-bold").Append(fmt.Sprintf(" %d threads", len(d.Threads)), "text-muted") + if len(d.Deadlocks) > 0 { + text = text.Append(" · DEADLOCK", "text-red-600") + } + for _, thread := range d.Threads { + text = text.NewLine().Add(thread.Pretty()) + } + for _, line := range d.Deadlocks { + text = text.NewLine().Append(line, "text-red-600") + } + return text +} + +// Pretty is the thread's header over its indented frames, locks muted. +func (t DumpedThread) Pretty() api.Text { + return t.render("") +} + +func (t DumpedThread) render(suffix string) api.Text { + facts := []string{fmt.Sprintf("#%d", t.ID)} + if t.Daemon { + facts = append(facts, "daemon") + } + if t.Priority > 0 { + facts = append(facts, fmt.Sprintf("prio %d", t.Priority)) + } + text := api.Text{}.Append(t.Name, "font-bold").Append(" "+strings.Join(facts, " · ")+" · ", "text-muted"). + Append(t.State, stateClass(t.State)).Append(suffix, "text-muted") + for _, line := range t.Stack { + class := "" + if strings.HasPrefix(line, "- ") { + class = "text-muted" + } + text = text.NewLine().Append(" "+line, class) + } + if len(t.Synchronizers) > 0 { + text = text.NewLine().Append(" locked synchronizers: "+strings.Join(t.Synchronizers, ", "), "text-muted") + } + return text +} diff --git a/pkg/tracing/jvm/jvmruntime/jvmruntime.go b/pkg/tracing/jvm/jvmruntime/jvmruntime.go new file mode 100644 index 0000000..cd467ea --- /dev/null +++ b/pkg/tracing/jvm/jvmruntime/jvmruntime.go @@ -0,0 +1,54 @@ +// Package jvmruntime holds the typed results of the Java agent's runtime +// operations: status, metrics, settings, resources, VM flags, and the thread +// sample, dump and stack. +package jvmruntime + +import ( + "encoding/json" + "fmt" + "time" +) + +// Results decode this package's operation results into their typed models. +var Results = map[string]func(json.RawMessage) (any, error){ + StatusOp: func(raw json.RawMessage) (any, error) { return DecodeStatus(raw) }, + MetricsOp: func(raw json.RawMessage) (any, error) { return DecodeMetrics(raw) }, + ThreadsOp: func(raw json.RawMessage) (any, error) { return DecodeThreads(raw) }, + ThreadDumpOp: func(raw json.RawMessage) (any, error) { return DecodeThreadDump(raw) }, + ThreadStackOp: func(raw json.RawMessage) (any, error) { return DecodeThreadStack(raw) }, + SettingsOp: func(raw json.RawMessage) (any, error) { return DecodeSettings(raw) }, + ResourcesOp: func(raw json.RawMessage) (any, error) { return DecodeResources(raw) }, + FlagsOp: func(raw json.RawMessage) (any, error) { return DecodeFlags(raw) }, +} + +// decodeObject decodes an agent result object, refusing one that is missing +// (or nulls) any of the keys the agent always sends for op. +func decodeObject[T any](op string, raw json.RawMessage, keys ...string) (T, error) { + var out T + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil { + return out, fmt.Errorf("decode %s result: %w", op, err) + } + for _, key := range keys { + if value, ok := fields[key]; !ok || string(value) == "null" { + return out, fmt.Errorf("%s result has no %q: %s", op, key, raw) + } + } + if err := json.Unmarshal(raw, &out); err != nil { + return out, fmt.Errorf("decode %s result: %w", op, err) + } + return out, nil +} + +// millis renders a millisecond count the way a reader scans it: 11ms, 1.4s, 4m20s. +func millis(ms int64) string { + duration := time.Duration(ms) * time.Millisecond + switch { + case duration < time.Second: + return fmt.Sprintf("%dms", ms) + case duration < time.Minute: + return duration.Round(100 * time.Millisecond).String() + default: + return duration.Round(time.Second).String() + } +} diff --git a/pkg/tracing/jvm/jvmruntime/jvmruntime_test.go b/pkg/tracing/jvm/jvmruntime/jvmruntime_test.go new file mode 100644 index 0000000..c18e3f2 --- /dev/null +++ b/pkg/tracing/jvm/jvmruntime/jvmruntime_test.go @@ -0,0 +1,143 @@ +package jvmruntime_test + +import ( + "encoding/json" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/flanksource/clicky/api" + "github.com/flanksource/telltale/pkg/tracing/jvm/jvmruntime" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestJVMRuntime(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "JVM runtime operation results") +} + +func lines(text interface{ String() string }) []string { + return strings.Split(text.String(), "\n") +} + +var _ = Describe("registry", func() { + It("decodes every runtime operation", func() { + Expect(jvmruntime.Results).To(HaveLen(8)) + for _, op := range []string{jvmruntime.StatusOp, jvmruntime.MetricsOp, jvmruntime.ThreadsOp, jvmruntime.ThreadDumpOp, jvmruntime.ThreadStackOp, jvmruntime.SettingsOp, jvmruntime.ResourcesOp, jvmruntime.FlagsOp} { + Expect(jvmruntime.Results).To(HaveKey(op)) + _, err := jvmruntime.Results[op](json.RawMessage(`{}`)) + Expect(err).To(HaveOccurred(), op) + } + }) + + It("decodes and renders each full capture from a live agent", func() { + for op := range jvmruntime.Results { + raw, err := os.ReadFile(filepath.Join("testdata", op+".json")) + Expect(err).NotTo(HaveOccurred()) + result, err := jvmruntime.Results[op](raw) + Expect(err).NotTo(HaveOccurred(), op) + Expect(result.(interface{ Pretty() api.Text }).Pretty().String()).NotTo(BeEmpty(), op) + } + }) +}) + +var _ = Describe("status", func() { + const live = `{"allowMutation":true,"instrumentation":true,"patches":[]}` + + It("decodes the agent's status and renders it as one line", func() { + status, err := jvmruntime.DecodeStatus(json.RawMessage(live)) + Expect(err).NotTo(HaveOccurred()) + Expect(status).To(Equal(jvmruntime.Status{AllowMutation: true, Instrumentation: true, Patches: []jvmruntime.Patch{}})) + Expect(status.Pretty().String()).To(Equal("mutations allowed · instrumentation on · 0 patches")) + }) + + It("lists active patches compactly with their lease", func() { + lease := time.Date(2026, 11, 16, 5, 0, 0, 0, time.UTC).UnixMilli() + status, err := jvmruntime.DecodeStatus(json.RawMessage(`{"allowMutation":false,"instrumentation":true,"patches":[ +{"class":"example.Cache","loader":"app","digest":"b","originalDigest":"a","patched":true,"leaseExpiresAt":` + strconv.FormatInt(lease, 10) + `}, +{"class":"example.Pinned","loader":"app","digest":"b","originalDigest":"a","patched":true,"leaseExpiresAt":0}, +{"class":"example.Gone","loader":"old","patched":false,"error":"no longer loaded: x"}]}`)) + Expect(err).NotTo(HaveOccurred()) + Expect(status.Patches[0]).To(Equal(jvmruntime.Patch{Class: "example.Cache", Loader: "app", Digest: "b", OriginalDigest: "a", Patched: true, LeaseExpiresAt: lease})) + Expect(status.Pretty().String()).To(Equal("mutations denied · instrumentation on · 3 patches: example.Cache lease until 2026-11-16T05:00:00Z, example.Pinned no lease, example.Gone no longer loaded: x")) + }) + + It("says instrumentation is off without inventing a patch count", func() { + status, err := jvmruntime.DecodeStatus(json.RawMessage(`{"allowMutation":true,"instrumentation":false}`)) + Expect(err).NotTo(HaveOccurred()) + Expect(status.Pretty().String()).To(Equal("mutations allowed · instrumentation off")) + }) + + It("refuses a result that is not a status", func() { + for _, raw := range []string{`{}`, `{"allowMutation":true}`, `{"allowMutation":true,"instrumentation":true}`, `[]`, `"text"`} { + _, err := jvmruntime.DecodeStatus(json.RawMessage(raw)) + Expect(err).To(HaveOccurred(), raw) + } + }) +}) + +var _ = Describe("metrics", func() { + const live = `{"capturedAt":"2026-11-16T04:53:48.543631Z","memory":{"heap":[{"name":"G1 Eden Space","type":"HEAP","used":20971520,"committed":29360128,"max":-1},{"name":"G1 Old Gen","type":"HEAP","used":1230584,"committed":4194304,"max":8589934592}],"nonHeap":[{"name":"Metaspace","type":"NON_HEAP","used":24222544,"committed":24444928,"max":-1}],"buffer":[{"name":"direct","type":"BUFFER","used":0,"committed":0,"max":-1}]},"garbageCollectors":[{"name":"G1 Young Generation","collectionCount":2,"collectionTime":11},{"name":"G1 Old Generation","collectionCount":0,"collectionTime":0}],"runtime":{"javaVersion":"25.0.2","javaHome":"/opt/graalvm/Contents/Home","osName":"Mac OS X","osVersion":"26.5.2","processors":12,"systemLoadAverage":7.25341796875,"uptime":260192},"processCpuPercent":0.0,"connectors":[{"name":"\"http-nio-8080\"","objectName":"Catalina:name=\"http-nio-8080\",type=ThreadPool","currentThreadsBusy":3,"maxThreads":200}],"dataSources":[],"series":{"collectors":["G1 Young Generation","G1 Old Generation"],"connectors":["\"http-nio-8080\""],"dataSources":[]}}` + + It("decodes every metric the agent sends", func() { + metrics, err := jvmruntime.DecodeMetrics(json.RawMessage(live)) + Expect(err).NotTo(HaveOccurred()) + Expect(metrics).To(Equal(jvmruntime.Metrics{ + CapturedAt: time.Date(2026, 11, 16, 4, 53, 48, 543631000, time.UTC), + Memory: jvmruntime.Memory{ + Heap: []jvmruntime.MemoryPool{ + {Name: "G1 Eden Space", Type: "HEAP", Used: 20971520, Committed: 29360128, Max: -1}, + {Name: "G1 Old Gen", Type: "HEAP", Used: 1230584, Committed: 4194304, Max: 8589934592}, + }, + NonHeap: []jvmruntime.MemoryPool{{Name: "Metaspace", Type: "NON_HEAP", Used: 24222544, Committed: 24444928, Max: -1}}, + Buffer: []jvmruntime.MemoryPool{{Name: "direct", Type: "BUFFER", Max: -1}}, + }, + GarbageCollectors: []jvmruntime.GarbageCollector{{Name: "G1 Young Generation", CollectionCount: 2, CollectionTime: 11}, {Name: "G1 Old Generation"}}, + Runtime: jvmruntime.Runtime{JavaVersion: "25.0.2", JavaHome: "/opt/graalvm/Contents/Home", OSName: "Mac OS X", OSVersion: "26.5.2", Processors: 12, SystemLoadAverage: 7.25341796875, Uptime: 260192}, + ProcessCPUPercent: new(0.0), + Connectors: []map[string]any{{"name": `"http-nio-8080"`, "objectName": `Catalina:name="http-nio-8080",type=ThreadPool`, "currentThreadsBusy": 3.0, "maxThreads": 200.0}}, + DataSources: []map[string]any{}, + Series: jvmruntime.MetricSeries{Collectors: []string{"G1 Young Generation", "G1 Old Generation"}, Connectors: []string{`"http-nio-8080"`}, DataSources: []string{}}, + })) + }) + + It("renders a runtime heading, a memory pool table and one line per collector group", func() { + metrics, err := jvmruntime.DecodeMetrics(json.RawMessage(live)) + Expect(err).NotTo(HaveOccurred()) + rendered := lines(metrics.Pretty()) + Expect(rendered[0]).To(Equal("JVM metrics Java 25.0.2 · Mac OS X 26.5.2 · 12 cpus · load 7.25 · up 4m20s · process cpu 0.0% · 2026-11-16T04:53:48Z")) + Expect(rendered[len(rendered)-2:]).To(Equal([]string{ + "GC G1 Young Generation 2 in 11ms · G1 Old Generation 0 in 0ms", + `Connectors "http-nio-8080" currentThreadsBusy=3 maxThreads=200`, + })) + }) + + It("tabulates each pool with its area and an unbounded max", func() { + pool := jvmruntime.MemoryPool{Name: "G1 Eden Space", Type: "HEAP", Used: 20971520, Committed: 29360128, Max: -1} + var keys []string + for _, column := range pool.Columns() { + keys = append(keys, column.Name) + } + Expect(keys).To(Equal([]string{"name", "type", "used", "committed", "max"})) + row := pool.Row() + Expect(row["name"]).To(Equal("G1 Eden Space")) + Expect(row["type"]).To(Equal("heap")) + Expect(stringOf(row["used"])).To(Equal("20M")) + Expect(stringOf(row["max"])).To(Equal("unbounded")) + }) + + It("refuses a result that is not a metrics snapshot", func() { + for _, raw := range []string{`{}`, `{"capturedAt":"2026-11-16T04:53:48Z","memory":{}}`, `[]`} { + _, err := jvmruntime.DecodeMetrics(json.RawMessage(raw)) + Expect(err).To(HaveOccurred(), raw) + } + }) +}) + +func stringOf(value any) string { + return value.(interface{ String() string }).String() +} diff --git a/pkg/tracing/jvm/jvmruntime/metrics.go b/pkg/tracing/jvm/jvmruntime/metrics.go new file mode 100644 index 0000000..872a139 --- /dev/null +++ b/pkg/tracing/jvm/jvmruntime/metrics.go @@ -0,0 +1,154 @@ +package jvmruntime + +import ( + "encoding/json" + "fmt" + "maps" + "slices" + "strings" + "time" + + "github.com/flanksource/clicky/api" +) + +// MetricsOp snapshots the JVM's memory pools, collectors, runtime and the +// Tomcat connector and data source pools it can find over JMX. +const MetricsOp = "jvm.metrics" + +// Metrics is the agent's jvm.metrics snapshot. Connectors and DataSources +// carry whichever pool attributes the server exposes, so they stay maps. +type Metrics struct { + CapturedAt time.Time `json:"capturedAt"` + Memory Memory `json:"memory"` + GarbageCollectors []GarbageCollector `json:"garbageCollectors"` + Runtime Runtime `json:"runtime"` + ProcessCPUPercent *float64 `json:"processCpuPercent,omitempty"` + Connectors []map[string]any `json:"connectors"` + DataSources []map[string]any `json:"dataSources"` + Series MetricSeries `json:"series"` +} + +// Memory is the JVM's pools by area. +type Memory struct { + Heap []MemoryPool `json:"heap"` + NonHeap []MemoryPool `json:"nonHeap"` + Buffer []MemoryPool `json:"buffer"` +} + +// MemoryPool is one pool's usage in bytes; Max is -1 when the pool is unbounded. +type MemoryPool struct { + Name string `json:"name"` + Type string `json:"type"` + Used int64 `json:"used"` + Committed int64 `json:"committed"` + Max int64 `json:"max"` +} + +// GarbageCollector is one collector's lifetime totals; CollectionTime is millis. +type GarbageCollector struct { + Name string `json:"name"` + CollectionCount int64 `json:"collectionCount"` + CollectionTime int64 `json:"collectionTime"` +} + +// Runtime is the JVM and host it runs on; Uptime is millis and +// SystemLoadAverage is negative where the OS does not report one. +type Runtime struct { + JavaVersion string `json:"javaVersion"` + JavaHome string `json:"javaHome"` + OSName string `json:"osName"` + OSVersion string `json:"osVersion"` + Processors int `json:"processors"` + SystemLoadAverage float64 `json:"systemLoadAverage"` + Uptime int64 `json:"uptime"` +} + +// MetricSeries names the collectors, connectors and data sources a +// timeseries sampler should track. +type MetricSeries struct { + Collectors []string `json:"collectors"` + Connectors []string `json:"connectors"` + DataSources []string `json:"dataSources"` +} + +// DecodeMetrics decodes the agent's snapshot, refusing one without the +// sections the agent always sends. +func DecodeMetrics(raw json.RawMessage) (Metrics, error) { + metrics, err := decodeObject[Metrics](MetricsOp, raw, "capturedAt", "memory", "garbageCollectors", "runtime", "connectors", "dataSources", "series") + if err == nil && metrics.Memory.Heap == nil { + err = fmt.Errorf("%s returned no heap pools: %s", MetricsOp, raw) + } + return metrics, err +} + +func (MemoryPool) Columns() []api.ColumnDef { + return []api.ColumnDef{ + api.Column("name").Label("Pool").Build(), + api.Column("type").Label("Area").Build(), + api.Column("used").Label("Used").Build(), + api.Column("committed").Label("Committed").Build(), + api.Column("max").Label("Max").Build(), + } +} + +func (p MemoryPool) Row() map[string]any { + limit := api.HumanizeBytes(p.Max) + if p.Max < 0 { + limit = api.Text{}.Append("unbounded", "text-muted") + } + return map[string]any{ + "name": p.Name, + "type": strings.ReplaceAll(strings.ToLower(p.Type), "_", "-"), + "used": api.HumanizeBytes(p.Used), + "committed": api.HumanizeBytes(p.Committed), + "max": limit, + } +} + +// Pretty is a runtime heading, the memory pool table, then one line for the +// collectors and one for each pool group the server exposes. +func (m Metrics) Pretty() api.Text { + text := api.Text{}.Append("JVM metrics", "font-bold").Append(" "+m.runtimeSummary(), "text-muted"). + NewLine().Add(api.NewTableFrom(slices.Concat(m.Memory.Heap, m.Memory.NonHeap, m.Memory.Buffer))) + var collectors []string + for _, collector := range m.GarbageCollectors { + collectors = append(collectors, fmt.Sprintf("%s %s in %s", collector.Name, api.HumanNumber(collector.CollectionCount).String(), millis(collector.CollectionTime))) + } + text = text.NewLine().Append("GC", "font-bold").Append(" " + strings.Join(collectors, " · ")) + for _, group := range []struct { + label string + pools []map[string]any + }{{"Connectors", m.Connectors}, {"Data sources", m.DataSources}} { + if len(group.pools) > 0 { + text = text.NewLine().Append(group.label, "font-bold").Append(" " + poolSummary(group.pools)) + } + } + return text +} + +func (m Metrics) runtimeSummary() string { + facts := []string{"Java " + m.Runtime.JavaVersion, m.Runtime.OSName + " " + m.Runtime.OSVersion, fmt.Sprintf("%d cpus", m.Runtime.Processors)} + if m.Runtime.SystemLoadAverage >= 0 { + facts = append(facts, fmt.Sprintf("load %.2f", m.Runtime.SystemLoadAverage)) + } + facts = append(facts, "up "+millis(m.Runtime.Uptime)) + if m.ProcessCPUPercent != nil { + facts = append(facts, fmt.Sprintf("process cpu %.1f%%", *m.ProcessCPUPercent)) + } + return strings.Join(append(facts, m.CapturedAt.UTC().Format(time.RFC3339)), " · ") +} + +// poolSummary renders each pool as its name and attributes, objectName aside. +func poolSummary(pools []map[string]any) string { + var out []string + for _, pool := range pools { + facts := []string{fmt.Sprint(pool["name"])} + for _, key := range slices.Sorted(maps.Keys(pool)) { + if key != "name" && key != "objectName" { + facts = append(facts, fmt.Sprintf("%s=%v", key, pool[key])) + } + } + out = append(out, strings.Join(facts, " ")) + } + return strings.Join(out, " · ") +} diff --git a/pkg/tracing/jvm/jvmruntime/settings.go b/pkg/tracing/jvm/jvmruntime/settings.go new file mode 100644 index 0000000..cfeeb75 --- /dev/null +++ b/pkg/tracing/jvm/jvmruntime/settings.go @@ -0,0 +1,171 @@ +package jvmruntime + +import ( + "encoding/json" + "fmt" + "maps" + "regexp" + "slices" + "strings" + + "github.com/flanksource/clicky/api" + "github.com/flanksource/telltale/pkg/tracing/jvm/jmx" +) + +const ( + // SettingsOp reads the JVM's system properties, environment and its Runtime + // and OperatingSystem MBeans. + SettingsOp = "jvm.settings" + // ResourcesOp reads filesystem capacity and, on Linux, the process's limits + // and cgroup settings. + ResourcesOp = "jvm.resources" + // FlagsOp reads every HotSpot VM flag (VM.flags -all). + FlagsOp = "jvm.flags" +) + +// Settings is the JVM's configuration as the agent reads it. +type Settings struct { + Properties map[string]string `json:"properties"` + Environment map[string]string `json:"environment"` + Runtime jmx.MBeanAttributes `json:"runtime"` + OperatingSystem jmx.MBeanAttributes `json:"operatingSystem"` +} + +// DecodeSettings decodes jvm.settings, refusing one without its properties or MBeans. +func DecodeSettings(raw json.RawMessage) (Settings, error) { + settings, err := decodeObject[Settings](SettingsOp, raw, "properties", "environment", "runtime", "operatingSystem") + if err == nil && (settings.Runtime.ObjectName == "" || settings.OperatingSystem.ObjectName == "") { + err = fmt.Errorf("%s returned an MBean without its object name: %s", SettingsOp, raw) + } + return settings, err +} + +// Pretty is a one-line summary, then the environment's variable names: their +// values stay in the structured formats, since environments carry secrets. +func (s Settings) Pretty() api.Text { + p := s.Properties + facts := []string{ + fmt.Sprintf("Java %s (%s)", p["java.version"], p["java.vendor"]), + fmt.Sprintf("%s %s %s", p["os.name"], p["os.version"], p["os.arch"]), + fmt.Sprintf("%d properties", len(s.Properties)), + fmt.Sprintf("%d environment variables", len(s.Environment)), + } + text := api.Text{}.Append("JVM settings", "font-bold").Append(" "+strings.Join(facts, " · "), "text-muted") + if len(s.Environment) > 0 { + text = text.NewLine().Append("Environment", "font-bold").Append(" " + strings.Join(slices.Sorted(maps.Keys(s.Environment)), ", ")) + } + return text +} + +// Resources is the disk and, on Linux, process and cgroup resources the JVM +// runs under. Linux is set ({"status":"not-applicable"}) on any other OS, in +// place of the /proc files and Cgroups. +type Resources struct { + Filesystems []Filesystem `json:"filesystems"` + Linux map[string]string `json:"linux,omitempty"` + Limits string `json:"limits,omitempty"` + Status string `json:"status,omitempty"` + Cgroup string `json:"cgroup,omitempty"` + Mountinfo string `json:"mountinfo,omitempty"` + Cgroups map[string]map[string]string `json:"cgroups,omitempty"` +} + +// Filesystem is one filesystem root's capacity in bytes. +type Filesystem struct { + Path string `json:"path"` + Total int64 `json:"total"` + Usable int64 `json:"usable"` + Free int64 `json:"free"` +} + +// DecodeResources decodes jvm.resources, refusing one that has neither the +// Linux /proc reads nor a reason it skipped them. +func DecodeResources(raw json.RawMessage) (Resources, error) { + resources, err := decodeObject[Resources](ResourcesOp, raw, "filesystems") + if err != nil { + return resources, err + } + if resources.Linux == nil && resources.Cgroups == nil { + return resources, fmt.Errorf("%s returned neither cgroups nor a linux status: %s", ResourcesOp, raw) + } + return resources, nil +} + +// Pretty is a line per filesystem, then a line per cgroup with its limits, or +// why there are none. +func (r Resources) Pretty() api.Text { + text := api.Text{}.Append("Resources", "font-bold") + for _, fs := range r.Filesystems { + text = text.NewLine().Append(fs.Path, "font-bold"). + Append(fmt.Sprintf(" %s usable of %s", api.HumanizeBytes(fs.Usable).String(), api.HumanizeBytes(fs.Total).String()), "text-muted") + } + if r.Linux != nil { + return text.NewLine().Append("cgroups "+r.Linux["status"], "text-muted") + } + for _, group := range slices.Sorted(maps.Keys(r.Cgroups)) { + var limits []string + for _, name := range slices.Sorted(maps.Keys(r.Cgroups[group])) { + limits = append(limits, name+"="+strings.TrimSpace(r.Cgroups[group][name])) + } + text = text.NewLine().Append("cgroup "+group, "font-bold").Append(" "+strings.Join(limits, " "), "text-muted") + } + return text +} + +// VMFlags is every HotSpot flag as VM.flags -all prints it. +type VMFlags struct { + Flags []VMFlag `json:"flags"` +} + +// VMFlag is one flag line: its type, name and value, the kind of flag +// ("product", "manageable", …) and where its value came from ("default", +// "ergonomic", "command line", …). +type VMFlag struct { + Type string `json:"type"` + Name string `json:"name"` + Value string `json:"value"` + Kind string `json:"kind"` + Origin string `json:"origin"` +} + +var flagLine = regexp.MustCompile(`^\s*(\S+)\s+(\S+)\s+= ?(.*?)\s*\{([^}]*)\} \{([^}]*)\}\s*$`) + +// DecodeFlags decodes jvm.flags, a JSON string of VM.flags -all output, +// refusing any line it does not recognise rather than dropping it. +func DecodeFlags(raw json.RawMessage) (VMFlags, error) { + var text string + if err := json.Unmarshal(raw, &text); err != nil { + return VMFlags{}, fmt.Errorf("decode %s result: %w", FlagsOp, err) + } + flags := VMFlags{Flags: []VMFlag{}} + for number, line := range strings.Split(text, "\n") { + if strings.TrimSpace(line) == "" || line == "[Global flags]" { + continue + } + match := flagLine.FindStringSubmatch(line) + if match == nil { + return flags, fmt.Errorf("%s line %d is not a flag: %q", FlagsOp, number+1, line) + } + flags.Flags = append(flags.Flags, VMFlag{Type: match[1], Name: match[2], Value: match[3], Kind: match[4], Origin: match[5]}) + } + if len(flags.Flags) == 0 { + return flags, fmt.Errorf("%s returned no flags: %q", FlagsOp, text) + } + return flags, nil +} + +// Pretty is a heading counting every flag, then one line per flag whose value +// did not come from the default, which is what a reader is looking for. +func (f VMFlags) Pretty() api.Text { + var set []VMFlag + for _, flag := range f.Flags { + if flag.Origin != "default" { + set = append(set, flag) + } + } + text := api.Text{}.Append("VM flags", "font-bold").Append(fmt.Sprintf(" %d flags · %d not default", len(f.Flags), len(set)), "text-muted") + for _, flag := range set { + text = text.NewLine().Append(flag.Name, "font-bold").Append(" = " + flag.Value).Append(" "+flag.Origin, "text-muted") + } + return text +} diff --git a/pkg/tracing/jvm/jvmruntime/settings_test.go b/pkg/tracing/jvm/jvmruntime/settings_test.go new file mode 100644 index 0000000..38e1a2e --- /dev/null +++ b/pkg/tracing/jvm/jvmruntime/settings_test.go @@ -0,0 +1,79 @@ +package jvmruntime_test + +import ( + "encoding/json" + "os" + "path/filepath" + + "github.com/flanksource/telltale/pkg/tracing/jvm/jvmruntime" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func fixture(op string) json.RawMessage { + raw, err := os.ReadFile(filepath.Join("testdata", op+".json")) + Expect(err).NotTo(HaveOccurred()) + return raw +} + +var _ = Describe("settings", func() { + It("summarises the runtime on one line and names, but does not print, the environment's values", func() { + settings, err := jvmruntime.DecodeSettings(fixture(jvmruntime.SettingsOp)) + Expect(err).NotTo(HaveOccurred()) + Expect(settings.Runtime.ObjectName).To(Equal("java.lang:type=Runtime")) + Expect(lines(settings.Pretty())).To(Equal([]string{ + "JVM settings Java 25.0.2 (GraalVM Community) · Mac OS X 26.5.2 aarch64 · 15 properties · 2 environment variables", + "Environment PATH, __CF_USER_TEXT_ENCODING", + })) + }) + + It("refuses settings without their MBeans", func() { + for _, raw := range []string{`{}`, `{"properties":{},"environment":{}}`, `{"properties":{},"environment":{},"runtime":{},"operatingSystem":{}}`} { + _, err := jvmruntime.DecodeSettings(json.RawMessage(raw)) + Expect(err).To(HaveOccurred(), raw) + } + }) +}) + +var _ = Describe("resources", func() { + It("prints each filesystem's free space and each cgroup's limits", func() { + resources, err := jvmruntime.DecodeResources(fixture(jvmruntime.ResourcesOp)) + Expect(err).NotTo(HaveOccurred()) + Expect(lines(resources.Pretty())).To(Equal([]string{ + "Resources", + "/ 57.3G usable of 97.9G", + "cgroup /sys/fs/cgroup cpu.max=200000 100000 memory.current=612368384 memory.max=2147483648", + })) + }) + + It("says why a non-Linux JVM has no cgroups", func() { + resources, err := jvmruntime.DecodeResources(json.RawMessage(`{"filesystems":[{"path":"/","total":1024,"usable":512,"free":512}],"linux":{"status":"not-applicable"}}`)) + Expect(err).NotTo(HaveOccurred()) + Expect(lines(resources.Pretty())[2]).To(Equal("cgroups not-applicable")) + }) + + It("refuses resources with neither cgroups nor a reason for their absence", func() { + _, err := jvmruntime.DecodeResources(json.RawMessage(`{"filesystems":[]}`)) + Expect(err).To(MatchError(ContainSubstring("neither cgroups nor a linux status"))) + }) +}) + +var _ = Describe("VM flags", func() { + It("parses every flag and prints only the ones not at their default", func() { + flags, err := jvmruntime.DecodeFlags(fixture(jvmruntime.FlagsOp)) + Expect(err).NotTo(HaveOccurred()) + Expect(flags.Flags).To(HaveLen(10)) + Expect(flags.Flags[0]).To(Equal(jvmruntime.VMFlag{Type: "ccstr", Name: "AOTCache", Value: "", Kind: "product", Origin: "default"})) + Expect(flags.Flags[4]).To(Equal(jvmruntime.VMFlag{Type: "bool", Name: "EnableDynamicAgentLoading", Value: "true", Kind: "product", Origin: "command line"})) + Expect(flags.Flags[8].Kind).To(Equal("product lp64_product")) + rendered := lines(flags.Pretty()) + Expect(rendered[0]).To(Equal("VM flags 10 flags · 7 not default")) + Expect(rendered).To(ContainElement("EnableDynamicAgentLoading = true command line")) + Expect(rendered).NotTo(ContainElement(ContainSubstring("AOTCache"))) + }) + + It("refuses a line it cannot read rather than dropping it", func() { + _, err := jvmruntime.DecodeFlags(json.RawMessage(`"[Global flags]\n bool UseG1GC = true {product} {ergonomic}\nnot a flag\n"`)) + Expect(err).To(MatchError(ContainSubstring(`line 3 is not a flag: "not a flag"`))) + }) +}) diff --git a/pkg/tracing/jvm/jvmruntime/stack.go b/pkg/tracing/jvm/jvmruntime/stack.go new file mode 100644 index 0000000..d1bde9b --- /dev/null +++ b/pkg/tracing/jvm/jvmruntime/stack.go @@ -0,0 +1,56 @@ +package jvmruntime + +import ( + "encoding/json" + "fmt" + + "github.com/flanksource/clicky/api" +) + +// ThreadStackOp reads one thread's row, jstack text and frames at full depth. +const ThreadStackOp = "jvm.threads.stack" + +// ThreadStack is the agent's jvm.threads.stack. Dump is Text read back into a +// jstack block for rendering; it carries the lock lines Frames does not. +type ThreadStack struct { + Thread Thread `json:"thread"` + Text string `json:"text"` + Frames []StackFrame `json:"frames"` + Dump DumpedThread `json:"-"` +} + +// StackFrame is one stack element. FileName is null when the class carries no +// source name; LineNumber is -2 for a native method and -1 when unknown. +type StackFrame struct { + ClassName string `json:"className"` + MethodName string `json:"methodName"` + FileName *string `json:"fileName"` + LineNumber int `json:"lineNumber"` +} + +// DecodeThreadStack decodes the agent's stack, refusing one whose text is not +// exactly the requested thread's jstack block. +func DecodeThreadStack(raw json.RawMessage) (ThreadStack, error) { + stack, err := decodeObject[ThreadStack](ThreadStackOp, raw, "thread", "text", "frames") + if err != nil { + return stack, err + } + dump, err := parseThreadDump(ThreadStackOp, stack.Text) + if err != nil { + return stack, err + } + if len(dump.Threads) != 1 || dump.Threads[0].ID != stack.Thread.ID { + return stack, fmt.Errorf("%s text is not thread %d's stack: %q", ThreadStackOp, stack.Thread.ID, stack.Text) + } + stack.Dump = dump.Threads[0] + return stack, nil +} + +// Pretty is the thread's header, with its CPU time, over its frames and locks. +func (s ThreadStack) Pretty() api.Text { + suffix := "" + if s.Thread.CPUTimeMs != nil { + suffix = " · cpu " + millis(*s.Thread.CPUTimeMs) + } + return s.Dump.render(suffix) +} diff --git a/pkg/tracing/jvm/jvmruntime/status.go b/pkg/tracing/jvm/jvmruntime/status.go new file mode 100644 index 0000000..9cd7369 --- /dev/null +++ b/pkg/tracing/jvm/jvmruntime/status.go @@ -0,0 +1,81 @@ +package jvmruntime + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/flanksource/clicky/api" +) + +// StatusOp reports whether the agent allows mutation, whether instrumentation +// is installed and, when it is, every class the agent has patched. +const StatusOp = "jvm.status" + +// Status is the agent's jvm.status. Patches is nil when instrumentation is +// off: the agent cannot know about patches then, which is not "no patches". +type Status struct { + AllowMutation bool `json:"allowMutation"` + Instrumentation bool `json:"instrumentation"` + Patches []Patch `json:"patches"` +} + +// Patch is one class whose original bytecode the agent holds. LeaseExpiresAt +// is epoch millis when the agent will restore it; 0 is a patch nothing will undo. +type Patch struct { + Class string `json:"class"` + Loader string `json:"loader"` + Digest string `json:"digest,omitempty"` + OriginalDigest string `json:"originalDigest,omitempty"` + Patched bool `json:"patched"` + LeaseExpiresAt int64 `json:"leaseExpiresAt,omitempty"` + Error string `json:"error,omitempty"` +} + +// DecodeStatus decodes the agent's status, refusing one that leaves out what +// the agent always reports. +func DecodeStatus(raw json.RawMessage) (Status, error) { + status, err := decodeObject[Status](StatusOp, raw, "allowMutation", "instrumentation") + if err == nil && status.Instrumentation && status.Patches == nil { + err = fmt.Errorf("%s reported instrumentation without its patches: %s", StatusOp, raw) + } + return status, err +} + +// Pretty is the status as one line, with any patches listed compactly. +func (s Status) Pretty() api.Text { + text := api.Text{}.Append("mutations allowed", "text-green-600") + if !s.AllowMutation { + text = api.Text{}.Append("mutations denied", "text-yellow-600") + } + text = text.Append(" · ") + if !s.Instrumentation { + return text.Append("instrumentation off", "text-red-600") + } + noun := "patches" + if len(s.Patches) == 1 { + noun = "patch" + } + text = text.Append("instrumentation on", "text-green-600").Append(fmt.Sprintf(" · %d %s", len(s.Patches), noun), "text-muted") + separator := ": " + for _, patch := range s.Patches { + text = text.Append(separator).Add(patch.Pretty()) + separator = ", " + } + return text +} + +// Pretty is the patched class and whether, and when, it will be restored. +func (p Patch) Pretty() api.Text { + text := api.Text{}.Append(p.Class, "font-bold") + switch { + case p.Error != "": + return text.Append(" "+p.Error, "text-red-600") + case !p.Patched: + return text.Append(" restored", "text-muted") + case p.LeaseExpiresAt == 0: + return text.Append(" no lease", "text-yellow-600") + default: + return text.Append(" lease until "+time.UnixMilli(p.LeaseExpiresAt).UTC().Format(time.RFC3339), "text-muted") + } +} diff --git a/pkg/tracing/jvm/jvmruntime/testdata/jvm.flags.json b/pkg/tracing/jvm/jvmruntime/testdata/jvm.flags.json new file mode 100644 index 0000000..0efd8c5 --- /dev/null +++ b/pkg/tracing/jvm/jvmruntime/testdata/jvm.flags.json @@ -0,0 +1 @@ +"[Global flags]\n ccstr AOTCache = {product} {default}\n bool AOTClassLinking = false {product} {default}\n intx CICompilerCount = 4 {product} {ergonomic}\nccstrlist CompileCommand = {product} {default}\n bool EnableDynamicAgentLoading = true {product} {command line}\n bool EnableJVMCI = true {JVMCI product} {jimage}\n size_t MaxHeapSize = 8589934592 {product} {ergonomic}\n size_t SoftMaxHeapSize = 8589934592 {manageable} {ergonomic}\n bool UseCompressedOops = true {product lp64_product} {ergonomic}\n bool UseG1GC = true {product} {ergonomic}\n" diff --git a/pkg/tracing/jvm/jvmruntime/testdata/jvm.metrics.json b/pkg/tracing/jvm/jvmruntime/testdata/jvm.metrics.json new file mode 100644 index 0000000..c9754c1 --- /dev/null +++ b/pkg/tracing/jvm/jvmruntime/testdata/jvm.metrics.json @@ -0,0 +1 @@ +{"capturedAt":"2026-11-16T04:59:26.652930Z","memory":{"heap":[{"name":"G1 Eden Space","type":"HEAP","used":8388608,"committed":58720256,"max":-1},{"name":"G1 Old Gen","type":"HEAP","used":17925592,"committed":54525952,"max":8589934592},{"name":"G1 Survivor Space","type":"HEAP","used":17704008,"committed":20971520,"max":-1}],"nonHeap":[{"name":"CodeHeap \u0027non-nmethods\u0027","type":"NON_HEAP","used":1739520,"committed":2621440,"max":6406144},{"name":"Metaspace","type":"NON_HEAP","used":55839576,"committed":56426496,"max":-1},{"name":"CodeHeap \u0027profiled nmethods\u0027","type":"NON_HEAP","used":14589696,"committed":14614528,"max":122617856},{"name":"Compressed Class Space","type":"NON_HEAP","used":6819344,"committed":7077888,"max":1073741824},{"name":"CodeHeap \u0027non-profiled nmethods\u0027","type":"NON_HEAP","used":1571264,"committed":2555904,"max":122634240}],"buffer":[{"name":"mapped","type":"BUFFER","used":0,"committed":0,"max":-1},{"name":"direct","type":"BUFFER","used":0,"committed":0,"max":-1},{"name":"mapped - \u0027non-volatile memory\u0027","type":"BUFFER","used":0,"committed":0,"max":-1}]},"garbageCollectors":[{"name":"G1 Young Generation","collectionCount":15,"collectionTime":45},{"name":"G1 Concurrent GC","collectionCount":6,"collectionTime":5},{"name":"G1 Old Generation","collectionCount":1,"collectionTime":21}],"runtime":{"javaVersion":"25.0.2","javaHome":"/opt/graalvm/Contents/Home","osName":"Mac OS X","osVersion":"26.5.2","processors":12,"systemLoadAverage":6.634765625,"uptime":598318},"processCpuPercent":0.09098106439764791,"connectors":[],"dataSources":[],"series":{"collectors":["G1 Young Generation","G1 Concurrent GC","G1 Old Generation"],"connectors":[],"dataSources":[]}} diff --git a/pkg/tracing/jvm/jvmruntime/testdata/jvm.resources.json b/pkg/tracing/jvm/jvmruntime/testdata/jvm.resources.json new file mode 100644 index 0000000..48f68dd --- /dev/null +++ b/pkg/tracing/jvm/jvmruntime/testdata/jvm.resources.json @@ -0,0 +1 @@ +{"filesystems":[{"path":"/","total":105088212992,"usable":61505916928,"free":66881417216}],"limits":"Limit Soft Limit Hard Limit Units \nMax open files 1048576 1048576 files \n","status":"Name:\tjava\nVmRSS:\t 412344 kB\nThreads:\t42\n","cgroup":"0::/\n","mountinfo":"1210 1180 0:28 / /sys/fs/cgroup ro,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw\n","cgroups":{"/sys/fs/cgroup":{"cpu.max":"200000 100000\n","memory.max":"2147483648\n","memory.current":"612368384\n"}}} diff --git a/pkg/tracing/jvm/jvmruntime/testdata/jvm.settings.json b/pkg/tracing/jvm/jvmruntime/testdata/jvm.settings.json new file mode 100644 index 0000000..9e02450 --- /dev/null +++ b/pkg/tracing/jvm/jvmruntime/testdata/jvm.settings.json @@ -0,0 +1 @@ +{"properties":{"file.encoding":"UTF-8","java.class.path":".","java.home":"/opt/jdk","java.runtime.version":"25.0.2+10-jvmci-b01","java.vendor":"GraalVM Community","java.version":"25.0.2","java.vm.name":"OpenJDK 64-Bit Server VM","os.arch":"aarch64","os.name":"Mac OS X","os.version":"26.5.2","service.name":"idle-demo","user.dir":"/home/example/app","user.home":"/home/example","user.name":"example","user.timezone":"UTC"},"environment":{"PATH":"/usr/bin:/bin","__CF_USER_TEXT_ENCODING":"0x1F5:0x0:0x0"},"runtime":{"objectName":"java.lang:type=Runtime","attributes":[{"name":"Pid","value":25367},{"name":"VmName","value":"OpenJDK 64-Bit Server VM"},{"name":"VmVersion","value":"25.0.2+10-jvmci-b01"},{"name":"InputArguments","value":["-XX:+EnableDynamicAgentLoading","-Duser.name=example","-Dservice.name=idle-demo"]},{"name":"Uptime","value":10534},{"name":"Name","value":"25367@host.example"},{"name":"ObjectName","value":"java.lang:type=Runtime"}]},"operatingSystem":{"objectName":"java.lang:type=OperatingSystem","attributes":[{"name":"OpenFileDescriptorCount","value":16},{"name":"MaxFileDescriptorCount","value":10240},{"name":"TotalMemorySize","value":34359738368},{"name":"AvailableProcessors","value":12},{"name":"Arch","value":"aarch64"},{"name":"SystemLoadAverage","value":8.0693359375},{"name":"Name","value":"Mac OS X"},{"name":"Version","value":"26.5.2"},{"name":"ObjectName","value":"java.lang:type=OperatingSystem"}]}} diff --git a/pkg/tracing/jvm/jvmruntime/testdata/jvm.status.json b/pkg/tracing/jvm/jvmruntime/testdata/jvm.status.json new file mode 100644 index 0000000..18e13d5 --- /dev/null +++ b/pkg/tracing/jvm/jvmruntime/testdata/jvm.status.json @@ -0,0 +1 @@ +{"allowMutation":true,"instrumentation":true,"patches":[]} diff --git a/pkg/tracing/jvm/jvmruntime/testdata/jvm.threads.dump.json b/pkg/tracing/jvm/jvmruntime/testdata/jvm.threads.dump.json new file mode 100644 index 0000000..bbfd129 --- /dev/null +++ b/pkg/tracing/jvm/jvmruntime/testdata/jvm.threads.dump.json @@ -0,0 +1 @@ +"\"main\" #3 prio\u003d5\n java.lang.Thread.State: TIMED_WAITING\n\tat java.lang.Thread.sleepNanos0(Native Method)\n\tat java.lang.Thread.sleepNanos(Thread.java:509)\n\tat java.lang.Thread.sleep(Thread.java:540)\n\tat Idle.main(Unknown Source)\n\tat java.lang.invoke.LambdaForm$DMH/0x00000e000106c000.invokeStatic(LambdaForm$DMH)\n\tat java.lang.invoke.LambdaForm$MH/0x00000e000114d400.invoke(LambdaForm$MH)\n\tat java.lang.invoke.Invokers$Holder.invokeExact_MT(Invokers$Holder)\n\tat jdk.internal.reflect.DirectMethodHandleAccessor.invokeImpl(DirectMethodHandleAccessor.java:155)\n\tat jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)\n\tat java.lang.reflect.Method.invoke(Method.java:565)\n\tat com.sun.tools.javac.launcher.SourceLauncher.execute(SourceLauncher.java:258)\n\tat com.sun.tools.javac.launcher.SourceLauncher.run(SourceLauncher.java:138)\n\tat com.sun.tools.javac.launcher.SourceLauncher.main(SourceLauncher.java:76)\n\n Locked ownable synchronizers:\n\t- None\n\n\"Reference Handler\" #14 daemon prio\u003d10\n java.lang.Thread.State: RUNNABLE\n\tat java.lang.ref.Reference.waitForReferencePendingList(Native Method)\n\tat java.lang.ref.Reference.processPendingReferences(Reference.java:246)\n\tat java.lang.ref.Reference$ReferenceHandler.run(Reference.java:208)\n\n Locked ownable synchronizers:\n\t- None\n\n\"Finalizer\" #15 daemon prio\u003d8\n java.lang.Thread.State: WAITING\n\tat java.lang.Object.wait0(Native Method)\n\t- waiting on \u003c0x7ce5d946\u003e (a java.lang.ref.ReferenceQueue$Lock)\n\tat java.lang.Object.wait(Object.java:389)\n\tat java.lang.Object.wait(Object.java:351)\n\tat java.lang.ref.ReferenceQueue.remove0(ReferenceQueue.java:137)\n\tat java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:215)\n\tat java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:165)\n\n Locked ownable synchronizers:\n\t- None\n\n\"Signal Dispatcher\" #16 daemon prio\u003d9\n java.lang.Thread.State: RUNNABLE\n\n Locked ownable synchronizers:\n\t- None\n\n\"Notification Thread\" #23 daemon prio\u003d9\n java.lang.Thread.State: RUNNABLE\n\n Locked ownable synchronizers:\n\t- None\n\n\"Common-Cleaner\" #24 daemon prio\u003d8\n java.lang.Thread.State: TIMED_WAITING\n\tat java.lang.Object.wait0(Native Method)\n\t- waiting on \u003c0x2ae76c3d\u003e (a java.lang.ref.ReferenceQueue$Lock)\n\tat java.lang.Object.wait(Object.java:389)\n\tat java.lang.ref.ReferenceQueue.remove0(ReferenceQueue.java:123)\n\tat java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:201)\n\tat jdk.internal.ref.CleanerImpl.run(CleanerImpl.java:146)\n\tat java.lang.Thread.runWith(Thread.java:1487)\n\tat java.lang.Thread.run(Thread.java:1474)\n\tat jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:148)\n\n Locked ownable synchronizers:\n\t- None\n\n\"Attach Listener\" #25 daemon prio\u003d9\n java.lang.Thread.State: RUNNABLE\n\n Locked ownable synchronizers:\n\t- None\n\n\"telltale-agent\" #27 daemon prio\u003d9\n java.lang.Thread.State: RUNNABLE\n\tat sun.nio.ch.Net.accept(Native Method)\n\tat sun.nio.ch.NioSocketImpl.accept(NioSocketImpl.java:751)\n\tat java.net.ServerSocket.implAccept(ServerSocket.java:593)\n\tat java.net.ServerSocket.platformImplAccept(ServerSocket.java:560)\n\tat java.net.ServerSocket.implAccept(ServerSocket.java:536)\n\tat java.net.ServerSocket.implAccept(ServerSocket.java:482)\n\tat java.net.ServerSocket.accept(ServerSocket.java:441)\n\tat com.flanksource.telltale.Agent.accept(Agent.java:75)\n\tat com.flanksource.telltale.Agent.lambda$new$0(Agent.java:61)\n\tat com.flanksource.telltale.Agent$$Lambda/0x00000e00011c9f40.run(Unknown Source)\n\tat java.lang.Thread.runWith(Thread.java:1487)\n\tat java.lang.Thread.run(Thread.java:1474)\n\n Locked ownable synchronizers:\n\t- \u003c0x3bd736a7\u003e (a java.util.concurrent.locks.ReentrantLock$NonfairSync)\n\n\"telltale-diag-lease-reaper\" #84 daemon prio\u003d9\n java.lang.Thread.State: TIMED_WAITING\n\tat java.lang.Thread.sleepNanos0(Native Method)\n\tat java.lang.Thread.sleepNanos(Thread.java:509)\n\tat java.lang.Thread.sleep(Thread.java:540)\n\tat com.flanksource.telltale.diag.agent.DiagAgent$3.run(DiagAgent.java:1096)\n\tat java.lang.Thread.runWith(Thread.java:1487)\n\tat java.lang.Thread.run(Thread.java:1474)\n\n Locked ownable synchronizers:\n\t- None\n\n\"telltale-session\" #156 daemon prio\u003d9\n java.lang.Thread.State: RUNNABLE\n\tat sun.nio.ch.Net.poll(Native Method)\n\tat sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:190)\n\tat sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:279)\n\tat sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:302)\n\tat sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:354)\n\tat sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:798)\n\tat java.net.Socket$SocketInputStream.implRead(Socket.java:974)\n\tat java.net.Socket$SocketInputStream.read(Socket.java:964)\n\tat java.io.DataInputStream.readFully(DataInputStream.java:213)\n\tat java.io.DataInputStream.readInt(DataInputStream.java:390)\n\tat com.flanksource.telltale.Wire.read(Wire.java:24)\n\tat com.flanksource.telltale.Session.run(Session.java:61)\n\tat com.flanksource.telltale.Agent.lambda$accept$1(Agent.java:82)\n\tat com.flanksource.telltale.Agent$$Lambda/0x00000e00011ca170.run(Unknown Source)\n\tat java.lang.Thread.runWith(Thread.java:1487)\n\tat java.lang.Thread.run(Thread.java:1474)\n\n Locked ownable synchronizers:\n\t- \u003c0x149d7a9a\u003e (a java.util.concurrent.locks.ReentrantLock$NonfairSync)\n\n\"telltale-operation\" #157 daemon prio\u003d9\n java.lang.Thread.State: RUNNABLE\n\tat sun.management.ThreadImpl.dumpThreads0(Native Method)\n\tat sun.management.ThreadImpl.dumpAllThreads(ThreadImpl.java:505)\n\tat sun.management.ThreadImpl.dumpAllThreads(ThreadImpl.java:493)\n\tat com.flanksource.telltale.diag.jvm.ThreadDumpService.dump(ThreadDumpService.java:138)\n\tat com.flanksource.telltale.Diagnostics.lambda$new$13(Diagnostics.java:76)\n\tat com.flanksource.telltale.Diagnostics$$Lambda/0x00000e00011c33a0.call(Unknown Source)\n\tat com.flanksource.telltale.Diagnostics.call(Diagnostics.java:193)\n\tat com.flanksource.telltale.RuntimeService.dispatch(RuntimeService.java:110)\n\tat com.flanksource.telltale.RuntimeService.invoke(RuntimeService.java:82)\n\tat com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:628)\n\tat com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:790)\n\tat com.flanksource.telltale.RuntimeClient.request(RuntimeClient.java:15)\n\tat com.flanksource.telltale.RuntimeClient.call(RuntimeClient.java:50)\n\tat com.flanksource.telltale.Session.operation(Session.java:312)\n\tat com.flanksource.telltale.Session.lambda$dispatch$2(Session.java:72)\n\tat com.flanksource.telltale.Session$$Lambda/0x00000e00011cb9a8.run(Unknown Source)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)\n\tat java.lang.Thread.runWith(Thread.java:1487)\n\tat java.lang.Thread.run(Thread.java:1474)\n\n Locked ownable synchronizers:\n\t- \u003c0x173fc06f\u003e (a java.util.concurrent.ThreadPoolExecutor$Worker)\n\n" diff --git a/pkg/tracing/jvm/jvmruntime/testdata/jvm.threads.json b/pkg/tracing/jvm/jvmruntime/testdata/jvm.threads.json new file mode 100644 index 0000000..f3fe8ff --- /dev/null +++ b/pkg/tracing/jvm/jvmruntime/testdata/jvm.threads.json @@ -0,0 +1 @@ +{"sampledMs":100,"cpuTimeEnabled":true,"deadlocked":[],"threads":[{"id":3,"name":"main","state":"TIMED_WAITING","daemon":false,"priority":5,"cpuPercent":0.0,"cpuTimeMs":660,"blockedCount":0,"blockedTimeMs":null,"waitedCount":2948,"waitedTimeMs":null,"lockName":null,"lockOwnerId":null,"lockOwnerName":null,"inNative":false,"suspended":false},{"id":14,"name":"Reference Handler","state":"RUNNABLE","daemon":true,"priority":10,"cpuPercent":0.0,"cpuTimeMs":0,"blockedCount":0,"blockedTimeMs":null,"waitedCount":0,"waitedTimeMs":null,"lockName":null,"lockOwnerId":null,"lockOwnerName":null,"inNative":false,"suspended":false},{"id":15,"name":"Finalizer","state":"WAITING","daemon":true,"priority":8,"cpuPercent":0.0,"cpuTimeMs":0,"blockedCount":6,"blockedTimeMs":null,"waitedCount":7,"waitedTimeMs":null,"lockName":"java.lang.ref.ReferenceQueue$Lock@7ce5d946","lockOwnerId":null,"lockOwnerName":null,"inNative":false,"suspended":false},{"id":16,"name":"Signal Dispatcher","state":"RUNNABLE","daemon":true,"priority":9,"cpuPercent":0.0,"cpuTimeMs":0,"blockedCount":0,"blockedTimeMs":null,"waitedCount":0,"waitedTimeMs":null,"lockName":null,"lockOwnerId":null,"lockOwnerName":null,"inNative":false,"suspended":false},{"id":23,"name":"Notification Thread","state":"RUNNABLE","daemon":true,"priority":9,"cpuPercent":0.0,"cpuTimeMs":0,"blockedCount":0,"blockedTimeMs":null,"waitedCount":0,"waitedTimeMs":null,"lockName":null,"lockOwnerId":null,"lockOwnerName":null,"inNative":false,"suspended":false},{"id":24,"name":"Common-Cleaner","state":"TIMED_WAITING","daemon":true,"priority":8,"cpuPercent":0.0,"cpuTimeMs":2,"blockedCount":1,"blockedTimeMs":null,"waitedCount":11,"waitedTimeMs":null,"lockName":"java.lang.ref.ReferenceQueue$Lock@2ae76c3d","lockOwnerId":null,"lockOwnerName":null,"inNative":false,"suspended":false},{"id":25,"name":"Attach Listener","state":"RUNNABLE","daemon":true,"priority":9,"cpuPercent":0.0,"cpuTimeMs":121,"blockedCount":0,"blockedTimeMs":null,"waitedCount":0,"waitedTimeMs":null,"lockName":null,"lockOwnerId":null,"lockOwnerName":null,"inNative":false,"suspended":false},{"id":27,"name":"telltale-agent","state":"RUNNABLE","daemon":true,"priority":9,"cpuPercent":0.0,"cpuTimeMs":14,"blockedCount":0,"blockedTimeMs":null,"waitedCount":0,"waitedTimeMs":null,"lockName":null,"lockOwnerId":null,"lockOwnerName":null,"inNative":true,"suspended":false},{"id":84,"name":"telltale-diag-lease-reaper","state":"TIMED_WAITING","daemon":true,"priority":9,"cpuPercent":0.0,"cpuTimeMs":12,"blockedCount":0,"blockedTimeMs":null,"waitedCount":62,"waitedTimeMs":null,"lockName":null,"lockOwnerId":null,"lockOwnerName":null,"inNative":false,"suspended":false},{"id":154,"name":"telltale-session","state":"RUNNABLE","daemon":true,"priority":9,"cpuPercent":0.0,"cpuTimeMs":0,"blockedCount":0,"blockedTimeMs":null,"waitedCount":0,"waitedTimeMs":null,"lockName":null,"lockOwnerId":null,"lockOwnerName":null,"inNative":true,"suspended":false},{"id":155,"name":"telltale-operation","state":"RUNNABLE","daemon":true,"priority":9,"cpuPercent":0.1,"cpuTimeMs":0,"blockedCount":0,"blockedTimeMs":null,"waitedCount":1,"waitedTimeMs":null,"lockName":null,"lockOwnerId":null,"lockOwnerName":null,"inNative":false,"suspended":false}]} diff --git a/pkg/tracing/jvm/jvmruntime/testdata/jvm.threads.stack.json b/pkg/tracing/jvm/jvmruntime/testdata/jvm.threads.stack.json new file mode 100644 index 0000000..bbd880c --- /dev/null +++ b/pkg/tracing/jvm/jvmruntime/testdata/jvm.threads.stack.json @@ -0,0 +1 @@ +{"thread":{"id":3,"name":"main","state":"TIMED_WAITING","daemon":false,"priority":5,"cpuPercent":null,"cpuTimeMs":661,"blockedCount":0,"blockedTimeMs":null,"waitedCount":2952,"waitedTimeMs":null,"lockName":null,"lockOwnerId":null,"lockOwnerName":null,"inNative":false,"suspended":false},"text":"\"main\" #3 prio\u003d5\n java.lang.Thread.State: TIMED_WAITING\n\tat java.lang.Thread.sleepNanos0(Native Method)\n\tat java.lang.Thread.sleepNanos(Thread.java:509)\n\tat java.lang.Thread.sleep(Thread.java:540)\n\tat Idle.main(Unknown Source)\n\tat java.lang.invoke.LambdaForm$DMH/0x00000e000106c000.invokeStatic(LambdaForm$DMH)\n\tat java.lang.invoke.LambdaForm$MH/0x00000e000114d400.invoke(LambdaForm$MH)\n\tat java.lang.invoke.Invokers$Holder.invokeExact_MT(Invokers$Holder)\n\tat jdk.internal.reflect.DirectMethodHandleAccessor.invokeImpl(DirectMethodHandleAccessor.java:155)\n\tat jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)\n\tat java.lang.reflect.Method.invoke(Method.java:565)\n\tat com.sun.tools.javac.launcher.SourceLauncher.execute(SourceLauncher.java:258)\n\tat com.sun.tools.javac.launcher.SourceLauncher.run(SourceLauncher.java:138)\n\tat com.sun.tools.javac.launcher.SourceLauncher.main(SourceLauncher.java:76)\n\n Locked ownable synchronizers:\n\t- None\n","frames":[{"className":"java.lang.Thread","methodName":"sleepNanos0","fileName":"Thread.java","lineNumber":-2},{"className":"java.lang.Thread","methodName":"sleepNanos","fileName":"Thread.java","lineNumber":509},{"className":"java.lang.Thread","methodName":"sleep","fileName":"Thread.java","lineNumber":540},{"className":"Idle","methodName":"main","fileName":null,"lineNumber":21},{"className":"java.lang.invoke.LambdaForm$DMH/0x00000e000106c000","methodName":"invokeStatic","fileName":"LambdaForm$DMH","lineNumber":-1},{"className":"java.lang.invoke.LambdaForm$MH/0x00000e000114d400","methodName":"invoke","fileName":"LambdaForm$MH","lineNumber":-1},{"className":"java.lang.invoke.Invokers$Holder","methodName":"invokeExact_MT","fileName":"Invokers$Holder","lineNumber":-1},{"className":"jdk.internal.reflect.DirectMethodHandleAccessor","methodName":"invokeImpl","fileName":"DirectMethodHandleAccessor.java","lineNumber":155},{"className":"jdk.internal.reflect.DirectMethodHandleAccessor","methodName":"invoke","fileName":"DirectMethodHandleAccessor.java","lineNumber":104},{"className":"java.lang.reflect.Method","methodName":"invoke","fileName":"Method.java","lineNumber":565},{"className":"com.sun.tools.javac.launcher.SourceLauncher","methodName":"execute","fileName":"SourceLauncher.java","lineNumber":258},{"className":"com.sun.tools.javac.launcher.SourceLauncher","methodName":"run","fileName":"SourceLauncher.java","lineNumber":138},{"className":"com.sun.tools.javac.launcher.SourceLauncher","methodName":"main","fileName":"SourceLauncher.java","lineNumber":76}]} diff --git a/pkg/tracing/jvm/jvmruntime/threads.go b/pkg/tracing/jvm/jvmruntime/threads.go new file mode 100644 index 0000000..e3b0947 --- /dev/null +++ b/pkg/tracing/jvm/jvmruntime/threads.go @@ -0,0 +1,103 @@ +package jvmruntime + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/flanksource/clicky/api" +) + +// ThreadsOp samples every live thread's state and CPU over a short window. +const ThreadsOp = "jvm.threads" + +// ThreadSample is the agent's jvm.threads: every live thread, with CPU % over +// SampledMs when the JVM has thread CPU timing on. +type ThreadSample struct { + SampledMs int `json:"sampledMs"` + CPUTimeEnabled bool `json:"cpuTimeEnabled"` + Deadlocked []int64 `json:"deadlocked"` + Threads []Thread `json:"threads"` +} + +// Thread is one thread's row. The pointer fields are null when the JVM does +// not measure them (CPU timing or contention monitoring off), never 0. +type Thread struct { + ID int64 `json:"id"` + Name string `json:"name"` + State string `json:"state"` + Daemon bool `json:"daemon"` + Priority int `json:"priority"` + CPUPercent *float64 `json:"cpuPercent"` + CPUTimeMs *int64 `json:"cpuTimeMs"` + BlockedCount int64 `json:"blockedCount"` + BlockedTimeMs *int64 `json:"blockedTimeMs"` + WaitedCount int64 `json:"waitedCount"` + WaitedTimeMs *int64 `json:"waitedTimeMs"` + LockName *string `json:"lockName"` + LockOwnerID *int64 `json:"lockOwnerId"` + LockOwnerName *string `json:"lockOwnerName"` + InNative bool `json:"inNative"` + Suspended bool `json:"suspended"` +} + +// DecodeThreads decodes the agent's sample, refusing one with no threads: a +// live JVM always has at least the thread answering. +func DecodeThreads(raw json.RawMessage) (ThreadSample, error) { + sample, err := decodeObject[ThreadSample](ThreadsOp, raw, "sampledMs", "cpuTimeEnabled", "deadlocked", "threads") + if err == nil && (len(sample.Threads) == 0 || sample.SampledMs <= 0) { + err = fmt.Errorf("%s returned no thread sample: %s", ThreadsOp, raw) + } + return sample, err +} + +func stateClass(state string) string { + switch state { + case "RUNNABLE": + return "text-green-600" + case "BLOCKED": + return "text-red-600" + default: + return "text-muted" + } +} + +func (Thread) Columns() []api.ColumnDef { + return []api.ColumnDef{ + api.Column("id").Label("ID").Build(), + api.Column("name").Label("Name").MaxWidth(60).Build(), + api.Column("state").Label("State").Build(), + api.Column("cpu").Label("CPU").Build(), + api.Column("daemon").Label("Daemon").Build(), + } +} + +func (t Thread) Row() map[string]any { + cpu := "" + if t.CPUPercent != nil { + cpu = fmt.Sprintf("%.1f%%", *t.CPUPercent) + } + return map[string]any{ + "id": t.ID, + "name": t.Name, + "state": api.Text{}.Append(t.State, stateClass(t.State)), + "cpu": cpu, + "daemon": t.Daemon, + } +} + +// Pretty is the sample's heading, with any deadlock in red, over its Thread table. +func (s ThreadSample) Pretty() api.Text { + text := api.Text{}.Append("Threads", "font-bold").Append(fmt.Sprintf(" %d threads · sampled %dms", len(s.Threads), s.SampledMs), "text-muted") + if !s.CPUTimeEnabled { + text = text.Append(" · CPU time off", "text-yellow-600") + } + if len(s.Deadlocked) > 0 { + ids := make([]string, len(s.Deadlocked)) + for i, id := range s.Deadlocked { + ids[i] = fmt.Sprint(id) + } + text = text.Append(" · DEADLOCK threads "+strings.Join(ids, ", "), "text-red-600") + } + return text.NewLine().Add(api.NewTableFrom(s.Threads)) +} diff --git a/pkg/tracing/jvm/jvmruntime/threads_test.go b/pkg/tracing/jvm/jvmruntime/threads_test.go new file mode 100644 index 0000000..3710e29 --- /dev/null +++ b/pkg/tracing/jvm/jvmruntime/threads_test.go @@ -0,0 +1,158 @@ +package jvmruntime_test + +import ( + "encoding/json" + "strings" + + "github.com/flanksource/clicky/api" + "github.com/flanksource/telltale/pkg/tracing/jvm/jvmruntime" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const liveThreads = `{"sampledMs":100,"cpuTimeEnabled":true,"deadlocked":[],"threads":[ +{"id":3,"name":"main","state":"TIMED_WAITING","daemon":false,"priority":5,"cpuPercent":0.1,"cpuTimeMs":443,"blockedCount":0,"blockedTimeMs":null,"waitedCount":1282,"waitedTimeMs":null,"lockName":null,"lockOwnerId":null,"lockOwnerName":null,"inNative":false,"suspended":false}, +{"id":15,"name":"Finalizer","state":"WAITING","daemon":true,"priority":8,"cpuPercent":0.0,"cpuTimeMs":0,"blockedCount":0,"blockedTimeMs":null,"waitedCount":1,"waitedTimeMs":null,"lockName":"java.lang.ref.ReferenceQueue$Lock@7ce5d946","lockOwnerId":null,"lockOwnerName":null,"inNative":false,"suspended":false}, +{"id":27,"name":"telltale-agent","state":"RUNNABLE","daemon":true,"priority":9,"cpuPercent":0.0,"cpuTimeMs":3,"blockedCount":0,"blockedTimeMs":null,"waitedCount":0,"waitedTimeMs":null,"lockName":null,"lockOwnerId":null,"lockOwnerName":null,"inNative":true,"suspended":false}]}` + +const liveDump = "\"main\" #3 prio=5\n java.lang.Thread.State: TIMED_WAITING\n\tat java.lang.Thread.sleepNanos0(Native Method)\n\tat Idle.main(Unknown Source)\n\n Locked ownable synchronizers:\n\t- None\n\n" + + "\"Finalizer\" #15 daemon prio=8\n java.lang.Thread.State: WAITING\n\tat java.lang.Object.wait0(Native Method)\n\t- waiting on <0x7ce5d946> (a java.lang.ref.ReferenceQueue$Lock)\n\tat java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:165)\n\n Locked ownable synchronizers:\n\t- None\n\n" + + "\"Signal Dispatcher\" #16 daemon prio=9\n java.lang.Thread.State: RUNNABLE\n\n Locked ownable synchronizers:\n\t- None\n\n" + + "\"telltale-agent\" #27 daemon prio=9\n java.lang.Thread.State: RUNNABLE\n\tat sun.nio.ch.Net.accept(Native Method)\n\n Locked ownable synchronizers:\n\t- <0x3bd736a7> (a java.util.concurrent.locks.ReentrantLock$NonfairSync)\n\n" + +const deadlockTrailer = "Found 1 deadlock:\n\n\"worker-a\" Id=31:\n waiting for <0x1> (a java.lang.Object),\n which is held by \"worker-b\" Id=32\n" + +const liveStack = `{"thread":{"id":15,"name":"Finalizer","state":"WAITING","daemon":true,"priority":8,"cpuPercent":null,"cpuTimeMs":0,"blockedCount":0,"blockedTimeMs":null,"waitedCount":1,"waitedTimeMs":null,"lockName":"java.lang.ref.ReferenceQueue$Lock@7ce5d946","lockOwnerId":null,"lockOwnerName":null,"inNative":false,"suspended":false}, +"text":"\"Finalizer\" #15 daemon prio=8\n java.lang.Thread.State: WAITING\n\tat java.lang.Object.wait0(Native Method)\n\t- waiting on <0x7ce5d946> (a java.lang.ref.ReferenceQueue$Lock)\n\tat java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:165)\n\n Locked ownable synchronizers:\n\t- None\n", +"frames":[{"className":"java.lang.Object","methodName":"wait0","fileName":"Object.java","lineNumber":-2},{"className":"java.lang.ref.Finalizer$FinalizerThread","methodName":"run","fileName":"Finalizer.java","lineNumber":165}]}` + +func jsonString(text string) json.RawMessage { + raw, err := json.Marshal(text) + Expect(err).NotTo(HaveOccurred()) + return raw +} + +var _ = Describe("thread sample", func() { + It("decodes every thread row the agent sends", func() { + sample, err := jvmruntime.DecodeThreads(json.RawMessage(liveThreads)) + Expect(err).NotTo(HaveOccurred()) + Expect(sample.Threads[1]).To(Equal(jvmruntime.Thread{ + ID: 15, Name: "Finalizer", State: "WAITING", Daemon: true, Priority: 8, CPUPercent: new(0.0), CPUTimeMs: new(int64(0)), + WaitedCount: 1, LockName: new("java.lang.ref.ReferenceQueue$Lock@7ce5d946"), + })) + Expect(sample.SampledMs).To(Equal(100)) + Expect(sample.Deadlocked).To(Equal([]int64{})) + }) + + It("heads the table with the sample window", func() { + sample, err := jvmruntime.DecodeThreads(json.RawMessage(liveThreads)) + Expect(err).NotTo(HaveOccurred()) + Expect(lines(sample.Pretty())[0]).To(Equal("Threads 3 threads · sampled 100ms")) + }) + + It("flags deadlocked threads and disabled CPU timing in the heading", func() { + sample, err := jvmruntime.DecodeThreads(json.RawMessage(strings.Replace(strings.Replace(liveThreads, `"deadlocked":[]`, `"deadlocked":[3,15]`, 1), `"cpuTimeEnabled":true`, `"cpuTimeEnabled":false`, 1))) + Expect(err).NotTo(HaveOccurred()) + Expect(lines(sample.Pretty())[0]).To(Equal("Threads 3 threads · sampled 100ms · CPU time off · DEADLOCK threads 3, 15")) + }) + + It("tabulates each thread with a colored state", func() { + row := jvmruntime.Thread{ID: 31, Name: "worker-a", State: "BLOCKED", CPUPercent: new(12.5)} + Expect(row.Row()).To(Equal(map[string]any{ + "id": int64(31), "name": "worker-a", "state": api.Text{}.Append("BLOCKED", "text-red-600"), "cpu": "12.5%", "daemon": false, + })) + Expect(jvmruntime.Thread{State: "RUNNABLE"}.Row()["state"]).To(Equal(api.Text{}.Append("RUNNABLE", "text-green-600"))) + Expect(jvmruntime.Thread{State: "TIMED_WAITING"}.Row()["state"]).To(Equal(api.Text{}.Append("TIMED_WAITING", "text-muted"))) + Expect(jvmruntime.Thread{}.Row()["cpu"]).To(Equal("")) + }) + + It("refuses a result that is not a thread sample", func() { + for _, raw := range []string{`{}`, `{"sampledMs":100,"cpuTimeEnabled":true,"deadlocked":[],"threads":[]}`, `{"sampledMs":100,"cpuTimeEnabled":true,"threads":[{"id":1}]}`, `[]`} { + _, err := jvmruntime.DecodeThreads(json.RawMessage(raw)) + Expect(err).To(HaveOccurred(), raw) + } + }) +}) + +var _ = Describe("thread dump", func() { + It("parses the agent's jstack text into threads", func() { + dump, err := jvmruntime.DecodeThreadDump(jsonString(liveDump)) + Expect(err).NotTo(HaveOccurred()) + Expect(dump).To(Equal(jvmruntime.ThreadDump{Threads: []jvmruntime.DumpedThread{ + {Name: "main", ID: 3, Priority: 5, State: "TIMED_WAITING", Stack: []string{"at java.lang.Thread.sleepNanos0(Native Method)", "at Idle.main(Unknown Source)"}}, + {Name: "Finalizer", ID: 15, Daemon: true, Priority: 8, State: "WAITING", Stack: []string{ + "at java.lang.Object.wait0(Native Method)", "- waiting on <0x7ce5d946> (a java.lang.ref.ReferenceQueue$Lock)", "at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:165)", + }}, + {Name: "Signal Dispatcher", ID: 16, Daemon: true, Priority: 9, State: "RUNNABLE", Stack: []string{}}, + {Name: "telltale-agent", ID: 27, Daemon: true, Priority: 9, State: "RUNNABLE", Stack: []string{"at sun.nio.ch.Net.accept(Native Method)"}, + Synchronizers: []string{"<0x3bd736a7> (a java.util.concurrent.locks.ReentrantLock$NonfairSync)"}}, + }})) + }) + + It("renders each thread as a header over its indented frames", func() { + dump, err := jvmruntime.DecodeThreadDump(jsonString(liveDump)) + Expect(err).NotTo(HaveOccurred()) + Expect(lines(dump.Pretty())).To(Equal([]string{ + "Thread dump 4 threads", + "main #3 · prio 5 · TIMED_WAITING", + " at java.lang.Thread.sleepNanos0(Native Method)", + " at Idle.main(Unknown Source)", + "Finalizer #15 · daemon · prio 8 · WAITING", + " at java.lang.Object.wait0(Native Method)", + " - waiting on <0x7ce5d946> (a java.lang.ref.ReferenceQueue$Lock)", + " at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:165)", + "Signal Dispatcher #16 · daemon · prio 9 · RUNNABLE", + "telltale-agent #27 · daemon · prio 9 · RUNNABLE", + " at sun.nio.ch.Net.accept(Native Method)", + " locked synchronizers: <0x3bd736a7> (a java.util.concurrent.locks.ReentrantLock$NonfairSync)", + })) + }) + + It("keeps the deadlock report and counts it in the heading", func() { + dump, err := jvmruntime.DecodeThreadDump(jsonString(liveDump + deadlockTrailer)) + Expect(err).NotTo(HaveOccurred()) + Expect(dump.Deadlocks).To(Equal([]string{"Found 1 deadlock:", `"worker-a" Id=31:`, " waiting for <0x1> (a java.lang.Object),", ` which is held by "worker-b" Id=32`})) + rendered := lines(dump.Pretty()) + Expect(rendered[0]).To(Equal("Thread dump 4 threads · DEADLOCK")) + Expect(rendered[len(rendered)-4:]).To(Equal(dump.Deadlocks)) + }) + + It("refuses anything that is not jstack text", func() { + for _, raw := range []json.RawMessage{json.RawMessage(`{}`), jsonString(""), jsonString("not a dump\n"), jsonString("\"x\" #1 prio=5\n\tunexpected\n"), jsonString("\"x\" #1 prio=5\n")} { + _, err := jvmruntime.DecodeThreadDump(raw) + Expect(err).To(HaveOccurred(), string(raw)) + } + }) +}) + +var _ = Describe("thread stack", func() { + It("decodes the thread row, text and frames", func() { + stack, err := jvmruntime.DecodeThreadStack(json.RawMessage(liveStack)) + Expect(err).NotTo(HaveOccurred()) + Expect(stack.Thread.ID).To(Equal(int64(15))) + Expect(stack.Thread.CPUPercent).To(BeNil()) + Expect(stack.Frames).To(Equal([]jvmruntime.StackFrame{ + {ClassName: "java.lang.Object", MethodName: "wait0", FileName: new("Object.java"), LineNumber: -2}, + {ClassName: "java.lang.ref.Finalizer$FinalizerThread", MethodName: "run", FileName: new("Finalizer.java"), LineNumber: 165}, + })) + }) + + It("renders the thread header with its CPU time over its frames and locks", func() { + stack, err := jvmruntime.DecodeThreadStack(json.RawMessage(liveStack)) + Expect(err).NotTo(HaveOccurred()) + Expect(lines(stack.Pretty())).To(Equal([]string{ + "Finalizer #15 · daemon · prio 8 · WAITING · cpu 0ms", + " at java.lang.Object.wait0(Native Method)", + " - waiting on <0x7ce5d946> (a java.lang.ref.ReferenceQueue$Lock)", + " at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:165)", + })) + }) + + It("refuses a stack whose text is not the requested thread", func() { + other := strings.Replace(liveStack, `{"id":15,`, `{"id":16,`, 1) + for _, raw := range []string{`{}`, `{"thread":{"id":15},"frames":[]}`, other} { + _, err := jvmruntime.DecodeThreadStack(json.RawMessage(raw)) + Expect(err).To(HaveOccurred(), raw) + } + }) +}) diff --git a/pkg/tracing/jvm/results.go b/pkg/tracing/jvm/results.go new file mode 100644 index 0000000..c3abd47 --- /dev/null +++ b/pkg/tracing/jvm/results.go @@ -0,0 +1,83 @@ +package jvm + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/flanksource/clicky/api" + "github.com/flanksource/telltale/pkg/tracing/jvm/agentops" + "github.com/flanksource/telltale/pkg/tracing/jvm/classes" + "github.com/flanksource/telltale/pkg/tracing/jvm/jmx" + "github.com/flanksource/telltale/pkg/tracing/jvm/jvmruntime" +) + +// results decode the operations that have a typed model, so they print as +// records in any output format instead of raw JSON. +var results = mergeResults( + map[string]func(json.RawMessage) (any, error){ + HeapHistogramOp: func(raw json.RawMessage) (any, error) { return DecodeHeapHistogram(raw) }, + }, + jvmruntime.Results, classes.Results, jmx.Results, agentops.Results, +) + +func mergeResults(registries ...map[string]func(json.RawMessage) (any, error)) map[string]func(json.RawMessage) (any, error) { + merged := map[string]func(json.RawMessage) (any, error){} + for _, registry := range registries { + for operation, decode := range registry { + if _, taken := merged[operation]; taken { + panic(fmt.Sprintf("operation %s has two result models", operation)) + } + merged[operation] = decode + } + } + return merged +} + +// requested is a result that names what its request changed, for an agent +// that answers only that it applied it. +type requested interface { + Requested(params json.RawMessage) (any, error) +} + +// Result decodes an operation's result into its typed model, or keeps one +// without a model, such as an extension's, as the RawResult the agent sent. +// params are the request's, for a result that names what it changed. +func Result(operation string, params, raw json.RawMessage) (any, error) { + if decode, ok := results[operation]; ok { + result, err := decode(raw) + if err != nil { + return nil, err + } + if named, ok := result.(requested); ok { + return named.Requested(params) + } + return result, nil + } + if len(raw) > 0 && !json.Valid(raw) { + return nil, fmt.Errorf("%s returned a result that is not JSON: %.200s", operation, raw) + } + return RawResult(raw), nil +} + +// RawResult is a result without a typed model: --format json prints it as +// sent, and the terminal indented and highlighted. +type RawResult json.RawMessage + +func (r RawResult) MarshalJSON() ([]byte, error) { + if len(r) == 0 { + return []byte("null"), nil + } + return r, nil +} + +func (r RawResult) Pretty() api.Text { + if len(r) == 0 || string(r) == "null" { + return api.Text{}.Append("✓ done", "text-green-600").Append(" · no result", "text-muted") + } + var indented bytes.Buffer + if err := json.Indent(&indented, r, "", " "); err != nil { + return api.Text{}.Append(string(r)) + } + return api.Text{}.Add(api.CodeBlock("json", indented.String())) +} diff --git a/pkg/tracing/jvm/results_test.go b/pkg/tracing/jvm/results_test.go new file mode 100644 index 0000000..d9c0f05 --- /dev/null +++ b/pkg/tracing/jvm/results_test.go @@ -0,0 +1,65 @@ +package jvm_test + +import ( + "encoding/json" + + "github.com/flanksource/clicky/api" + catalog "github.com/flanksource/telltale/internal/jvm" + "github.com/flanksource/telltale/pkg/tracing/jvm" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("operation results", func() { + It("decodes an operation with a typed model into it, and refuses a malformed one", func() { + result, err := jvm.Result(jvm.HeapHistogramOp, nil, json.RawMessage(agentHistogram)) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(BeAssignableToTypeOf(jvm.HeapHistogram{})) + _, err = jvm.Result(jvm.HeapHistogramOp, nil, json.RawMessage(`{}`)) + Expect(err).To(HaveOccurred()) + }) + + It("models every built-in operation in the catalog, refusing an empty object instead of printing it raw", func() { + var raw []string + for _, operation := range catalog.Operations() { + if result, err := jvm.Result(operation.Name, nil, json.RawMessage(`{}`)); err == nil { + if _, isRaw := result.(jvm.RawResult); isRaw { + raw = append(raw, operation.Name) + } + } + } + Expect(catalog.Operations()).To(HaveLen(37)) + Expect(raw).To(BeEmpty()) + }) + + It("names what a write changed from its request", func() { + result, err := jvm.Result("jvm.logging.set", json.RawMessage(`{"name":"global","level":"FINE"}`), json.RawMessage(`{"applied":true}`)) + Expect(err).NotTo(HaveOccurred()) + Expect(result.(interface{ Pretty() api.Text }).Pretty().String()).To(Equal("✓ logger global → FINE")) + }) + + It("keeps an operation without a model as the agent sent it, indented for the terminal", func() { + raw := json.RawMessage(`{"id":"ext-1","operations":["acme.ping"]}`) + result, err := jvm.Result("acme.extension.ping", nil, raw) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(jvm.RawResult(raw))) + encoded, err := json.Marshal(result) + Expect(err).NotTo(HaveOccurred()) + Expect(encoded).To(MatchJSON(raw)) + Expect(result.(jvm.RawResult).Pretty().String()).To(Equal("{\n \"id\": \"ext-1\",\n \"operations\": [\n \"acme.ping\"\n ]\n}")) + }) + + It("reads an empty result as no result", func() { + result, err := jvm.Result("acme.extension.reset", nil, nil) + Expect(err).NotTo(HaveOccurred()) + encoded, err := json.Marshal(result) + Expect(err).NotTo(HaveOccurred()) + Expect(string(encoded)).To(Equal("null")) + Expect(result.(jvm.RawResult).Pretty().String()).To(Equal("✓ done · no result")) + }) + + It("refuses a result that is not JSON", func() { + _, err := jvm.Result("acme.extension.ping", nil, json.RawMessage(`{"id":`)) + Expect(err).To(MatchError(ContainSubstring("acme.extension.ping"))) + }) +}) diff --git a/pkg/tracing/jvmtrace/calls.go b/pkg/tracing/jvmtrace/calls.go new file mode 100644 index 0000000..f9b0435 --- /dev/null +++ b/pkg/tracing/jvmtrace/calls.go @@ -0,0 +1,234 @@ +package jvmtrace + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/flanksource/clicky/api" +) + +// Calls folds a stream's method events into calls, one line each: a call is +// complete at its exit, and an outermost call prints with the calls nested +// inside it, so a traced call reads as one tree. It keeps the run's stats. +type Calls struct { + // Location formats each call's start; the local zone when nil. + Location *time.Location + open map[frameKey]*call + stats Stats +} + +type frameKey struct{ thread, frame int64 } + +type call struct { + event Event + value json.RawMessage + err string + duration time.Duration + children []*call + nested bool +} + +// Stats are what the calls a run folded add up to. +type Stats struct { + Calls int + Threw int + Dropped int64 + Slowest string + Longest time.Duration +} + +// Add folds one event, returning the lines it completes. +func (c *Calls) Add(event Event) []api.Text { + if c.open == nil { + c.open = map[frameKey]*call{} + } + key := frameKey{event.ThreadID, event.Frame} + switch event.Kind { + case "dropped": + c.stats.Dropped += event.Dropped + return []api.Text{api.Text{}.Append(c.clock(event.Timestamp)+" ", "text-muted").Append(fmt.Sprintf("⚠ %d events dropped: the agent's buffer was full", event.Dropped), "text-yellow-600")} + case "enter": + opened := &call{event: event} + if parent := c.open[frameKey{event.ThreadID, event.Parent}]; event.Parent != 0 && parent != nil { + opened.nested = true + parent.children = append(parent.children, opened) + } + c.open[key] = opened + case "return": + if opened := c.open[key]; opened != nil { + opened.value = event.Value + } + case "throw": + if opened := c.open[key]; opened != nil { + opened.err = event.Error + c.stats.Threw++ + } + case "exit": + opened := c.open[key] + if opened == nil { + return nil + } + delete(c.open, key) + opened.duration = time.Duration(event.DurationNs) + if opened.nested { + return nil + } + c.stats.Calls++ + if opened.duration >= c.stats.Longest { + c.stats.Longest, c.stats.Slowest = opened.duration, opened.name() + } + return c.lines(opened, threadName(opened.event), "", "") + } + return nil +} + +// EventOf reads a stored jvm_trace row back into its event; a null column is unset. +func EventOf(row map[string]any) (Event, error) { + set := map[string]any{} + for name, value := range row { + if value != nil { + set[name] = value + } + } + data, err := json.Marshal(set) + if err != nil { + return Event{}, fmt.Errorf("encode jvm_trace row: %w", err) + } + var event Event + if err := json.Unmarshal(data, &event); err != nil { + return Event{}, fmt.Errorf("decode jvm_trace row: %w", err) + } + return event, nil +} + +// Stats are the calls folded so far. +func (c *Calls) Stats() Stats { return c.stats } + +// lines are a call's line and, below it, its nested calls' as a tree. A +// nested call's enter carries no thread name, so every line shows its root's. +func (c *Calls) lines(done *call, thread, branch, indent string) []api.Text { + line := api.Text{}.Append(c.clock(done.event.Timestamp)+" "+thread+" ", "text-muted").Append(branch, "text-muted").Append(done.name(), "font-bold") + if !done.nested { + line = line.Append("(" + strings.Join(values(done.event.Args), ", ") + ")") + } + if done.err != "" { + line = line.Append(" ✗ "+shortError(done.err), "text-red-600") + } else if len(done.value) > 0 { + line = line.Append(" → " + value(done.value)) + } + result := []api.Text{line.Append(" "+formatDuration(done.duration), "text-muted")} + for i, child := range done.children { + mark, next := "├─ ", "│ " + if i == len(done.children)-1 { + mark, next = "└─ ", " " + } + result = append(result, c.lines(child, thread, indent+mark, indent+next)...) + } + return result +} + +func threadName(event Event) string { + if event.Thread != "" { + return event.Thread + } + return fmt.Sprintf("thread-%d", event.ThreadID) +} + +func (c *Calls) clock(at time.Time) string { + location := c.Location + if location == nil { + location = time.Local + } + return at.In(location).Format("15:04:05.000") +} + +func (done *call) name() string { return shortType(done.event.Class) + "." + done.event.Method } + +// Pretty is the stats on one line: calls, failures, the slowest and any loss. +func (s Stats) Pretty() api.Text { + parts := []string{} + switch s.Calls { + case 0: + parts = append(parts, "no calls completed") + case 1: + parts = append(parts, "1 call") + default: + parts = append(parts, fmt.Sprintf("%d calls", s.Calls)) + } + if s.Threw > 0 { + parts = append(parts, fmt.Sprintf("%d threw", s.Threw)) + } + if s.Slowest != "" { + parts = append(parts, "slowest "+s.Slowest+" "+formatDuration(s.Longest)) + } + if s.Dropped > 0 { + parts = append(parts, fmt.Sprintf("%d events dropped", s.Dropped)) + } + return api.Text{}.Append(strings.Join(parts, " · ")) +} + +// node is one value of a call as the agent's object graph renders it. +type node struct { + Kind string `json:"kind"` + Type string `json:"type"` + Value string `json:"value"` + Children []any `json:"children"` +} + +func values(raw json.RawMessage) []string { + var nodes []json.RawMessage + if len(raw) == 0 || json.Unmarshal(raw, &nodes) != nil { + return nil + } + result := make([]string, len(nodes)) + for i, data := range nodes { + result[i] = value(data) + } + return result +} + +// value is a scalar as itself, a string quoted, a null as null, and anything +// else by its short type with its size or member count. +func value(raw json.RawMessage) string { + var v node + if err := json.Unmarshal(raw, &v); err != nil { + return string(raw) + } + switch { + case v.Kind == "scalar" && (v.Type == "null" || v.Type == ""): + if v.Value == "" { + return "null" + } + return v.Value + case v.Kind == "scalar" && v.Type == "java.lang.String": + return fmt.Sprintf("%q", v.Value) + case v.Kind == "scalar": + return v.Value + case v.Value != "": + return shortType(v.Type) + "{" + v.Value + "}" + } + return fmt.Sprintf("%s{%d}", shortType(v.Type), len(v.Children)) +} + +func shortType(name string) string { return name[strings.LastIndex(name, ".")+1:] } + +// shortError is an exception with its class's simple name. +func shortError(text string) string { + class, message, found := strings.Cut(text, ": ") + if !found || strings.Contains(class, " ") { + return text + } + return shortType(class) + ": " + message +} + +func formatDuration(d time.Duration) string { + switch { + case d < time.Millisecond: + return fmt.Sprintf("%dµs", (d+500*time.Nanosecond)/time.Microsecond) + case d < time.Second: + return fmt.Sprintf("%.1fms", float64(d)/float64(time.Millisecond)) + } + return fmt.Sprintf("%.2fs", d.Seconds()) +} diff --git a/pkg/tracing/jvmtrace/calls_test.go b/pkg/tracing/jvmtrace/calls_test.go new file mode 100644 index 0000000..9ff1358 --- /dev/null +++ b/pkg/tracing/jvmtrace/calls_test.go @@ -0,0 +1,101 @@ +package jvmtrace_test + +import ( + "encoding/json" + "time" + + "github.com/flanksource/telltale/pkg/tracing/jvmtrace" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("folding method events into calls", func() { + start := time.Date(2026, 11, 16, 0, 5, 9, 912_000_000, time.UTC) + at := func(offset time.Duration) time.Time { return start.Add(offset) } + scalar := func(kind, value string) json.RawMessage { + data, err := json.Marshal(map[string]string{"kind": "scalar", "type": kind, "value": value}) + Expect(err).NotTo(HaveOccurred()) + return data + } + lines := func(calls *jvmtrace.Calls, events ...jvmtrace.Event) []string { + var printed []string + for _, event := range events { + for _, line := range calls.Add(event) { + printed = append(printed, line.String()) + } + } + return printed + } + + It("prints a watched call as one line once it exits: time, thread, call with its arguments, value and duration", func() { + calls := &jvmtrace.Calls{Location: time.UTC} + Expect(lines(calls, + jvmtrace.Event{Kind: "enter", Timestamp: at(0), Thread: "main", ThreadID: 3, Frame: 1, Class: "example.Idle", Method: "work", Args: json.RawMessage(`[` + string(scalar("java.lang.Long", "100045")) + `,` + string(scalar("java.lang.String", "fast")) + `]`)}, + jvmtrace.Event{Kind: "return", Timestamp: at(time.Millisecond), ThreadID: 3, Frame: 1, Value: scalar("java.lang.Long", "450198")}, + jvmtrace.Event{Kind: "exit", Timestamp: at(2 * time.Millisecond), ThreadID: 3, Frame: 1, DurationNs: 1_996_875}, + )).To(Equal([]string{`00:05:09.912 main Idle.work(100045, "fast") → 450198 2.0ms`})) + Expect(calls.Stats().Pretty().String()).To(Equal("1 call · slowest Idle.work 2.0ms")) + }) + + It("prints a traced call with the calls nested inside it as a tree once the outermost call exits", func() { + calls := &jvmtrace.Calls{Location: time.UTC} + printed := lines(calls, + jvmtrace.Event{Kind: "enter", Timestamp: at(0), Thread: "main", ThreadID: 3, Frame: 1, Class: "Idle", Method: "work", Args: json.RawMessage(`[` + string(scalar("java.lang.Long", "100045")) + `]`)}, + jvmtrace.Event{Kind: "enter", Timestamp: at(0), ThreadID: 3, Depth: 1, Frame: 2, Parent: 1, Class: "Idle", Method: "step"}, + jvmtrace.Event{Kind: "exit", ThreadID: 3, Depth: 1, Frame: 2, DurationNs: 1_457_917}, + jvmtrace.Event{Kind: "enter", Timestamp: at(2 * time.Millisecond), ThreadID: 3, Depth: 1, Frame: 3, Parent: 1, Class: "Idle", Method: "step"}, + jvmtrace.Event{Kind: "enter", Timestamp: at(2 * time.Millisecond), ThreadID: 3, Depth: 2, Frame: 4, Parent: 3, Class: "Idle", Method: "check"}, + jvmtrace.Event{Kind: "throw", ThreadID: 3, Depth: 2, Frame: 4, Error: "java.lang.IllegalStateException: negative -1"}, + jvmtrace.Event{Kind: "exit", ThreadID: 3, Depth: 2, Frame: 4, DurationNs: 41_000}, + jvmtrace.Event{Kind: "exit", ThreadID: 3, Depth: 1, Frame: 3, DurationNs: 173_708}, + ) + Expect(printed).To(BeEmpty(), "nothing prints until the outermost call exits") + Expect(lines(calls, + jvmtrace.Event{Kind: "return", ThreadID: 3, Frame: 1, Value: scalar("java.lang.Long", "450198")}, + jvmtrace.Event{Kind: "exit", ThreadID: 3, Frame: 1, DurationNs: 1_996_875}, + )).To(Equal([]string{ + "00:05:09.912 main Idle.work(100045) → 450198 2.0ms", + "00:05:09.912 main ├─ Idle.step 1.5ms", + "00:05:09.914 main └─ Idle.step 174µs", + "00:05:09.914 main └─ Idle.check ✗ IllegalStateException: negative -1 41µs", + })) + Expect(calls.Stats().Pretty().String()).To(Equal("1 call · 1 threw · slowest Idle.work 2.0ms")) + }) + + It("shows an object value by its short type and size, and a null as null", func() { + calls := &jvmtrace.Calls{Location: time.UTC} + Expect(lines(calls, + jvmtrace.Event{Kind: "enter", Timestamp: at(0), Thread: "http-1", ThreadID: 9, Frame: 7, Class: "com.example.Orders", Method: "find", + Args: json.RawMessage(`[{"kind":"scalar","type":"null"},{"kind":"map","type":"java.util.HashMap","value":"size=3"}]`)}, + jvmtrace.Event{Kind: "return", ThreadID: 9, Frame: 7, Value: json.RawMessage(`{"kind":"object","type":"com.example.Order","children":[{"kind":"scalar"},{"kind":"scalar"}]}`)}, + jvmtrace.Event{Kind: "exit", ThreadID: 9, Frame: 7, DurationNs: 3_200_000_000}, + )).To(Equal([]string{"00:05:09.912 http-1 Orders.find(null, HashMap{size=3}) → Order{2} 3.20s"})) + }) + + It("reports events the agent dropped on their own line and in the stats", func() { + calls := &jvmtrace.Calls{Location: time.UTC} + Expect(lines(calls, jvmtrace.Event{Kind: "dropped", Timestamp: at(0), Dropped: 12})).To(Equal([]string{"00:05:09.912 ⚠ 12 events dropped: the agent's buffer was full"})) + Expect(calls.Stats().Pretty().String()).To(Equal("no calls completed · 12 events dropped")) + }) + + It("reads a stored row back into its event, with unset columns left empty", func() { + event, err := jvmtrace.EventOf(map[string]any{ + "kind": "return", "threadId": int64(3), "frame": int64(1), "timestamp": start, "class": nil, "args": nil, "durationNs": nil, + "value": map[string]any{"kind": "scalar", "type": "java.lang.Long", "value": "450198"}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(event).To(Equal(jvmtrace.Event{Kind: "return", ThreadID: 3, Frame: 1, Timestamp: start, Value: json.RawMessage(`{"kind":"scalar","type":"java.lang.Long","value":"450198"}`)})) + _, err = jvmtrace.EventOf(map[string]any{"kind": "enter", "frame": "one"}) + Expect(err).To(MatchError(ContainSubstring("frame"))) + }) + + It("keeps calls on different threads apart", func() { + calls := &jvmtrace.Calls{Location: time.UTC} + Expect(lines(calls, + jvmtrace.Event{Kind: "enter", Timestamp: at(0), Thread: "a", ThreadID: 1, Frame: 1, Class: "W", Method: "run"}, + jvmtrace.Event{Kind: "enter", Timestamp: at(0), Thread: "b", ThreadID: 2, Frame: 1, Class: "W", Method: "run"}, + jvmtrace.Event{Kind: "exit", ThreadID: 2, Frame: 1, DurationNs: 5_000}, + jvmtrace.Event{Kind: "exit", ThreadID: 1, Frame: 1, DurationNs: 9_000}, + )).To(Equal([]string{"00:05:09.912 b W.run() 5µs", "00:05:09.912 a W.run() 9µs"})) + }) +}) diff --git a/pkg/tracing/jvmtrace/checkpoint.go b/pkg/tracing/jvmtrace/checkpoint.go new file mode 100644 index 0000000..2054ef2 --- /dev/null +++ b/pkg/tracing/jvmtrace/checkpoint.go @@ -0,0 +1,20 @@ +package jvmtrace + +import ( + "context" + "encoding/json" + + "github.com/flanksource/telltale/pkg/tracing" +) + +type CheckpointParams struct { + Reason string `json:"reason,omitempty"` +} + +func checkpointParams(ctx context.Context) (json.RawMessage, error) { + var params CheckpointParams + if _, err := tracing.DecodeCheckpointParams(ctx, ¶ms); err != nil { + return nil, err + } + return json.Marshal(params) +} diff --git a/pkg/tracing/jvmtrace/diagnostics.go b/pkg/tracing/jvmtrace/diagnostics.go new file mode 100644 index 0000000..b2952d6 --- /dev/null +++ b/pkg/tracing/jvmtrace/diagnostics.go @@ -0,0 +1,142 @@ +package jvmtrace + +import ( + "encoding/json" + "fmt" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/seam" + "github.com/flanksource/telltale/pkg/tracing/target" +) + +const NotificationsKind = "jvm_notifications" +const ExpressionKind = "jvm_ognl" + +type NotificationOptions struct { + MBean string `json:"mbean" clicky:"required,title=MBean or ObjectName pattern"` + Types []string `json:"types,omitempty" clicky:"title=Notification type prefixes"` + Capacity int `json:"capacity,omitempty" clicky:"title=Buffer capacity"` +} + +func (o NotificationOptions) Validate() error { + if o.MBean == "" { + return fmt.Errorf("mbean is required") + } + if o.Capacity < 0 || o.Capacity > 50000 { + return fmt.Errorf("capacity must be between one and 50000, or zero for the default") + } + return nil +} + +type NotificationParams struct { + Target target.Target `json:"target,omitzero"` + NotificationOptions +} + +func (p NotificationParams) Validate() error { return p.NotificationOptions.Validate() } + +type Notification struct { + Timestamp time.Time `json:"timestamp"` + TimeStamp int64 `json:"timeStamp"` + Type string `json:"type"` + SequenceNumber int64 `json:"sequenceNumber,omitempty"` + Message string `json:"message,omitempty"` + Source string `json:"source,omitempty"` + UserData json.RawMessage `json:"userData,omitempty"` + GC json.RawMessage `json:"gc,omitempty"` + GCError string `json:"gcError,omitempty"` + Dropped int64 `json:"dropped,omitempty"` + Resource map[string]string `json:"resource,omitempty"` +} +type ExpressionOptions struct { + Expression string `json:"expression" clicky:"required,title=OGNL expression,type=code"` + Loader string `json:"loader,omitempty" clicky:"title=Application loader hash"` + IntervalMs int `json:"intervalMs,omitempty" clicky:"title=Interval in milliseconds"` +} + +func (o ExpressionOptions) Validate() error { + if o.Expression == "" { + return fmt.Errorf("expression is required") + } + if o.IntervalMs != 0 && (o.IntervalMs < 100 || o.IntervalMs > 3600000) { + return fmt.Errorf("intervalMs must be between 100 and 3600000") + } + return nil +} + +type ExpressionParams struct { + Target target.Target `json:"target,omitzero"` + ExpressionOptions +} + +func (p ExpressionParams) Validate() error { return p.ExpressionOptions.Validate() } + +type Expression struct { + Timestamp time.Time `json:"timestamp"` + Expression string `json:"expression"` + Result json.RawMessage `json:"result"` + Resource map[string]string `json:"resource,omitempty"` +} + +type diagnostic[P, R any] struct { + agent seam.AgentSeam[P, R] + defaults P + schema recordresults.ResultType[R] +} + +func (h diagnostic[P, R]) Params() P { return h.defaults } +func (h diagnostic[P, R]) Schema() recordresults.ResultType[R] { return h.schema } +func (h diagnostic[P, R]) AgentAvailable(probe string, t target.Target) (bool, error) { + return h.agent.Available(probe, t) +} +func (h diagnostic[P, R]) Prepare(ctx dbcontext.Context, p P, out tracing.Emitter[R]) (dbcontext.Context, func() error, error) { + return h.agent.Prepare(ctx, p, out) +} +func (h diagnostic[P, R]) Handle(ctx dbcontext.Context, p P, out tracing.Emitter[R], _ tracing.Records[R]) error { + return h.agent.Handle(ctx, p, out) +} +func (h diagnostic[P, R]) Checkpoint(ctx dbcontext.Context, _ P, out tracing.Emitter[R]) error { + raw, err := checkpointParams(ctx) + if err != nil { + return err + } + return h.agent.Checkpoint(ctx, raw, out) +} + +func NewNotificationKind(registry *agentclient.Registry) *tracing.Handler[NotificationParams, Notification] { + agent := seam.AgentSeam[NotificationParams, Notification]{Probe: NotificationsKind, Registry: registry, Target: func(p NotificationParams) target.Target { return p.Target }, Mutates: func(NotificationParams) bool { return false }, ArmParams: func(p NotificationParams) (json.RawMessage, error) { return json.Marshal(p.NotificationOptions) }, DecodeBlob: func(b agentclient.Blob, resource map[string]string) (seam.DecodedBlob[Notification], error) { + if b.Info.ContentType != "application/json" { + return seam.DecodedBlob[Notification]{}, fmt.Errorf("notifications require application/json") + } + var rows []Notification + if err := json.Unmarshal(b.Data, &rows); err != nil { + return seam.DecodedBlob[Notification]{}, err + } + for i := range rows { + rows[i].Timestamp = time.UnixMilli(rows[i].TimeStamp).UTC() + rows[i].Resource = resource + } + return seam.DecodedBlob[Notification]{Rows: rows}, nil + }} + return tracing.NewHandler[NotificationParams, Notification](diagnostic[NotificationParams, Notification]{agent: agent, defaults: NotificationParams{NotificationOptions: NotificationOptions{MBean: "java.lang:type=GarbageCollector,*", Capacity: 500}}, schema: recordresults.ResultType[Notification]{Title: "JMX notifications and GC", TimeColumn: "timestamp", Follow: true, SearchColumns: []string{"type", "source", "message"}}}, tracing.Capabilities{Live: true}, target.Agent) +} +func NewExpressionKind(registry *agentclient.Registry) *tracing.Handler[ExpressionParams, Expression] { + agent := seam.AgentSeam[ExpressionParams, Expression]{Probe: ExpressionKind, Registry: registry, Target: func(p ExpressionParams) target.Target { return p.Target }, Mutates: func(ExpressionParams) bool { return true }, ArmParams: func(p ExpressionParams) (json.RawMessage, error) { return json.Marshal(p.ExpressionOptions) }, DecodeBlob: func(b agentclient.Blob, resource map[string]string) (seam.DecodedBlob[Expression], error) { + if b.Info.ContentType != "application/json" { + return seam.DecodedBlob[Expression]{}, fmt.Errorf("expressions require application/json") + } + var rows []Expression + if err := json.Unmarshal(b.Data, &rows); err != nil { + return seam.DecodedBlob[Expression]{}, err + } + for i := range rows { + rows[i].Resource = resource + } + return seam.DecodedBlob[Expression]{Rows: rows}, nil + }} + return tracing.NewHandler[ExpressionParams, Expression](diagnostic[ExpressionParams, Expression]{agent: agent, defaults: ExpressionParams{ExpressionOptions: ExpressionOptions{IntervalMs: 1000}}, schema: recordresults.ResultType[Expression]{Title: "OGNL samples", TimeColumn: "timestamp", Follow: true, SearchColumns: []string{"expression"}}}, tracing.Capabilities{Live: true}, target.Agent) +} diff --git a/pkg/tracing/jvmtrace/jvmtrace.go b/pkg/tracing/jvmtrace/jvmtrace.go new file mode 100644 index 0000000..1983b4a --- /dev/null +++ b/pkg/tracing/jvmtrace/jvmtrace.go @@ -0,0 +1,231 @@ +package jvmtrace + +import ( + "encoding/json" + "fmt" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/seam" + "github.com/flanksource/telltale/pkg/tracing/target" + "github.com/flanksource/telltale/sdk/protocol" +) + +const Kind = "jvm_trace" + +type MethodTarget struct { + Class string `json:"class" clicky:"required,title=Class"` + Method string `json:"method" clicky:"required,title=Method"` + Loader string `json:"loader,omitempty" clicky:"title=Class loader"` +} + +type Options struct { + Targets []MethodTarget `json:"targets" clicky:"required,title=Methods"` + Args bool `json:"args,omitempty" clicky:"title=Arguments"` + Return bool `json:"return,omitempty" clicky:"title=Return values"` + Exception bool `json:"exception,omitempty" clicky:"title=Exceptions"` + Calls bool `json:"calls,omitempty" clicky:"title=Nested calls"` + Platform bool `json:"platform,omitempty" clicky:"title=Platform calls"` + Events int `json:"events,omitempty" clicky:"title=Event limit"` + Expand int `json:"expand,omitempty" clicky:"title=Object graph depth"` + Chars int `json:"arg-chars,omitempty" clicky:"title=String character limit"` + GC bool `json:"gc,omitempty" clicky:"title=GC counters"` + Alloc bool `json:"alloc,omitempty" clicky:"title=Allocation counters"` + // Root-call filters: the agent holds each outermost call on its thread and + // publishes it, with everything nested in it, only when it qualifies. + MinCost string `json:"minCost,omitempty" clicky:"title=Minimum cost,desc=Keep only root calls that ran at least this long such as 250ms"` + ExceptionsOnly bool `json:"exceptionsOnly,omitempty" clicky:"title=Exceptions only,desc=Keep only root calls that threw"` + Match []ValueMatch `json:"match,omitempty" clicky:"title=Value matches,desc=Keep only root calls whose rendered argument or return value matches every predicate"` +} + +// ValueMatch tests one rendered value of a root call: argument Index, or the +// return value, optionally narrowed by a dotted Path of child labels. +type ValueMatch struct { + On MatchOn `json:"on" clicky:"required,title=Subject"` + Index int `json:"index" clicky:"title=Argument index"` + Path string `json:"path,omitempty" clicky:"title=Path,desc=Dotted child labels inside the value such as customer.name"` + Op MatchOp `json:"op" clicky:"required,title=Operator,desc=regex is a Java regular expression found anywhere in the value"` + Value string `json:"value" clicky:"required,title=Value"` +} + +// MatchOn is which value of a root call a ValueMatch tests. +type MatchOn string + +func (MatchOn) JSONSchema() map[string]any { + return map[string]any{"type": "string", "enum": []any{"arg", "return"}} +} + +// MatchOp is how a ValueMatch compares the rendered value. +type MatchOp string + +func (MatchOp) JSONSchema() map[string]any { + return map[string]any{"type": "string", "enum": []any{"equals", "contains", "regex"}} +} + +// FilterProbeAPI is the jvm_trace probe generation that implements MinCost, +// ExceptionsOnly and Match. +const FilterProbeAPI = 10 + +func (o Options) filtered() bool { + return o.MinCost != "" || o.ExceptionsOnly || len(o.Match) > 0 +} + +// CompatibleWith refuses filters an agent advertising spec cannot honour. An +// older agent ignores unknown options, so it would return every call as though +// each had matched. +func (o Options) CompatibleWith(spec protocol.ProbeSpec) error { + if o.filtered() && spec.Version < FilterProbeAPI { + return fmt.Errorf("minCost, exceptionsOnly and match need jvm_trace probe API v%d, and the agent implements v%d; its agent jar cannot be replaced while the JVM runs, so restart the JVM to filter calls, or trace without filters", FilterProbeAPI, spec.Version) + } + return nil +} + +func (o Options) validateFilters() error { + if o.MinCost != "" { + cost, err := time.ParseDuration(o.MinCost) + if err != nil || cost < 0 { + return fmt.Errorf("minCost must be a non-negative duration such as 250ms, got %q", o.MinCost) + } + } + for _, m := range o.Match { + if err := m.validate(o); err != nil { + return err + } + } + return nil +} + +func (m ValueMatch) validate(o Options) error { + switch m.Op { + case "equals", "contains", "regex": + default: + return fmt.Errorf("match op must be equals, contains or regex, got %q", m.Op) + } + switch m.On { + case "arg": + if !o.Args { + return fmt.Errorf("a match on arg needs args, or the agent never renders the value it would test") + } + if m.Index < 0 { + return fmt.Errorf("match index must be a non-negative argument position, got %d", m.Index) + } + case "return": + if !o.Return { + return fmt.Errorf("a match on return needs return, or the agent never renders the value it would test") + } + if m.Index != 0 { + return fmt.Errorf("a match on return takes no index, got %d", m.Index) + } + default: + return fmt.Errorf("match on must be arg or return, got %q", m.On) + } + return nil +} + +func (o Options) Validate() error { + if err := o.validateFilters(); err != nil { + return err + } + if len(o.Targets) == 0 || len(o.Targets) > 8 { + return fmt.Errorf("targets must contain one to eight methods") + } + for _, t := range o.Targets { + if t.Class == "" || t.Method == "" { + return fmt.Errorf("each target requires class and method") + } + } + if o.Events < 0 || o.Expand < 0 || o.Expand > 10 || o.Chars < 0 { + return fmt.Errorf("event and character limits must be nonnegative; graph depth must be between zero and ten") + } + if o.Platform && !o.Calls { + return fmt.Errorf("platform requires calls") + } + return nil +} + +type Params struct { + Target target.Target `json:"target,omitzero"` + Options +} + +func (p Params) Validate() error { return p.Options.Validate() } + +type Event struct { + Timestamp time.Time `json:"timestamp"` + TS int64 `json:"ts"` + ProbeSeq int64 `json:"probeSeq"` + Kind string `json:"kind"` + Key string `json:"key"` + ThreadID int64 `json:"threadId"` + Thread string `json:"thread,omitempty"` + Depth int `json:"depth"` + Frame int64 `json:"frame"` + Parent int64 `json:"parent,omitempty"` + Class string `json:"class,omitempty"` + Method string `json:"method,omitempty"` + Descriptor string `json:"descriptor,omitempty"` + Line int `json:"line,omitempty"` + Args json.RawMessage `json:"args,omitempty"` + Value json.RawMessage `json:"value,omitempty"` + Error string `json:"error,omitempty"` + DurationNs int64 `json:"durationNs,omitempty"` + AllocBytes *int64 `json:"allocBytes,omitempty"` + GCCount *int64 `json:"gcCount,omitempty"` + GCPauseMs *int64 `json:"gcPauseMs,omitempty"` + Unwound int `json:"unwound,omitempty"` + Truncated bool `json:"truncated,omitempty"` + Omitted int `json:"omitted,omitempty"` + Dropped int64 `json:"dropped,omitempty"` + Resource map[string]string `json:"resource,omitempty"` +} + +func Decode(blob agentclient.Blob, resource map[string]string) ([]Event, error) { + if blob.Info.ContentType != "application/json" { + return nil, fmt.Errorf("method events require application/json, received %q", blob.Info.ContentType) + } + var events []Event + if err := json.Unmarshal(blob.Data, &events); err != nil { + return nil, fmt.Errorf("decode method events: %w", err) + } + for i := range events { + events[i].Timestamp = time.Unix(0, events[i].TS).UTC() + events[i].Resource = resource + } + return events, nil +} + +type handler struct{ agent seam.AgentSeam[Params, Event] } + +func (handler) Params() Params { + return Params{Options: Options{Args: true, Return: true, Exception: true, Events: 1000, Expand: 3}} +} +func (handler) Schema() recordresults.ResultType[Event] { + return recordresults.ResultType[Event]{Title: "JVM method events", TimeColumn: "timestamp", Follow: true, SearchColumns: []string{"class", "method", "thread", "error"}} +} +func (h handler) AgentAvailable(probe string, t target.Target) (bool, error) { + return h.agent.Available(probe, t) +} +func (h handler) Prepare(ctx dbcontext.Context, p Params, out tracing.Emitter[Event]) (dbcontext.Context, func() error, error) { + return h.agent.Prepare(ctx, p, out) +} +func (h handler) Handle(ctx dbcontext.Context, p Params, out tracing.Emitter[Event], _ tracing.Records[Event]) error { + return h.agent.Handle(ctx, p, out) +} +func (h handler) Checkpoint(ctx dbcontext.Context, _ Params, out tracing.Emitter[Event]) error { + raw, err := checkpointParams(ctx) + if err != nil { + return err + } + return h.agent.Checkpoint(ctx, raw, out) +} + +func NewKind(registry *agentclient.Registry) *tracing.Handler[Params, Event] { + agent := seam.AgentSeam[Params, Event]{Probe: Kind, Registry: registry, Target: func(p Params) target.Target { return p.Target }, Mutates: func(Params) bool { return true }, ArmParams: func(p Params) (json.RawMessage, error) { return json.Marshal(p.Options) }, Compatible: func(p Params, spec protocol.ProbeSpec) error { return p.CompatibleWith(spec) }, DecodeBlob: func(blob agentclient.Blob, resource map[string]string) (seam.DecodedBlob[Event], error) { + rows, err := Decode(blob, resource) + return seam.DecodedBlob[Event]{Rows: rows}, err + }} + return tracing.NewHandler[Params, Event](handler{agent: agent}, tracing.Capabilities{Live: true}, target.Agent) +} diff --git a/pkg/tracing/jvmtrace/jvmtrace_test.go b/pkg/tracing/jvmtrace/jvmtrace_test.go new file mode 100644 index 0000000..c9acf7b --- /dev/null +++ b/pkg/tracing/jvmtrace/jvmtrace_test.go @@ -0,0 +1,89 @@ +package jvmtrace_test + +import ( + "encoding/json" + "testing" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/tracing/jvmtrace" + "github.com/flanksource/telltale/sdk/protocol" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestJVMTrace(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "JVM method traces") } + +var worker = []jvmtrace.MethodTarget{{Class: "example.Worker", Method: "compute"}} + +var _ = Describe("method trace contracts", func() { + It("rejects an empty target list and invalid value limits", func() { + Expect((jvmtrace.Options{}).Validate()).To(HaveOccurred()) + options := jvmtrace.Options{Targets: worker, Expand: 11} + Expect(options.Validate()).To(HaveOccurred()) + options.Expand = 3 + Expect(options.Validate()).To(Succeed()) + }) + It("decodes exact frame identities and object graphs from the agent", func() { + blob := agentclient.Blob{Data: []byte(`[{"ts":1794556800000000000,"probeSeq":17,"kind":"enter","key":"capture","threadId":12,"frame":23,"parent":19,"class":"example.Worker","method":"compute","descriptor":"(I)I","args":[{"kind":"scalar","value":"7"}]}]`)} + blob.Info.ContentType = "application/json" + events, err := jvmtrace.Decode(blob, map[string]string{"service.name": "example"}) + Expect(err).NotTo(HaveOccurred()) + Expect(events).To(HaveLen(1)) + Expect(events[0].Frame).To(Equal(int64(23))) + Expect(events[0].ProbeSeq).To(Equal(int64(17))) + Expect(events[0].Parent).To(Equal(int64(19))) + Expect(events[0].Args).To(MatchJSON(json.RawMessage(`[{"kind":"scalar","value":"7"}]`))) + Expect(events[0].Resource).To(HaveKeyWithValue("service.name", "example")) + }) +}) + +var _ = Describe("root-call filters", func() { + DescribeTable("refuses a filter the agent could not evaluate", + func(options jvmtrace.Options, reason string) { + options.Targets = worker + Expect(options.Validate()).To(MatchError(ContainSubstring(reason))) + }, + Entry("an unparseable minCost", jvmtrace.Options{MinCost: "fast"}, "minCost"), + Entry("a negative minCost", jvmtrace.Options{MinCost: "-1s"}, "minCost"), + Entry("an unknown operator", jvmtrace.Options{Args: true, Match: []jvmtrace.ValueMatch{{On: "arg", Op: "startsWith", Value: "x"}}}, "startsWith"), + Entry("an unknown subject", jvmtrace.Options{Args: true, Match: []jvmtrace.ValueMatch{{On: "this", Op: "equals", Value: "x"}}}, "arg or return"), + Entry("an argument that is not recorded", jvmtrace.Options{Match: []jvmtrace.ValueMatch{{On: "arg", Op: "equals", Value: "x"}}}, "args"), + Entry("a return value that is not recorded", jvmtrace.Options{Match: []jvmtrace.ValueMatch{{On: "return", Op: "equals", Value: "x"}}}, "return"), + Entry("an index on a return value", jvmtrace.Options{Return: true, Match: []jvmtrace.ValueMatch{{On: "return", Index: 1, Op: "equals", Value: "x"}}}, "index"), + Entry("a negative argument index", jvmtrace.Options{Args: true, Match: []jvmtrace.ValueMatch{{On: "arg", Index: -1, Op: "equals", Value: "x"}}}, "index"), + ) + + It("accepts a filter whose every predicate names a recorded value", func() { + options := jvmtrace.Options{Targets: worker, Args: true, Return: true, MinCost: "250ms", ExceptionsOnly: true, Match: []jvmtrace.ValueMatch{ + {On: "arg", Index: 1, Path: "customer.name", Op: "regex", Value: "^acme-[0-9]+$"}, + {On: "return", Op: "contains", Value: "sent"}, + }} + Expect(options.Validate()).To(Succeed()) + wire, err := json.Marshal(options) + Expect(err).NotTo(HaveOccurred()) + Expect(wire).To(MatchJSON(`{"targets":[{"class":"example.Worker","method":"compute"}],"args":true,"return":true,"minCost":"250ms","exceptionsOnly":true,"match":[{"on":"arg","index":1,"path":"customer.name","op":"regex","value":"^acme-[0-9]+$"},{"on":"return","index":0,"op":"contains","value":"sent"}]}`)) + }) + + DescribeTable("checks the agent's probe API before arming", + func(options jvmtrace.Options, version int, refused bool) { + err := options.CompatibleWith(protocol.ProbeSpec{Name: jvmtrace.Kind, Version: version}) + if refused { + Expect(err).To(MatchError(And(ContainSubstring("v10"), ContainSubstring("restart the JVM")))) + } else { + Expect(err).NotTo(HaveOccurred()) + } + }, + Entry("filters against an agent that predates them", jvmtrace.Options{MinCost: "1ms"}, 9, true), + Entry("filters against an agent that reports no version", jvmtrace.Options{ExceptionsOnly: true}, 0, true), + Entry("filters against a current agent", jvmtrace.Options{MinCost: "1ms"}, 10, false), + Entry("no filters against an old agent", jvmtrace.Options{Args: true}, 0, false), + ) + + It("decodes how many events a kept call omitted", func() { + blob := agentclient.Blob{Data: []byte(`[{"ts":1,"kind":"throw","key":"k","threadId":1,"frame":1,"omitted":7}]`)} + blob.Info.ContentType = "application/json" + events, err := jvmtrace.Decode(blob, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(events[0].Omitted).To(Equal(7)) + }) +}) diff --git a/pkg/tracing/jvmtrace/redefine.go b/pkg/tracing/jvmtrace/redefine.go new file mode 100644 index 0000000..be9bce2 --- /dev/null +++ b/pkg/tracing/jvmtrace/redefine.go @@ -0,0 +1,80 @@ +package jvmtrace + +import ( + "encoding/json" + "fmt" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/seam" + "github.com/flanksource/telltale/pkg/tracing/target" +) + +const RedefineKind = "jvm_redefine" + +type RedefineOptions struct { + Class string `json:"class" clicky:"required,title=Class"` + Method string `json:"method,omitempty" clicky:"title=Method,desc=Empty replaces the whole class"` + Loader string `json:"loader,omitempty" clicky:"title=Class loader"` + Source string `json:"source" clicky:"required,title=Source,type=code"` + Classpath string `json:"classpath,omitempty" clicky:"title=Compile classpath"` +} + +func (o RedefineOptions) Validate() error { + if o.Class == "" || o.Source == "" { + return fmt.Errorf("class and replacement source are required") + } + return nil +} + +type RedefineParams struct { + Target target.Target `json:"target,omitzero"` + RedefineOptions +} + +func (p RedefineParams) Validate() error { return p.RedefineOptions.Validate() } + +type Patch struct { + Timestamp time.Time `json:"timestamp"` + Class string `json:"class"` + Loader string `json:"loader"` + Applied bool `json:"applied"` + Restored bool `json:"restored"` + Digest string `json:"digest,omitempty"` + OriginalDigest string `json:"originalDigest,omitempty"` +} +type redefine struct { + agent seam.AgentSeam[RedefineParams, Patch] +} + +func (redefine) Params() RedefineParams { return RedefineParams{} } +func (redefine) Schema() recordresults.ResultType[Patch] { + return recordresults.ResultType[Patch]{Title: "JVM redefinitions", TimeColumn: "timestamp", Follow: true} +} +func (h redefine) AgentAvailable(probe string, t target.Target) (bool, error) { + return h.agent.Available(probe, t) +} +func (h redefine) Prepare(ctx dbcontext.Context, p RedefineParams, out tracing.Emitter[Patch]) (dbcontext.Context, func() error, error) { + return h.agent.Prepare(ctx, p, out) +} +func (h redefine) Handle(ctx dbcontext.Context, p RedefineParams, out tracing.Emitter[Patch], _ tracing.Records[Patch]) error { + return h.agent.Handle(ctx, p, out) +} +func (h redefine) Checkpoint(ctx dbcontext.Context, _ RedefineParams, out tracing.Emitter[Patch]) error { + raw, err := checkpointParams(ctx) + if err != nil { + return err + } + return h.agent.Checkpoint(ctx, raw, out) +} +func NewRedefineKind(registry *agentclient.Registry) *tracing.Handler[RedefineParams, Patch] { + agent := seam.AgentSeam[RedefineParams, Patch]{Probe: RedefineKind, Registry: registry, Target: func(p RedefineParams) target.Target { return p.Target }, Mutates: func(RedefineParams) bool { return true }, ArmParams: func(p RedefineParams) (json.RawMessage, error) { return json.Marshal(p.RedefineOptions) }, DecodeBlob: func(b agentclient.Blob, r map[string]string) (seam.DecodedBlob[Patch], error) { + var rows []Patch + err := json.Unmarshal(b.Data, &rows) + return seam.DecodedBlob[Patch]{Rows: rows}, err + }} + return tracing.NewHandler[RedefineParams, Patch](redefine{agent: agent}, tracing.Capabilities{Live: true}, target.Agent) +} diff --git a/pkg/tracing/kinds.go b/pkg/tracing/kinds.go index 5f791e2..ba411aa 100644 --- a/pkg/tracing/kinds.go +++ b/pkg/tracing/kinds.go @@ -13,15 +13,23 @@ import ( "github.com/flanksource/commons-db/query" "github.com/flanksource/commons-db/recordstore" "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/telltale/pkg/otlp/profilepublication" ) // KindInfo describes one registered kind. +type KindResult struct { + Name string `json:"name"` + Columns []query.ColumnDef `json:"columns"` +} + type KindInfo struct { + Top *TopSpec `json:"top,omitempty"` Name string `json:"name"` Title string `json:"title"` Capabilities Capabilities `json:"capabilities"` Params *rpc.OpenAPISchema `json:"params"` Columns []query.ColumnDef `json:"columns"` + Results []KindResult `json:"results,omitempty"` // ExportFormats are the formats a window of the kind's records downloads // in, NDJSON first. ExportFormats []ExportFormat `json:"exportFormats"` @@ -31,10 +39,11 @@ type KindInfo struct { type Kinds struct { mu sync.RWMutex plugins map[string]TracePlugin + results map[string]TracePlugin } func NewKinds() *Kinds { - return &Kinds{plugins: map[string]TracePlugin{}} + return &Kinds{plugins: map[string]TracePlugin{}, results: map[string]TracePlugin{}} } // RegisterKind serves plugin as kind name, refusing a plugin whose params or @@ -55,18 +64,58 @@ func (k *Kinds) RegisterKind(name string, plugin TracePlugin) error { if _, err := plugin.Params(); err != nil { return fmt.Errorf("trace kind %q: %w", name, err) } - if _, err := plugin.Schema(name); err != nil { + schema, err := plugin.Schema(name) + if err != nil { return fmt.Errorf("trace kind %q: %w", name, err) } + if top := TopOf(plugin); top != nil { + if top.Result != "" { + found := false + for _, result := range plugin.ResultTables() { + if result.Name == top.Result { + schema, err = result.plugin.Schema(name + "_" + top.Result) + found = true + break + } + } + if err != nil { + return err + } + if !found { + return fmt.Errorf("Top result %q is not registered", top.Result) + } + } + if err := top.validate(schema); err != nil { + return fmt.Errorf("trace kind %q: %w", name, err) + } + } if err := validateFormats(plugin.ExportFormats()); err != nil { return fmt.Errorf("trace kind %q: %w", name, err) } k.mu.Lock() defer k.mu.Unlock() - if _, ok := k.plugins[name]; ok { + if k.plugins[name] != nil || k.results[name] != nil { return fmt.Errorf("trace kind %q is already registered", name) } + seen := map[string]bool{name: true} + for _, table := range plugin.ResultTables() { + if err := recordstore.ValidateKind(table.Name); err != nil { + return err + } + storage := name + "_" + table.Name + if seen[storage] || k.results[storage] != nil || k.plugins[storage] != nil { + return fmt.Errorf("result table %s already registered", storage) + } + seen[storage] = true + if _, err := table.plugin.Schema(storage); err != nil { + return err + } + } plugin.setKind(name) + for _, table := range plugin.ResultTables() { + table.plugin.setKind(name + "_" + table.Name) + k.results[name+"_"+table.Name] = table.plugin + } k.plugins[name] = plugin return nil } @@ -76,6 +125,9 @@ func (k *Kinds) Get(name string) (TracePlugin, bool) { k.mu.RLock() defer k.mu.RUnlock() plugin, ok := k.plugins[name] + if !ok { + plugin, ok = k.results[name] + } return plugin, ok } @@ -95,9 +147,17 @@ func (k *Kinds) List() ([]KindInfo, error) { return nil, fmt.Errorf("trace kind %q: %w", name, err) } infos = append(infos, KindInfo{ + Top: TopOf(plugin), Name: name, Title: plugin.Title(), Capabilities: plugin.Capabilities(), Params: params, Columns: schema.Columns, ExportFormats: plugin.ExportFormats(), }) + for _, table := range plugin.ResultTables() { + schema, err := table.plugin.Schema(name + "_" + table.Name) + if err != nil { + return nil, err + } + infos[len(infos)-1].Results = append(infos[len(infos)-1].Results, KindResult{Name: table.Name, Columns: schema.Columns}) + } } return infos, nil } @@ -128,7 +188,7 @@ func (k *Kinds) RegisterResultTypes(registry *recordresults.Registry) error { return fmt.Errorf("trace kind %q: %w", name, err) } } - return nil + return recordresults.RegisterResultType(registry, profilepublication.ResultType()) } func (k *Kinds) names() []string { diff --git a/pkg/tracing/kinds_spec_test.go b/pkg/tracing/kinds_spec_test.go index 784fe21..14e713e 100644 --- a/pkg/tracing/kinds_spec_test.go +++ b/pkg/tracing/kinds_spec_test.go @@ -66,6 +66,20 @@ func (labeller) Handle(ctx dbcontext.Context, p labelParams, records tracing.Emi } var _ = Describe("Kinds", func() { + It("advertises named results under one capture and refuses table name collisions", func() { + kinds := tracing.NewKinds() + plugin := tickPlugin().WithResults(tracing.NewResultTable("metrics", ticker{}.Schema())) + Expect(kinds.RegisterKind("ticks", plugin)).To(Succeed()) + infos, err := kinds.List() + Expect(err).NotTo(HaveOccurred()) + Expect(infos).To(HaveLen(1)) + Expect(infos[0].Results).To(ConsistOf(HaveField("Name", "metrics"))) + Expect(kinds.RegisterKind("ticks_metrics", tickPlugin())).To(MatchError(ContainSubstring("already registered"))) + duplicate := tickPlugin().WithResults(tracing.NewResultTable("metrics", ticker{}.Schema()), tracing.NewResultTable("metrics", ticker{}.Schema())) + Expect(kinds.RegisterKind("duplicate", duplicate)).To(MatchError(ContainSubstring("already registered"))) + _, exists := kinds.Get("duplicate_metrics") + Expect(exists).To(BeFalse()) + }) It("passes the resolved seam into the handler's typed params", func() { got := make(chan target.Target, 1) plugin := tracing.NewHandler[targetParams, tick](targetRecorder{got: got}, tracing.Capabilities{Live: true}, target.Self) diff --git a/pkg/tracing/llm/pretty.go b/pkg/tracing/llm/pretty.go new file mode 100644 index 0000000..4d767ca --- /dev/null +++ b/pkg/tracing/llm/pretty.go @@ -0,0 +1,33 @@ +package llm + +import ( + "fmt" + "time" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" +) + +func (r LLMCall) Pretty() api.Text { + t := clicky.Text(r.Phase, "text-blue-500 font-bold").Space().Append(r.Model, "font-bold"). + Space().Append(r.Backend, "text-muted").Space(). + Append("tokens in ", "text-muted").Add(api.HumanNumber(int64(r.Usage.InputTokens), "text-blue-500")). + Append(" out ", "text-muted").Add(api.HumanNumber(int64(r.Usage.OutputTokens), "text-blue-500")). + Space().Append(fmt.Sprintf("$%.6f", r.Cost), "text-yellow-500").Space(). + Add(api.Human(time.Duration(r.DurationMs*float64(time.Millisecond)), "text-muted")) + if r.Error != "" { + t = t.Space().Append(r.Error, "text-red-500") + } + return t +} + +func (r LLMCall) PrettyFull() api.Textable { + t := r.Pretty() + if r.Prompt != "" { + t = t.NewLine().Append("Prompt", "font-bold").NewLine().Add(clicky.CodeBlock("text", r.Prompt)) + } + if r.Response != "" { + t = t.NewLine().Append("Response", "font-bold").NewLine().Add(clicky.CodeBlock("json", r.Response)) + } + return t +} diff --git a/pkg/tracing/llm/summary.go b/pkg/tracing/llm/summary.go new file mode 100644 index 0000000..4b847d1 --- /dev/null +++ b/pkg/tracing/llm/summary.go @@ -0,0 +1,22 @@ +package llm + +import "iter" + +func (capture) Summarize(rows iter.Seq[LLMCall]) (map[string]any, error) { + count, failed := 0, 0 + duration, cost := 0.0, 0.0 + models := map[string]int{} + input, output := 0, 0 + for row := range rows { + count++ + duration += row.DurationMs + cost += row.Cost + input += row.Usage.InputTokens + output += row.Usage.OutputTokens + models[row.Model]++ + if row.Error != "" { + failed++ + } + } + return map[string]any{"calls": count, "failures": failed, "durationMs": duration, "cost": cost, "models": models, "inputTokens": input, "outputTokens": output}, nil +} diff --git a/pkg/tracing/logs/pretty.go b/pkg/tracing/logs/pretty.go new file mode 100644 index 0000000..f1ad09b --- /dev/null +++ b/pkg/tracing/logs/pretty.go @@ -0,0 +1,19 @@ +package logs + +import ( + "strings" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" + "github.com/flanksource/telltale/pkg/tracing" +) + +func (r Log) Pretty() api.Text { + return clicky.Text(r.FirstObserved.Format("15:04:05.000"), "text-muted").Space(). + Append(strings.ToUpper(r.Severity), tracing.SeverityStyle(r.Severity)). + Space().Append(r.Logger, "text-blue-500").Space().Append(r.Message) +} + +func (r Log) PrettyFull() api.Textable { + return r.Pretty().NewLine().Add(clicky.Map(r.Labels)).NewLine().Add(clicky.Map(r.Resource)) +} diff --git a/pkg/tracing/logs/summary.go b/pkg/tracing/logs/summary.go new file mode 100644 index 0000000..b48ffab --- /dev/null +++ b/pkg/tracing/logs/summary.go @@ -0,0 +1,11 @@ +package logs + +import "iter" + +func (capture) Summarize(rows iter.Seq[Log]) (map[string]any, error) { + levels := map[string]int{} + for row := range rows { + levels[row.Severity]++ + } + return map[string]any{"levels": levels}, nil +} diff --git a/pkg/tracing/pgproxy/unified.go b/pkg/tracing/pgproxy/unified.go index a856a4b..c4fe74b 100644 --- a/pkg/tracing/pgproxy/unified.go +++ b/pkg/tracing/pgproxy/unified.go @@ -42,9 +42,23 @@ type unified struct { func (u unified) AgentAvailable(probe string, t target.Target) (bool, error) { return u.self.AgentAvailable(probe, t) } -func (unified) Params() UnifiedParams { +func (unified) Params() UnifiedParams { return DefaultParams() } + +// ProxyKind is the `sql` kind armed on its Postgres proxy seam (ProxyPreset). +const ProxyKind = "pg_proxy" + +// DefaultParams are the `sql` kind's defaults: telltale's own connection on +// its self and agent seams, and the proxy's connection name and statement cap +// on its proxy seam. A host adjusts a copy of them for Handler.WithDefaults. +func DefaultParams() UnifiedParams { return UnifiedParams{Params: pgProxy{}.Params(), Connections: []string{"self"}} } + +// ProxyPreset arms the `sql` kind only on its Postgres proxy seam, as +// ProxyKind, without the connections only its other seams observe. +func ProxyPreset() tracing.PresetOptions[UnifiedParams] { + return tracing.PresetOptions[UnifiedParams]{Title: "Postgres proxy", Seam: target.Proxy, Defaults: func(p *UnifiedParams) { p.Connections = nil }} +} func (unified) Schema() recordresults.ResultType[tracesql.StatementRow] { return tracesql.ResultType() } func (u unified) Prepare(ctx dbcontext.Context, p UnifiedParams, out tracing.Emitter[tracesql.StatementRow]) (dbcontext.Context, func() error, error) { if p.Target.Seam == target.Proxy { diff --git a/pkg/tracing/pgproxy/unified_spec_test.go b/pkg/tracing/pgproxy/unified_spec_test.go index d1895b0..af24862 100644 --- a/pkg/tracing/pgproxy/unified_spec_test.go +++ b/pkg/tracing/pgproxy/unified_spec_test.go @@ -16,4 +16,16 @@ var _ = Describe("unified SQL kind", func() { Expect(kind.ValidateParams(json.RawMessage(`{"target":{"seam":"self"},"connections":["self"]}`))).To(Succeed()) Expect(kind.ValidateParams(json.RawMessage(`{"target":{"seam":"proxy"},"dsn":"postgres://localhost/db"}`))).To(Succeed()) }) + It("arms the pg_proxy preset only on the proxy seam, with the proxy defaults and no connections", func() { + preset := UnifiedKind(nil).Preset(ProxyPreset()) + seams, defaultSeam := preset.TargetSeams() + Expect(seams).To(Equal([]target.Seam{target.Proxy})) + Expect(defaultSeam).To(Equal(target.Proxy)) + schema, err := preset.Params() + Expect(err).NotTo(HaveOccurred()) + Expect(schema.Properties["connection"].Default).To(Equal(defaultConnection)) + Expect(schema.Properties["connections"].Default).To(BeNil()) + Expect(preset.ValidateParams(json.RawMessage(`{"dsn":"postgres://localhost/db"}`))).To(Succeed()) + Expect(preset.ValidateParams(json.RawMessage(`{"target":{"seam":"self"}}`))).To(MatchError(ContainSubstring("arms seam"))) + }) }) diff --git a/pkg/tracing/plugin.go b/pkg/tracing/plugin.go index 1ed7167..5ed4dbb 100644 --- a/pkg/tracing/plugin.go +++ b/pkg/tracing/plugin.go @@ -20,6 +20,7 @@ import ( "github.com/flanksource/commons-db/query" "github.com/flanksource/commons-db/recordstore" "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/commons-db/recordstore/sampleframe" "github.com/flanksource/telltale/pkg/tracing/target" ) @@ -138,6 +139,7 @@ type TracePlugin interface { summarize(ctx context.Context, store scanner, ref TraceRef) (map[string]any, error) recordType() reflect.Type jsonFields() []string + ResultTables() []ResultTable } // openOptions are what a session gives the capture it opens. @@ -152,6 +154,8 @@ type openOptions struct { // Handler is a TraceHandler as a TracePlugin, with the processing its records // get before they are stored. type Handler[P, R any] struct { + top *TopSpec + tables []ResultTable handler TraceHandler[P, R] capabilities Capabilities defaults *P @@ -165,6 +169,11 @@ type Handler[P, R any] struct { seams []target.Seam defaultSeam target.Seam kind string + // title and presetSeam name a seam preset (Preset) and the seam it arms. + title string + presetSeam target.Seam + // invalid refuses the plugin's registration and every start. + invalid error } // TargetSeams returns the seams supported by this handler and its default. @@ -187,6 +196,9 @@ func (h *Handler[P, R]) ResolveTarget(params json.RawMessage) (target.Target, er return target.Target{}, fmt.Errorf("params: %w", err) } } + if err := h.presetSeamError(selection.Target.Seam); err != nil { + return target.Target{}, err + } var available func(target.Target) bool var lookupErr error if resolver, ok := h.handler.(interface { @@ -221,8 +233,20 @@ func NewHandler[P, R any](handler TraceHandler[P, R], capabilities Capabilities, // WithDefaults starts the kind's form and its captures from defaults rather // than the handler's own Params, as a host that knows its connections does. +// A seam the defaults' target names is the seam a start naming none is armed +// on; one the kind does not support refuses its registration. func (h *Handler[P, R]) WithDefaults(defaults P) *Handler[P, R] { h.defaults = &defaults + seam, err := seamOf(defaults) + switch { + case err != nil: + h.invalid = fmt.Errorf("params defaults: %w", err) + case seam == "": + case !slices.Contains(h.seams, seam): + h.invalid = fmt.Errorf("params defaults: %w %q; supported seams: %v", target.ErrUnsupportedSeam, seam, h.seams) + default: + h.defaultSeam = seam + } return h } @@ -230,6 +254,10 @@ func (h *Handler[P, R]) WithDefaults(defaults P) *Handler[P, R] { // copied, so decoding a start's params into them never reaches the slices and // maps the host and every other start share. func (h *Handler[P, R]) params() (P, error) { + if h.invalid != nil { + var zero P + return zero, h.invalid + } if h.defaults == nil { return h.handler.Params(), nil } @@ -296,7 +324,7 @@ func (h *Handler[P, R]) processors(schema recordstore.KindSchema) []rowProcessor return processors } -func (h *Handler[P, R]) Title() string { return h.handler.Schema().Title } +func (h *Handler[P, R]) Title() string { return h.resultType(h.kind).Title } func (h *Handler[P, R]) Capabilities() Capabilities { return h.capabilities } @@ -331,6 +359,16 @@ func (h *Handler[P, R]) Params() (*rpc.OpenAPISchema, error) { property.Default = value } } + if connection := schema.Properties["connection"]; connection != nil { + endpoint := "/api/v1/connection" + if provider, ok := h.handler.(interface{ ConnectionTypes() []string }); ok { + endpoint += "?types=" + strings.Join(provider.ConnectionTypes(), ",") + } + if connection.Extensions == nil { + connection.Extensions = map[string]any{} + } + connection.Extensions["x-clicky-lookup"] = map[string]any{"url": endpoint, "filter": "connection", "searchParam": "__lookup_q"} + } return schema, nil } @@ -415,7 +453,15 @@ func (h *Handler[P, R]) ValidateParams(params json.RawMessage) error { } func (h *Handler[P, R]) register(registry *recordresults.Registry, kind string) error { - return recordresults.RegisterResultType(registry, h.resultType(kind)) + if err := recordresults.RegisterResultType(registry, h.resultType(kind)); err != nil { + return err + } + for _, table := range h.tables { + if err := table.plugin.register(registry, kind+"_"+table.Name); err != nil { + return err + } + } + return nil } // open decodes params and prepares the handler's capture into a new source, @@ -431,13 +477,21 @@ func (h *Handler[P, R]) open(ctx dbcontext.Context, options openOptions) (*sourc return nil, err } src := newSource(options.stream, options.bufferRows, options.bufferBytes, schema.Options.Key, schema.Options.OnConflict == recordstore.OnConflictReplace) - emitter := &emitter[R]{source: src, process: pipeline(h.processors(schema))} + emitter := &emitter[R]{source: src, process: pipeline(h.processors(schema)), top: h.top} + if h.top != nil { + backend, ok := options.store.(sampleframe.Backend) + if !ok { + return nil, fmt.Errorf("Top capture requires an atomic sample backend") + } + emitter.frames = &sampleframe.Writer{Backend: backend, Schema: schema, Stream: options.stream} + } if h.dedup != nil { if window := h.dedup.window(params); window > 0 { emitter.dedup = newDeduplicator(window, h.dedup.key) } } ctx = ctx.WithValue(streamKey{}, options.stream) + ctx = ctx.WithValue(captureBackendKey{}, options.store) captureCtx, release := ctx, func() error { return nil } if preparer, ok := h.handler.(Preparer[P, R]); ok { prepared, prepareRelease, err := preparer.Prepare(ctx, params, emitter) @@ -462,7 +516,13 @@ func (h *Handler[P, R]) open(ctx dbcontext.Context, options openOptions) (*sourc records := storedRecords[R]{store: options.store, stream: options.stream, jsonPaths: h.jsonPaths} src.prepare(captureCtx, func(capture dbcontext.Context) error { return h.handler.Handle(capture, params, emitter, records) - }, release) + }, func() error { + err := release() + if emitter.frames != nil { + err = errors.Join(err, emitter.frames.Close(context.WithoutCancel(captureCtx))) + } + return err + }) return src, nil } @@ -494,6 +554,9 @@ func StreamOf(ctx context.Context) string { func (h *Handler[P, R]) resultType(kind string) recordresults.ResultType[R] { resultType := h.handler.Schema() resultType.Kind = kind + if h.title != "" { + resultType.Title = h.title + } return resultType } diff --git a/pkg/tracing/presentation.go b/pkg/tracing/presentation.go new file mode 100644 index 0000000..d6c9e39 --- /dev/null +++ b/pkg/tracing/presentation.go @@ -0,0 +1,106 @@ +package tracing + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" + "github.com/flanksource/commons-db/query" +) + +type RecordPresentation struct { + Summary api.Text + Detail api.Textable +} + +// Present restores typed presentation from the processed, stored record. Schema +// fields provide the standard presentation for kinds without Pretty methods. +func (k *Kinds) Present(kind string, row query.Row) (RecordPresentation, error) { + plugin, ok := k.Get(kind) + if !ok { + return RecordPresentation{}, fmt.Errorf("%w: %s", ErrUnknownKind, kind) + } + encoded, err := json.Marshal(row) + if err != nil { + return RecordPresentation{}, err + } + var stored query.Row + decoder := json.NewDecoder(bytes.NewReader(encoded)) + decoder.UseNumber() + if err := decoder.Decode(&stored); err != nil { + return RecordPresentation{}, err + } + jsonAsText(stored, plugin.jsonFields()) + encoded, err = json.Marshal(stored) + if err != nil { + return RecordPresentation{}, err + } + if presenter, ok := plugin.(interface { + presentRecord([]byte) (RecordPresentation, bool, error) + }); ok { + if presentation, handled, err := presenter.presentRecord(encoded); handled || err != nil { + return presentation, err + } + } + value := reflect.New(plugin.recordType()) + if err := json.Unmarshal(encoded, value.Interface()); err != nil { + return RecordPresentation{}, fmt.Errorf("present %s: %w", kind, err) + } + if value.Elem().Kind() == reflect.Pointer { + value = value.Elem() + } + var summary api.Text + if pretty, ok := value.Interface().(api.Pretty); ok { + summary = pretty.Pretty() + } else if table, ok := value.Interface().(api.TableProvider); ok { + summary = prettyRecordFields(table.Columns(), table.Row()) + } else { + schema, err := plugin.Schema(kind) + if err != nil { + return RecordPresentation{}, err + } + columns := []api.ColumnDef{} + for _, column := range schema.Columns { + if column.Hidden || column.Type == query.ColumnTypeJSON || row[column.Name] == nil { + continue + } + columns = append(columns, api.Column(column.Name).Label(column.Label).Build()) + } + summary = prettyRecordFields(columns, row) + } + native, err := json.MarshalIndent(row, "", " ") + if err != nil { + return RecordPresentation{}, err + } + var detail api.Textable = clicky.CodeBlock("json", string(native)) + if full, ok := value.Interface().(api.PrettyFull); ok { + detail = full.PrettyFull() + } + return RecordPresentation{Summary: summary, Detail: detail}, nil +} + +func prettyRecordFields(columns []api.ColumnDef, row map[string]any) api.Text { + text := api.Text{} + count := 0 + for _, column := range columns { + if column.Hidden || column.DefaultHidden || row[column.Name] == nil { + continue + } + if count > 0 { + text = text.Append(" · ", "text-muted") + } + value, ok := row[column.Name].(api.Textable) + if !ok { + value = clicky.Human(row[column.Name]) + } + text = text.Append(column.DisplayLabel()+"=", "text-muted").Add(value) + count++ + if count == 4 { + break + } + } + return text +} diff --git a/pkg/tracing/presentation_handler.go b/pkg/tracing/presentation_handler.go new file mode 100644 index 0000000..c9e80c0 --- /dev/null +++ b/pkg/tracing/presentation_handler.go @@ -0,0 +1,19 @@ +package tracing + +import "encoding/json" + +type recordPresenter[R any] interface { + Present(R) RecordPresentation +} + +func (h *Handler[P, R]) presentRecord(data []byte) (RecordPresentation, bool, error) { + presenter, ok := h.handler.(recordPresenter[R]) + if !ok { + return RecordPresentation{}, false, nil + } + var record R + if err := json.Unmarshal(data, &record); err != nil { + return RecordPresentation{}, true, err + } + return presenter.Present(record), true, nil +} diff --git a/pkg/tracing/presentation_test.go b/pkg/tracing/presentation_test.go new file mode 100644 index 0000000..90c26c4 --- /dev/null +++ b/pkg/tracing/presentation_test.go @@ -0,0 +1,42 @@ +package tracing_test + +import ( + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/target" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type bodyRecord struct { + Nested struct { + Body string `json:"body"` + } `json:"nested"` +} +type bodyPresenter struct{} + +func (bodyPresenter) Params() struct{} { return struct{}{} } +func (bodyPresenter) Schema() recordresults.ResultType[bodyRecord] { + return recordresults.ResultType[bodyRecord]{Title: "JSON bodies"} +} +func (bodyPresenter) Handle(dbcontext.Context, struct{}, tracing.Emitter[bodyRecord], tracing.Records[bodyRecord]) error { + return nil +} + +var _ = Describe("record presentation", func() { + It("ranks profile functions by weight and breaks ties by name", func() { + Expect(tracing.RankedValues(map[string]int64{"z": 1, "b": 4, "a": 4})).To(Equal([]map[string]any{{"function": "a", "value": int64(4)}, {"function": "b", "value": int64(4)}, {"function": "z", "value": int64(1)}})) + }) + It("restores nested JSON text without changing the native stored payload", func() { + kinds := tracing.NewKinds() + plugin := tracing.NewHandler[struct{}, bodyRecord](bodyPresenter{}, tracing.Capabilities{Live: true}, target.Self).WithJSONProcessor("nested.body") + Expect(kinds.RegisterKind("json_body", plugin)).To(Succeed()) + row := query.Row{"nested": map[string]any{"body": map[string]any{"answer": int64(7)}}} + presentation, err := kinds.Present("json_body", row) + Expect(err).NotTo(HaveOccurred()) + Expect(presentation.Detail).NotTo(BeNil()) + Expect(row).To(Equal(query.Row{"nested": map[string]any{"body": map[string]any{"answer": int64(7)}}})) + }) +}) diff --git a/pkg/tracing/preset.go b/pkg/tracing/preset.go new file mode 100644 index 0000000..207a61e --- /dev/null +++ b/pkg/tracing/preset.go @@ -0,0 +1,87 @@ +package tracing + +import ( + "errors" + "fmt" + "slices" + + "github.com/flanksource/telltale/pkg/tracing/target" +) + +// ErrPresetSeam identifies a start that names another seam than the one its +// preset arms. +var ErrPresetSeam = errors.New("the kind's preset fixes the target seam") + +// PresetOptions describe a seam preset of a kind: the base kind armed on one +// fixed seam, under its own kind name and title. +type PresetOptions[P any] struct { + // Title names the preset in a start form and in help. + Title string + // Seam is the one seam the preset arms; a start may name it, never another. + Seam target.Seam + // Defaults adjusts a copy of the base kind's defaults for the preset. + Defaults func(*P) +} + +// Preset is a new plugin of h's capture armed only on options.Seam, starting +// from h's defaults adjusted by options.Defaults. Registered under its own +// name, its records are h's record type, stored and read under that name. A +// preset h could not serve is refused when it is registered. +func (h *Handler[P, R]) Preset(options PresetOptions[P]) *Handler[P, R] { + preset := *h + preset.title, preset.presetSeam = options.Title, options.Seam + preset.seams, preset.defaultSeam = []target.Seam{options.Seam}, options.Seam + if preset.invalid = h.presetError(options); preset.invalid != nil { + return &preset + } + defaults, err := h.params() + if err != nil { + preset.invalid = err + return &preset + } + if options.Defaults != nil { + options.Defaults(&defaults) + } + preset.defaults = &defaults + if seam, err := seamOf(defaults); err != nil { + preset.invalid = fmt.Errorf("preset defaults: %w", err) + } else if seam != "" && seam != options.Seam { + preset.invalid = fmt.Errorf("preset defaults name seam %q, not the preset's seam %q", seam, options.Seam) + } + return &preset +} + +func (h *Handler[P, R]) presetError(options PresetOptions[P]) error { + switch { + case h.invalid != nil: + return h.invalid + case options.Title == "": + return errors.New("a preset needs a title") + case options.Seam == "": + return errors.New("a preset needs a seam") + case !slices.Contains(h.seams, options.Seam): + return fmt.Errorf("preset seam %q is not among the base kind's seams %v", options.Seam, h.seams) + case len(h.tables) > 0: + return errors.New("a kind with result tables cannot have presets") + } + return nil +} + +// presetSeamError refuses a start naming another seam than the preset's. +func (h *Handler[P, R]) presetSeamError(selected target.Seam) error { + if h.presetSeam == "" || selected == "" || selected == h.presetSeam { + return nil + } + return fmt.Errorf("%w: kind %q arms seam %q, not %q", ErrPresetSeam, h.kind, h.presetSeam, selected) +} + +// seamOf is the seam params' target names, if any. +func seamOf(params any) (target.Seam, error) { + var selection struct { + Target target.Target `json:"target"` + } + if err := roundTrip(params, &selection); err != nil { + return "", err + } + return selection.Target.Seam, nil +} diff --git a/pkg/tracing/preset_spec_test.go b/pkg/tracing/preset_spec_test.go new file mode 100644 index 0000000..4759882 --- /dev/null +++ b/pkg/tracing/preset_spec_test.go @@ -0,0 +1,139 @@ +// Specs for seam presets: a kind registered under its own name that arms its +// base kind on one fixed target seam, and host defaults that name a seam. + +package tracing_test + +import ( + "context" + "encoding/json" + "errors" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/target" + "github.com/flanksource/telltale/pkg/tracing/tracingtest" +) + +type seamParams struct { + Target target.Target `json:"target,omitzero"` + Label string `json:"label,omitempty"` +} + +// seamTicker emits one tick naming the seam it was armed on and its label. +type seamTicker struct{} + +func (seamTicker) Params() seamParams { return seamParams{Label: "base"} } + +func (seamTicker) Schema() recordresults.ResultType[tick] { + return recordresults.ResultType[tick]{Title: "Seams", TimeColumn: "at"} +} + +func (seamTicker) Handle(ctx dbcontext.Context, p seamParams, records tracing.Emitter[tick], _ tracing.Records[tick]) error { + return records.Emit(ctx, tick{At: time.Now().UTC(), N: 1, Label: string(p.Target.Seam) + ":" + p.Label}) +} + +func seamPlugin() *tracing.Handler[seamParams, tick] { + return tracing.NewHandler[seamParams, tick](seamTicker{}, tracing.Capabilities{Historical: true}, target.Self, target.Proxy) +} + +func proxyPreset() *tracing.Handler[seamParams, tick] { + return seamPlugin().Preset(tracing.PresetOptions[seamParams]{ + Title: "Proxy seams", Seam: target.Proxy, + Defaults: func(p *seamParams) { p.Label = "preset" }, + }) +} + +var _ = Describe("Seam defaults", func() { + It("arms a capture with no target on the seam the host's defaults name", func() { + plugin := seamPlugin().WithDefaults(seamParams{Target: target.Target{Seam: target.Proxy}, Label: "host"}) + seams, defaultSeam := plugin.TargetSeams() + Expect(seams).To(Equal([]target.Seam{target.Self, target.Proxy})) + Expect(defaultSeam).To(Equal(target.Proxy)) + schema, err := plugin.Params() + Expect(err).NotTo(HaveOccurred()) + Expect(schema.Properties["target"].Properties["seam"].Default).To(Equal("proxy")) + + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{"seams": plugin}) + Expect(env.Rows(tracingtest.Ended(env.Start("seams", `{}`)))).To(ConsistOf(HaveKeyWithValue("label", "proxy:host"))) + Expect(env.Rows(tracingtest.Ended(env.Start("seams", `{"target":{"seam":"self"}}`)))).To(ConsistOf(HaveKeyWithValue("label", "self:host"))) + }) + + It("refuses defaults naming a seam the kind does not support", func() { + plugin := seamPlugin().WithDefaults(seamParams{Target: target.Target{Seam: target.Host}}) + Expect(tracing.NewKinds().RegisterKind("seams", plugin)).To(MatchError(And(ContainSubstring(`"host"`), ContainSubstring("[self proxy]")))) + }) +}) + +var _ = Describe("Seam presets", func() { + var kinds *tracing.Kinds + + BeforeEach(func() { + kinds = tracing.NewKinds() + Expect(kinds.RegisterKind("seams", seamPlugin())).To(Succeed()) + Expect(kinds.RegisterKind("proxyseams", proxyPreset())).To(Succeed()) + }) + + It("lists the preset as its own kind, its form offering only the fixed seam", func() { + infos, err := kinds.List() + Expect(err).NotTo(HaveOccurred()) + Expect(infos).To(HaveLen(2)) + preset := infos[0] + Expect(preset.Name).To(Equal("proxyseams")) + Expect(preset.Title).To(Equal("Proxy seams")) + Expect(preset.Columns).To(Equal(infos[1].Columns)) + seam := preset.Params.Properties["target"].Properties["seam"] + Expect(seam.Enum).To(Equal([]any{"proxy"})) + Expect(seam.Default).To(Equal("proxy")) + Expect(preset.Params.Properties["label"].Default).To(Equal("preset")) + Expect(infos[1].Params.Properties["label"].Default).To(Equal("base")) + }) + + It("starts on the fixed seam and stores the base kind's records under the preset's name", func() { + plugin := proxyPreset() + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{"proxyseams": plugin}) + info := tracingtest.Ended(env.Start("proxyseams", `{"target":{"seam":"proxy"},"label":"asked"}`)) + Expect(env.Rows(info)).To(ConsistOf(HaveKeyWithValue("label", "proxy:asked"))) + + var read []tick + Expect(tracing.ReadRecords(context.Background(), env.Runtime, tracing.TraceRef{Kind: "proxyseams", Stream: info.Events.Stream}, func(record tick) error { + read = append(read, record) + return nil + })).To(Succeed()) + Expect(read).To(ConsistOf(HaveField("Label", "proxy:asked"))) + Expect(env.Rows(tracingtest.Ended(env.Start("proxyseams", `{}`)))).To(ConsistOf(HaveKeyWithValue("label", "proxy:preset"))) + }) + + It("refuses a start naming another seam", func() { + plugin, _ := kinds.Get("proxyseams") + err := plugin.ValidateParams(json.RawMessage(`{"target":{"seam":"self"}}`)) + Expect(errors.Is(err, tracing.ErrPresetSeam)).To(BeTrue(), "%v", err) + Expect(err).To(MatchError(And(ContainSubstring(`"proxyseams"`), ContainSubstring(`"proxy"`), ContainSubstring(`"self"`)))) + }) + + DescribeTable("refuses a preset it could not serve", + func(preset func() tracing.TracePlugin, message string) { + Expect(tracing.NewKinds().RegisterKind("broken", preset())).To(MatchError(ContainSubstring(message))) + }, + Entry("a seam the base kind does not support", func() tracing.TracePlugin { + return seamPlugin().Preset(tracing.PresetOptions[seamParams]{Title: "Hosts", Seam: target.Host}) + }, `preset seam "host"`), + Entry("no seam", func() tracing.TracePlugin { + return seamPlugin().Preset(tracing.PresetOptions[seamParams]{Title: "None"}) + }, "preset needs a seam"), + Entry("no title", func() tracing.TracePlugin { + return seamPlugin().Preset(tracing.PresetOptions[seamParams]{Seam: target.Proxy}) + }, "preset needs a title"), + Entry("defaults naming another seam", func() tracing.TracePlugin { + return seamPlugin().Preset(tracing.PresetOptions[seamParams]{Title: "Self", Seam: target.Proxy, Defaults: func(p *seamParams) { p.Target.Seam = target.Self }}) + }, `defaults name seam "self"`), + Entry("a base kind with result tables", func() tracing.TracePlugin { + return seamPlugin().WithResults(tracing.NewResultTable("metrics", ticker{}.Schema())). + Preset(tracing.PresetOptions[seamParams]{Title: "Tables", Seam: target.Proxy}) + }, "result tables"), + ) +}) diff --git a/pkg/tracing/pretty.go b/pkg/tracing/pretty.go new file mode 100644 index 0000000..4825b11 --- /dev/null +++ b/pkg/tracing/pretty.go @@ -0,0 +1,32 @@ +package tracing + +import ( + "strings" + "time" + + "github.com/flanksource/clicky/api" +) + +func SeverityStyle(level string) string { + switch strings.ToLower(level) { + case "error", "fatal", "panic", "failed", "down": + return "text-red-500 font-bold" + case "warn", "warning": + return "text-yellow-500 font-bold" + case "ok", "up", "info": + return "text-green-500" + default: + return "text-muted" + } +} + +func PrettySampleValue(value int64, unit string) api.Text { + switch unit { + case "bytes": + return api.HumanizeBytes(value).Styles("text-blue-500") + case "ns", "nanoseconds": + return api.Human(time.Duration(value), "text-yellow-500") + default: + return api.HumanNumber(value, "text-blue-500").Space().Append(unit, "text-muted") + } +} diff --git a/pkg/tracing/processcapture/benchmark_test.go b/pkg/tracing/processcapture/benchmark_test.go new file mode 100644 index 0000000..12cadf6 --- /dev/null +++ b/pkg/tracing/processcapture/benchmark_test.go @@ -0,0 +1,129 @@ +package processcapture + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + clickyprocess "github.com/flanksource/clicky/process" + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/commons-db/recordstore/metrics" + "github.com/flanksource/commons-db/recordstore/sqlite" + "github.com/flanksource/telltale/pkg/tracing/process" +) + +func BenchmarkProcessCollection(b *testing.B) { + b.Run("process_listing", func(b *testing.B) { + for b.Loop() { + if _, err := clickyprocess.Discover(context.Background(), clickyprocess.SnapshotOptions{}); err != nil { + b.Fatal(err) + } + } + }) + b.Run("descriptors", func(b *testing.B) { + for b.Loop() { + if _, err := clickyprocess.Descriptors(context.Background(), os.Getpid()); err != nil { + b.Fatal(err) + } + } + }) +} + +func BenchmarkProcessObservationStorage(b *testing.B) { + for _, format := range []string{"snapshot", "unified"} { + b.Run(format, func(b *testing.B) { + ctx := context.Background() + path := filepath.Join(b.TempDir(), "records.sqlite") + schemas, err := Schemas() + if err != nil { + b.Fatal(err) + } + for kind, plugin := range map[string]interface { + Schema(string) (recordstore.KindSchema, error) + }{process.TreeKind: process.NewTreeKind(process.NewSampler(), process.TreeKindOptions{}), process.SamplesKind: process.NewSamplesKind(process.NewSampler())} { + schema, err := plugin.Schema(kind) + if err != nil { + b.Fatal(err) + } + schemas = append(schemas, schema) + } + backend, err := sqlite.Open(sqlite.Options{Path: path, SweepInterval: time.Hour, Schema: func(kind string) (recordstore.KindSchema, error) { + for _, schema := range schemas { + if schema.Kind == kind { + return schema, nil + } + } + return recordstore.KindSchema{}, fmt.Errorf("unknown benchmark kind %s", kind) + }}) + if err != nil { + b.Fatal(err) + } + defer func() { + if err := backend.Close(); err != nil { + b.Error(err) + } + }() + store, err := NewStore(StoreOptions{Backend: backend, Stream: "capture"}) + if err != nil { + b.Fatal(err) + } + const processes, files = 20, 100 + at := time.Unix(100, 0) + descriptors := make([]clickyprocess.Descriptor, files) + for i := range descriptors { + descriptors[i] = clickyprocess.Descriptor{FD: strconv.Itoa(i), Type: "REG", Name: "/work/checkout/source/" + strconv.Itoa(i) + ".go"} + } + b.ReportAllocs() + b.ResetTimer() + for tick := 1; tick <= b.N; tick++ { + descriptors[0].Name = "/work/output/" + strconv.Itoa(tick) + ".log" + o := Observation{At: at.Add(time.Duration(tick) * time.Second), Tick: int64(tick)} + nodes, samples := []recordstore.Row{}, []recordstore.Row{} + for pid := 1; pid <= processes; pid++ { + key := strconv.Itoa(pid) + identity := Identity{Key: key, PID: pid, Name: "worker", Command: "worker --mode compile --config /work/checkout/config.yaml", FirstObservedAt: at, StartedAt: at, Environment: map[string]string{"MODE": "compile", "PATH": "/usr/bin:/usr/local/bin"}} + if format == "unified" { + o.Processes = append(o.Processes, identity) + o.Metrics = append(o.Metrics, metrics.Sample{Entity: key, Values: map[string]any{"rssBytes": 1024 * tick, "cpuPercent": tick % 100}}) + o.Facets = append(o.Facets, FacetObservation{Process: key, Complete: true, Descriptors: descriptors}) + } else { + node, err := recordstore.EncodeRow(process.Node{Key: key, PID: pid, Name: identity.Name, Command: identity.Command, Environment: identity.Environment, StartedAt: at, LastSeenAt: o.At, RSSBytes: uint64(1024 * tick)}) + if err != nil { + b.Fatal(err) + } + sample, err := recordstore.EncodeRow(process.Sample{Key: key, PID: pid, SampledAt: o.At, Tick: o.Tick, RSSBytes: uint64(1024 * tick), OpenFileList: descriptors}) + if err != nil { + b.Fatal(err) + } + nodes, samples = append(nodes, node), append(samples, sample) + } + } + if format == "unified" { + if err := store.Apply(ctx, o); err != nil { + b.Fatal(err) + } + } else { + if _, err := backend.Append(ctx, "tree", process.TreeKind, nodes); err != nil { + b.Fatal(err) + } + if _, err := backend.Append(ctx, "samples", process.SamplesKind, samples); err != nil { + b.Fatal(err) + } + } + } + b.StopTimer() + if err := backend.Close(); err != nil { + b.Fatal(err) + } + info, err := os.Stat(backend.Path()) + if err != nil { + b.Fatal(err) + } + b.ReportMetric(float64(info.Size())/float64(b.N), "bytes/tick") + }) + } +} diff --git a/pkg/tracing/processcapture/facets.go b/pkg/tracing/processcapture/facets.go new file mode 100644 index 0000000..18f2a87 --- /dev/null +++ b/pkg/tracing/processcapture/facets.go @@ -0,0 +1,213 @@ +package processcapture + +import ( + "cmp" + "crypto/sha256" + "encoding/hex" + "encoding/json" + clickyprocess "github.com/flanksource/clicky/process" + "maps" + "net" + "slices" + "strings" + "time" +) + +const ( + Files = "files" + Sockets = "sockets" + ConnectionIn = "connection_in" + ConnectionOut = "connection_out" + ConnectionUnknown = "connection_unknown" +) + +type FacetEvent struct { + Process string `json:"process"` + At time.Time `json:"observedAt"` + Tick int64 `json:"tick"` + Facet string `json:"facet"` + Entity string `json:"entity"` + Operation string `json:"operation"` + DirectionSource string `json:"directionSource,omitempty"` + Reason string `json:"reason,omitempty"` + Resource clickyprocess.Descriptor `json:"resource"` +} + +type FacetObservation struct { + Process string + At time.Time + Tick int64 + Complete bool + Exited bool + Descriptors []clickyprocess.Descriptor + Enabled []string +} + +type facetResource struct { + facet, entity, direction string + descriptor clickyprocess.Descriptor +} + +// FacetState is an immutable observation checkpoint. Diff returns its successor; +// the collector adopts it only after the observation's storage transaction commits. +type FacetState struct { + resources map[string]map[string]facetResource + listeners map[string][]clickyprocess.Descriptor +} + +func NewFacetState() *FacetState { + return &FacetState{resources: map[string]map[string]facetResource{}, listeners: map[string][]clickyprocess.Descriptor{}} +} + +func (s *FacetState) Diff(o FacetObservation) ([]FacetEvent, *FacetState) { + if !o.Complete && !o.Exited { + return nil, s + } + next := s.clone() + return next.diff(o), next +} + +func (s *FacetState) clone() *FacetState { + return &FacetState{resources: maps.Clone(s.resources), listeners: maps.Clone(s.listeners)} +} + +func (s *FacetState) diff(o FacetObservation) []FacetEvent { + if !o.Complete && !o.Exited { + return nil + } + previous := s.resources[o.Process] + peers := map[string]facetResource{} + for _, old := range previous { + if strings.HasPrefix(old.facet, "connection_") { + peers[old.descriptor.Protocol+"/"+old.descriptor.RemoteAddr] = old + } + } + current := map[string]facetResource{} + if !o.Exited { + listeners := slices.Clone(s.listeners[o.Process]) + for _, d := range o.Descriptors { + if d.IsListening() && !slices.Contains(listeners, d) { + listeners = append(listeners, d) + } + } + s.listeners[o.Process] = listeners + for _, d := range o.Descriptors { + resource, ok := s.classify(o.Process, d, listeners) + if !ok { + continue + } + selection := resource.facet + if strings.HasPrefix(selection, "connection_") { + selection = "connections" + } + if o.Enabled != nil && !slices.Contains(o.Enabled, selection) { + continue + } + // Retain the first inferred direction for an active peer. A later + // polling observation cannot establish who initiated that connection. + if strings.HasPrefix(resource.facet, "connection_") { + if old, ok := peers[d.Protocol+"/"+d.RemoteAddr]; ok { + resource = old + } + } + current[resource.facet+"/"+resource.entity] = resource + } + } + events := []FacetEvent{} + for key, old := range previous { + if _, ok := current[key]; !ok { + events = append(events, facetEvent(o, old, "removed")) + } + } + for key, value := range current { + if _, ok := previous[key]; !ok { + events = append(events, facetEvent(o, value, "added")) + } + } + slices.SortFunc(events, func(a, b FacetEvent) int { + if n := cmp.Compare(a.Operation, b.Operation); n != 0 { + return -n + } + if n := cmp.Compare(a.Facet, b.Facet); n != 0 { + return n + } + return cmp.Compare(a.Entity, b.Entity) + }) + if o.Exited { + delete(s.resources, o.Process) + delete(s.listeners, o.Process) + } else { + s.resources[o.Process] = current + } + return events +} + +func (s *FacetState) classify(process string, d clickyprocess.Descriptor, listeners []clickyprocess.Descriptor) (facetResource, bool) { + facet, direction := Files, "" + if d.IsSocket() { + switch { + case d.IsListening(): + facet = Sockets + case d.Protocol == "TCP" && d.State == "ESTABLISHED" && d.RemoteAddr != "": + facet, direction = ConnectionUnknown, "unknown" + for _, listener := range listeners { + if receiving(listener, d) { + facet, direction = ConnectionIn, "inferred_listener" + break + } + } + if facet == ConnectionUnknown { + if _, observed := s.resources[process]; observed { + facet, direction = ConnectionOut, "inferred_no_listener" + } + } + d = clickyprocess.Descriptor{Protocol: d.Protocol, RemoteAddr: d.RemoteAddr} + default: + return facetResource{}, false + } + } + encoded, err := json.Marshal(d) + if err != nil { + panic(err) + } + sum := sha256.Sum256(encoded) + return facetResource{facet: facet, entity: hex.EncodeToString(sum[:]), direction: direction, descriptor: d}, true +} + +func receiving(listener, connection clickyprocess.Descriptor) bool { + if listener.Protocol != connection.Protocol { + return false + } + host, port, err := net.SplitHostPort(listener.LocalAddr) + if err != nil { + return listener.LocalAddr == connection.LocalAddr + } + localHost, localPort, err := net.SplitHostPort(connection.LocalAddr) + return err == nil && port == localPort && (host == localHost || host == "*" || host == "0.0.0.0" || host == "::") +} + +func facetEvent(o FacetObservation, r facetResource, operation string) FacetEvent { + event := FacetEvent{Process: o.Process, At: o.At, Tick: o.Tick, Facet: r.facet, Entity: r.entity, Operation: operation, DirectionSource: r.direction, Resource: r.descriptor} + if o.Exited { + event.Reason = "observed_exit" + } + return event +} + +func (s *FacetState) counts(o FacetObservation) map[string]any { + counts := map[string]int{} + for _, resource := range s.resources[o.Process] { + counts[resource.facet]++ + } + values := map[string]any{} + for facet, metric := range map[string]string{Files: "openFiles", Sockets: "receivingSockets", ConnectionIn: "connectionsIn", ConnectionOut: "connectionsOut", ConnectionUnknown: "connectionsUnknown"} { + selected := facet + if strings.HasPrefix(facet, "connection_") { + selected = "connections" + } + if o.Enabled == nil || slices.Contains(o.Enabled, selected) { + values[metric] = counts[facet] + } + } + return values +} diff --git a/pkg/tracing/processcapture/facets_test.go b/pkg/tracing/processcapture/facets_test.go new file mode 100644 index 0000000..ce4baf9 --- /dev/null +++ b/pkg/tracing/processcapture/facets_test.go @@ -0,0 +1,72 @@ +package processcapture + +import ( + clickyprocess "github.com/flanksource/clicky/process" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "testing" + "time" +) + +func TestProcessCapture(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Unified process capture") +} + +var _ = Describe("facet deltas", func() { + It("keeps files and receiving sockets independent and infers inbound peers from a listener", func() { + listener := clickyprocess.Descriptor{FD: "3", Type: "IPv4", Protocol: "TCP", State: "LISTEN", LocalAddr: "*:8080"} + peer := clickyprocess.Descriptor{FD: "4", Type: "IPv4", Protocol: "TCP", State: "ESTABLISHED", LocalAddr: "127.0.0.1:8080", RemoteAddr: "127.0.0.1:51000"} + o := FacetObservation{Process: "worker", At: time.Unix(1, 0), Tick: 1, Complete: true, Enabled: []string{Sockets, "connections"}, Descriptors: []clickyprocess.Descriptor{listener, peer, {FD: "5", Type: "REG", Name: "config"}}} + events, state := NewFacetState().Diff(o) + Expect(events).To(ConsistOf(HaveField("Facet", Sockets), And(HaveField("Facet", ConnectionIn), HaveField("DirectionSource", "inferred_listener")))) + o.Exited, o.Complete = true, false + events, _ = state.Diff(o) + Expect(events).To(HaveEach(And(HaveField("Operation", "removed"), HaveField("Reason", "observed_exit")))) + }) + It("infers outbound only after an initial complete observation and keeps reused process identities separate", func() { + state := NewFacetState() + o := FacetObservation{Process: "123@1", At: time.Unix(1, 0), Tick: 1, Complete: true} + _, state = state.Diff(o) + o.Descriptors = []clickyprocess.Descriptor{{Type: "IPv4", Protocol: "TCP", State: "ESTABLISHED", RemoteAddr: "127.0.0.1:5432", LocalAddr: "127.0.0.1:51000"}} + events, state := state.Diff(o) + Expect(events).To(ConsistOf(HaveField("Facet", ConnectionOut))) + o.Process = "123@2" + events, _ = state.Diff(o) + Expect(events).To(ConsistOf(HaveField("Facet", ConnectionUnknown))) + }) + It("aggregates parallel connections by peer and removes only the last one", func() { + state := NewFacetState() + first := clickyprocess.Descriptor{FD: "7u", Type: "IPv4", Protocol: "TCP", State: "ESTABLISHED", LocalAddr: "127.0.0.1:51000", RemoteAddr: "127.0.0.1:5432"} + second := first + second.FD = "8u" + second.LocalAddr = "127.0.0.1:51001" + observation := FacetObservation{Process: "p", At: time.Unix(1, 0), Tick: 1, Complete: true, Descriptors: []clickyprocess.Descriptor{first, second}} + events, next := state.Diff(observation) + Expect(events).To(HaveLen(1)) + Expect(events[0].Facet).To(Equal(ConnectionUnknown)) + Expect(events[0].Operation).To(Equal("added")) + observation.Tick++ + observation.Descriptors = []clickyprocess.Descriptor{second} + events, next = next.Diff(observation) + Expect(events).To(BeEmpty()) + observation.Tick++ + observation.Descriptors = nil + events, _ = next.Diff(observation) + Expect(events).To(HaveLen(1)) + Expect(events[0].Operation).To(Equal("removed")) + }) + It("preserves state for incomplete reads and ignores descriptor order", func() { + state := NewFacetState() + o := FacetObservation{Process: "p", At: time.Unix(1, 0), Tick: 1, Complete: true, Descriptors: []clickyprocess.Descriptor{{FD: "1", Type: "REG", Name: "a"}, {FD: "2", Type: "REG", Name: "b"}}} + _, state = state.Diff(o) + o.Complete = false + o.Descriptors = nil + events, next := state.Diff(o) + Expect(events).To(BeEmpty()) + o.Complete = true + o.Descriptors = []clickyprocess.Descriptor{{FD: "2", Type: "REG", Name: "b"}, {FD: "1", Type: "REG", Name: "a"}} + events, _ = next.Diff(o) + Expect(events).To(BeEmpty()) + }) +}) diff --git a/pkg/tracing/processcapture/follow.go b/pkg/tracing/processcapture/follow.go new file mode 100644 index 0000000..28d2b1c --- /dev/null +++ b/pkg/tracing/processcapture/follow.go @@ -0,0 +1,134 @@ +package processcapture + +import ( + "context" + "fmt" + "slices" + "strings" + "sync" + + clickyprocess "github.com/flanksource/clicky/process" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/groups" + "github.com/flanksource/telltale/pkg/tracing/process" +) + +type Follower struct { + mu sync.Mutex + runtime *tracing.Runtime + ref tracing.TraceRef + after int64 + latest groups.Snapshot +} + +func NewFollower(runtime *tracing.Runtime, ref tracing.TraceRef) (*Follower, error) { + if runtime == nil || ref.Kind != Kind { + return nil, fmt.Errorf("process follower requires a process capture and runtime") + } + if err := ref.Validate(); err != nil { + return nil, err + } + return &Follower{runtime: runtime, ref: ref}, nil +} + +// Frame derives the latest committed observation from identities and metrics. +func (f *Follower) Frame(ctx context.Context) (groups.Frame, bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + ref := tracing.TraceRef{Kind: MetricsTable, Stream: f.ref.Stream + "-metrics", AfterSeq: f.after} + meta, err := f.runtime.Results.Backend.Meta(ctx, ref.Stream) + if err != nil { + return groups.Frame{}, false, err + } + if meta.HighSeq == f.after { + return f.latest.Frame, f.latest.Frame.Tick > 0, nil + } + ref.ToSeq = meta.HighSeq + var identities []Identity + if err := tracing.ReadRecords(ctx, f.runtime, f.ref, func(p Identity) error { identities = append(identities, p); return nil }); err != nil { + return groups.Frame{}, false, err + } + var rows []MetricRow + var latest int64 + if err := tracing.ReadRecords(ctx, f.runtime, ref, func(row MetricRow) error { + if row.Tick > latest { + rows, latest = rows[:0], row.Tick + } + if row.Tick == latest { + rows = append(rows, row) + } + return nil + }); err != nil { + return groups.Frame{}, false, err + } + snapshot, err := GroupSnapshot(identities, rows) + if err != nil { + return groups.Frame{}, false, err + } + f.after, f.latest = meta.HighSeq, snapshot + return snapshot.Frame, snapshot.Frame.Tick > 0, nil +} + +func (f *Follower) Processes(filter groups.ProcessFilter) []groups.ProcessSample { + f.mu.Lock() + defer f.mu.Unlock() + return groups.FilterProcesses(f.latest.Processes, filter) +} + +// GroupSnapshot aggregates process measurements using startup grouping tags. +func GroupSnapshot(identities []Identity, rows []MetricRow) (groups.Snapshot, error) { + byKey := map[string]Identity{} + for _, p := range identities { + byKey[p.Key] = p + } + definitions := map[string]GroupInfo{} + classified := []groups.Group{} + indices := map[string]int{} + readings := map[string]groups.Reading{} + result := groups.Snapshot{} + for _, row := range rows { + p, ok := byKey[row.Entity] + if !ok { + return result, fmt.Errorf("metrics reference unknown process %q", row.Entity) + } + if len(p.Groups) == 0 { + return result, fmt.Errorf("process %s has no startup grouping tags", p.Key) + } + for _, info := range slices.Backward(p.Groups) { + if _, exists := definitions[info.Key]; exists { + continue + } + definitions[info.Key] = info + indices[info.Key] = len(classified) + classified = append(classified, groups.Group{Key: info.Key, ParentKey: info.ParentKey}) + } + observed := process.Observed{Process: clickyprocess.Process{PID: p.PID, PPID: p.PPID, Command: p.Command}} + if strings.Contains(p.Key, "@") { + observed.StartedAt = &p.StartedAt + } + if process.Key(observed.Process) != p.Key { + return result, fmt.Errorf("process identity %q does not match its pid and start", p.Key) + } + index := indices[p.Groups[0].Key] + classified[index].Members = append(classified[index].Members, observed) + var connections *int + if row.ConnectionsIn != nil && row.ConnectionsOut != nil && row.ConnectionsUnknown != nil { + n := *row.ConnectionsIn + *row.ConnectionsOut + *row.ConnectionsUnknown + connections = &n + } + reading := groups.Reading{CPUPercent: row.CPUPercent, RSSBytes: row.RSSBytes, SwapBytes: row.SwapBytes, CompressedBytes: row.CompressedBytes, Sockets: row.ReceivingSockets, Connections: connections, + PerSecond: groups.PerSecond{DiskRead: row.DiskReadBps, DiskWrite: row.DiskWriteBps, NetRx: row.NetRxBps, NetTx: row.NetTxBps}} + readings[p.Key] = reading + result.Processes = append(result.Processes, groups.ProcessSample{Key: p.Key, PID: p.PID, PPID: p.PPID, Name: p.Name, Command: p.Command, GroupKey: p.Groups[0].Key, + Origin: groups.Origin{Owner: p.Tags["owner"], Scope: p.Tags["scope"], Source: p.Tags["source"], Service: p.Tags["service"]}, + Usage: groups.Usage{Processes: 1, CPUPercent: reading.CPUPercent, RSSBytes: reading.RSSBytes, SwapBytes: reading.SwapBytes, CompressedBytes: reading.CompressedBytes, PerSecond: reading.PerSecond, Sockets: reading.Sockets, Connections: reading.Connections}}) + result.Frame.SampledAt, result.Frame.Tick = row.At, row.Tick + } + totals := groups.Aggregate(classified, readings) + for _, group := range classified { + info := definitions[group.Key] + result.Frame.Groups = append(result.Frame.Groups, groups.GroupSample{SampledAt: result.Frame.SampledAt, Tick: result.Frame.Tick, Key: info.Key, ParentKey: info.ParentKey, Name: info.Name, Kind: info.Kind, Target: info.Target, PID: info.PID, Command: info.Command, Context: info.Context, Origin: info.Origin, SkipAncestry: info.SkipAncestry, + Self: totals[info.Key].Self, Total: totals[info.Key].Total}) + } + return result, nil +} diff --git a/pkg/tracing/processcapture/frames.go b/pkg/tracing/processcapture/frames.go new file mode 100644 index 0000000..48f3c22 --- /dev/null +++ b/pkg/tracing/processcapture/frames.go @@ -0,0 +1,56 @@ +package processcapture + +import ( + "context" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/groups" +) + +type Frames struct { + HighSeq int64 `json:"highSeq"` + Frames []groups.Frame `json:"frames"` +} + +// ReadFrames derives complete ticks from an atomic process metrics stream. +func ReadFrames(ctx context.Context, runtime *tracing.Runtime, ref tracing.TraceRef) (Frames, error) { + result := Frames{Frames: []groups.Frame{}} + metricRef := tracing.TraceRef{Kind: MetricsTable, Stream: ref.Stream + "-metrics", AfterSeq: ref.AfterSeq} + meta, err := runtime.Results.Backend.Meta(ctx, metricRef.Stream) + if err != nil { + return result, err + } + metricRef.ToSeq, result.HighSeq = meta.HighSeq, meta.HighSeq + identities := []Identity{} + primary := tracing.TraceRef{Kind: Kind, Stream: ref.Stream} + if err := tracing.ReadRecords(ctx, runtime, primary, func(p Identity) error { identities = append(identities, p); return nil }); err != nil { + return result, err + } + var tick []MetricRow + flush := func() error { + if len(tick) == 0 { + return nil + } + snapshot, err := GroupSnapshot(identities, tick) + if err != nil { + return err + } + result.Frames = append(result.Frames, snapshot.Frame) + tick = tick[:0] + return nil + } + if err := tracing.ReadRecords(ctx, runtime, metricRef, func(row MetricRow) error { + if len(tick) > 0 && tick[0].Tick != row.Tick { + if err := flush(); err != nil { + return err + } + } + tick = append(tick, row) + return nil + }); err != nil { + return result, err + } + if err := flush(); err != nil { + return result, err + } + return result, nil +} diff --git a/pkg/tracing/processcapture/kind.go b/pkg/tracing/processcapture/kind.go new file mode 100644 index 0000000..045e601 --- /dev/null +++ b/pkg/tracing/processcapture/kind.go @@ -0,0 +1,180 @@ +package processcapture + +import ( + "context" + "fmt" + "os" + "slices" + "sync" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/groups" + "github.com/flanksource/telltale/pkg/tracing/process" + "github.com/flanksource/telltale/pkg/tracing/target" +) + +type Options struct { + process.Options + Environment *process.EnvironmentParams `json:"environment,omitempty"` + Facets []string `json:"facets,omitempty"` + Catalog string `json:"catalog,omitempty"` + Net bool `json:"net,omitempty"` +} + +type Params struct { + Options + Target target.Target `json:"target,omitzero"` +} + +func (p Params) Validate() error { + if err := p.Options.Validate(); err != nil { + return err + } + if p.Target.PID < 0 { + return fmt.Errorf("invalid process pid %d", p.Target.PID) + } + for i, facet := range p.Facets { + if !slices.Contains([]string{Files, Sockets, "connections"}, facet) || slices.Contains(p.Facets[:i], facet) { + return fmt.Errorf("invalid or repeated process facet %q: expected files, sockets or connections", facet) + } + } + return nil +} + +func (p Params) root() int { + if p.Target.Seam == target.Host { + return 0 + } + if p.Target.PID == 0 { + return os.Getpid() + } + return p.Target.PID +} + +type KindOptions struct{ Dumpers process.TransportLister } +type collector struct { + sampler *process.Sampler + options KindOptions +} +type captureKey struct{} +type capture struct { + mu sync.Mutex + store *Store + top *groups.Top + params Params + sampler *process.Sampler + rootGone bool + dumpers process.TransportLister +} + +func (collector) Params() Params { return Params{} } +func (collector) Schema() recordresults.ResultType[Identity] { + return recordresults.ResultType[Identity]{Title: "Processes", TimeColumn: "firstObservedAt", KeyColumn: "key", OnConflict: recordstore.OnConflictReplace, + Hierarchy: &recordresults.HierarchyColumns{ID: "key", Parent: "parentKey"}, SearchColumns: []string{"name", "command"}} +} + +func (c collector) Prepare(ctx dbcontext.Context, params Params, _ tracing.Emitter[Identity]) (dbcontext.Context, func() error, error) { + backend, err := tracing.CaptureBackend(ctx) + if err != nil { + return ctx, nil, err + } + store, err := NewStore(StoreOptions{Backend: backend, Stream: tracing.StreamOf(ctx)}) + if err != nil { + return ctx, nil, err + } + interval := time.Second + if params.Interval != "" { + interval, err = time.ParseDuration(params.Interval) + } + if err != nil { + return ctx, nil, err + } + var paths []string + if params.Catalog != "" { + paths = append(paths, params.Catalog) + } else { + path, err := groups.CatalogFile() + if err != nil { + return ctx, nil, err + } + if path != "" { + paths = append(paths, path) + } + } + catalog, err := groups.LoadCatalog(paths...) + if err != nil { + return ctx, nil, err + } + top, err := groups.NewTop(ctx, c.sampler, groups.TopOptions{Catalog: catalog, All: true, RootPID: params.root(), Interval: interval, Net: params.Net, SkipDescriptors: len(params.Facets) == 0}) + if err != nil { + return ctx, nil, err + } + run := &capture{store: store, top: top, params: params, sampler: c.sampler, dumpers: c.options.Dumpers} + if err := run.observe(ctx); err != nil { + top.Close() + return ctx, nil, err + } + if params.root() > 0 && run.rootGone { + top.Close() + return ctx, nil, fmt.Errorf("process %d is not running", params.root()) + } + return ctx.WithValue(captureKey{}, run), func() error { top.Close(); return store.Close(context.WithoutCancel(ctx)) }, nil +} + +func (collector) Handle(ctx dbcontext.Context, params Params, _ tracing.Emitter[Identity], _ tracing.Records[Identity]) error { + run := ctx.Value(captureKey{}).(*capture) + interval := time.Second + if params.Interval != "" { + var err error + interval, err = time.ParseDuration(params.Interval) + if err != nil { + return err + } + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return run.observe(context.WithoutCancel(ctx)) + case <-ticker.C: + if err := run.observe(ctx); err != nil { + if ctx.Err() != nil { + return run.observe(context.WithoutCancel(ctx)) + } + return err + } + run.mu.Lock() + gone := run.rootGone + run.mu.Unlock() + if gone { + return nil + } + } + } +} + +func (collector) Checkpoint(ctx dbcontext.Context, _ Params, _ tracing.Emitter[Identity]) error { + return ctx.Value(captureKey{}).(*capture).observe(ctx) +} + +// NewKind records one process table, resource deltas, and a wide metrics table. +func NewKind(sampler *process.Sampler, options KindOptions) *tracing.Handler[Params, Identity] { + return tracing.NewHandler[Params, Identity](collector{sampler: sampler, options: options}, tracing.Capabilities{Live: true}, target.Process, target.Host).WithResults( + tracing.NewResultTable("facets", recordresults.ResultType[FacetEvent]{Title: "Process facets", TimeColumn: "observedAt", Follow: true, Indexes: []recordstore.IndexDef{{Columns: []string{"process", "facet", "entity", "tick"}}}}), + tracing.NewResultTable("metrics", recordresults.ResultType[MetricRow]{Title: "Process metrics", TimeColumn: "observedAt", Dynamic: true, Follow: true, Indexes: []recordstore.IndexDef{{Columns: []string{"entity", "observedAt"}}}, Views: []recordresults.ResultView{latestMetricsView()}}), + ).WithTop(topSpec()) +} + +func (s *Store) Close(ctx context.Context) error { + s.mu.Lock() + defer s.mu.Unlock() + _, err := s.backend.AppendBatch(ctx, recordstore.Batch{ID: s.stream + "-close", Atomic: true, Producer: recordstore.Producer{Instance: s.stream, Seq: s.tick + 1}, Entries: []recordstore.BatchEntry{ + {Op: recordstore.BatchSeal, Stream: s.stream + "-facets"}, {Op: recordstore.BatchSeal, Stream: s.stream + "-metrics"}, + }}) + return err +} diff --git a/pkg/tracing/processcapture/kind_test.go b/pkg/tracing/processcapture/kind_test.go new file mode 100644 index 0000000..9388d51 --- /dev/null +++ b/pkg/tracing/processcapture/kind_test.go @@ -0,0 +1,54 @@ +package processcapture_test + +import ( + "context" + "fmt" + _ "github.com/flanksource/commons-db/query/providers/sqlite" + "os" + "time" + + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/telltale/pkg/traces/read" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/process" + "github.com/flanksource/telltale/pkg/tracing/processcapture" + "github.com/flanksource/telltale/pkg/tracing/tracingtest" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("unified process runtime", func() { + It("records startup once, numeric metrics separately, and seals all results", func() { + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{processcapture.Kind: processcapture.NewKind(process.NewSampler(), processcapture.KindOptions{})}) + session := env.Start(processcapture.Kind, fmt.Sprintf(`{"target":{"pid":%d},"interval":"250ms"}`, os.Getpid())) + tracingtest.Running(session) + Eventually(func() int64 { + meta, _ := env.Results.Backend.Meta(context.Background(), session.Snapshot().Events.Stream+"-metrics") + return meta.Total + }, 10*time.Second).Should(BeNumerically(">=", 2)) + session.Stop("test") + info := tracingtest.Ended(session) + Expect(info.Error).To(BeEmpty()) + stream := info.Events.Stream + for _, name := range []string{stream, stream + "-metrics", stream + "-facets"} { + meta, err := env.Results.Backend.Meta(context.Background(), name) + Expect(err).NotTo(HaveOccurred()) + Expect(meta.Sealed).To(BeTrue()) + } + identities := env.Rows(info) + Expect(identities).To(ContainElement(HaveKeyWithValue("pid", BeNumerically("==", os.Getpid())))) + Expect(env.Results.Backend.Scan(context.Background(), stream+"-metrics", 0, func(_ int64, row recordstore.Row) error { + Expect(row).To(HaveKey("rssBytes")) + Expect(row).NotTo(HaveKey("command")) + return nil + })).To(Succeed()) + top, err := read.Top(context.Background(), read.Ref{Runtime: env.Runtime, TraceRef: tracing.TraceRef{Kind: processcapture.Kind, Stream: stream}}, read.TopQuery{Query: read.Query{Filters: []string{fmt.Sprintf("pid=%d", os.Getpid())}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(top.Rows).To(HaveLen(1)) + Expect(top.Rows[0]["pid"]).To(BeNumerically("==", os.Getpid())) + Expect(top.Rows[0]["command"]).NotTo(BeEmpty()) + Expect(top.Display.Columns[0].Name).To(Equal("pid")) + Expect(top.Display.RowDetail).To(HaveLen(1)) + Expect(top.Display.RowDetail[0]).NotTo(BeNil()) + }) +}) diff --git a/pkg/tracing/processcapture/latest.go b/pkg/tracing/processcapture/latest.go new file mode 100644 index 0000000..4033c9e --- /dev/null +++ b/pkg/tracing/processcapture/latest.go @@ -0,0 +1,119 @@ +package processcapture + +import ( + "cmp" + "context" + "encoding/json" + "fmt" + "slices" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/query/profilestore" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/telltale/pkg/tracing" +) + +type ProcessRow struct { + Key string `json:"key"` + PID int `json:"pid"` + Name string `json:"name"` + Command string `json:"command"` + ExitedAt *time.Time `json:"exitedAt,omitempty"` + SampledAt *time.Time `json:"sampledAt,omitempty"` + CPUPercent *float64 `json:"cpuPercent"` + RSSBytes *uint64 `json:"rssBytes"` + MemoryPercent *float64 `json:"memoryPercent"` +} + +func LatestProcesses(identities []Identity, metrics []MetricRow) ([]ProcessRow, error) { + rows := make([]ProcessRow, len(identities)) + index := map[string]int{} + ticks := map[string]int64{} + for i, p := range identities { + if p.Key == "" || p.PID <= 0 { + return nil, fmt.Errorf("process identity requires key and pid") + } + if _, exists := index[p.Key]; exists { + return nil, fmt.Errorf("duplicate process identity %s", p.Key) + } + index[p.Key] = i + rows[i] = ProcessRow{Key: p.Key, PID: p.PID, Name: p.Name, Command: p.Command, ExitedAt: p.ExitedAt} + } + for _, sample := range metrics { + i, ok := index[sample.Entity] + if !ok { + return nil, fmt.Errorf("metrics reference unknown process %q", sample.Entity) + } + row := &rows[i] + if row.SampledAt == nil || sample.Tick >= ticks[sample.Entity] { + row.SampledAt, row.CPUPercent, row.RSSBytes, row.MemoryPercent = &sample.At, sample.CPUPercent, &sample.RSSBytes, &sample.MemoryPercent + ticks[sample.Entity] = sample.Tick + } + } + slices.SortFunc(rows, func(a, b ProcessRow) int { + if compared := cmp.Compare(a.PID, b.PID); compared != 0 { + return compared + } + return cmp.Compare(a.Key, b.Key) + }) + return rows, nil +} + +func latestMetricsView() recordresults.ResultView { + return recordresults.ResultView{Name: "latest", Title: "Latest process measurements", + Query: `SELECT entity, observedAt, tick, cpuPercent, rssBytes, memoryPercent, MAX(seq) AS seq FROM stream_rows GROUP BY entity`, + Columns: []query.ColumnDef{ + {Name: "entity", Type: query.ColumnTypeString}, {Name: "observedAt", Type: query.ColumnTypeDateTime}, + {Name: "tick", Type: query.ColumnTypeNumber}, {Name: "cpuPercent", Type: query.ColumnTypeNumber}, + {Name: "rssBytes", Type: query.ColumnTypeNumber}, {Name: "memoryPercent", Type: query.ColumnTypeNumber}, {Name: "seq", Type: query.ColumnTypeNumber}, + }, Order: query.Order{{Column: "entity", Unique: true}}} +} + +func ReadLatest(ctx context.Context, runtime *tracing.Runtime, ref tracing.TraceRef) ([]ProcessRow, error) { + if ref.Kind != Kind { + return nil, fmt.Errorf("process snapshot requires a process capture") + } + meta, err := runtime.Results.Backend.Meta(ctx, ref.Stream+"-metrics") + if err != nil { + return nil, err + } + var identities []Identity + if err := tracing.ReadRecords(ctx, runtime, tracing.TraceRef{Kind: Kind, Stream: ref.Stream}, func(p Identity) error { + identities = append(identities, p) + return nil + }); err != nil { + return nil, err + } + if meta.HighSeq == 0 { + return LatestProcesses(identities, nil) + } + profile, err := runtime.Results.Registry.Get(ctx, "traces/"+MetricsTable+"/latest") + if err != nil { + return nil, err + } + params := (tracing.TraceRef{Kind: MetricsTable, Stream: ref.Stream + "-metrics", ToSeq: meta.HighSeq}).Params() + release, err := runtime.Results.Registry.BeforeExecute(ctx, []profilestore.ReadRequest{{Profile: profile, Params: params}}) + if err != nil { + return nil, err + } + defer release() + dbctx := dbcontext.NewContext(ctx).WithConnectionResolver(runtime.Results.Registry.ResolveConnection) + var metrics []MetricRow + for row, err := range query.Rows(query.ExecutePages(dbctx, profile, query.PageRequest{Limit: 500, Strategy: query.PagingCursor}, params)) { + if err != nil { + return nil, err + } + data, err := json.Marshal(row) + if err != nil { + return nil, err + } + var sample MetricRow + if err := json.Unmarshal(data, &sample); err != nil { + return nil, err + } + metrics = append(metrics, sample) + } + return LatestProcesses(identities, metrics) +} diff --git a/pkg/tracing/processcapture/latest_test.go b/pkg/tracing/processcapture/latest_test.go new file mode 100644 index 0000000..808023c --- /dev/null +++ b/pkg/tracing/processcapture/latest_test.go @@ -0,0 +1,32 @@ +package processcapture + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("latest process header", func() { + It("retains each identity and its last sample including exits and reused PIDs", func() { + at := time.Unix(100, 0).UTC() + exited := at.Add(time.Second) + cpu := 125.0 + rows, err := LatestProcesses([]Identity{ + {Key: "7@new", PID: 7, Name: "new worker"}, + {Key: "7@old", PID: 7, Name: "old worker", ExitedAt: &exited}, + }, []MetricRow{ + {Entity: "7@old", At: at, Tick: 1, RSSBytes: 1024, MemoryPercent: 2}, + {Entity: "7@old", At: exited, Tick: 2, CPUPercent: &cpu, RSSBytes: 2048, MemoryPercent: 4}, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(rows).To(ConsistOf( + And(HaveField("Key", "7@old"), HaveField("ExitedAt", Equal(&exited)), HaveField("CPUPercent", Equal(&cpu)), HaveField("RSSBytes", HaveValue(Equal(uint64(2048))))), + And(HaveField("Key", "7@new"), HaveField("CPUPercent", BeNil()), HaveField("RSSBytes", BeNil())), + )) + }) + It("rejects measurements without their recorded identity", func() { + _, err := LatestProcesses(nil, []MetricRow{{Entity: "missing"}}) + Expect(err).To(MatchError(ContainSubstring("unknown process"))) + }) +}) diff --git a/pkg/tracing/processcapture/observe.go b/pkg/tracing/processcapture/observe.go new file mode 100644 index 0000000..4f0154a --- /dev/null +++ b/pkg/tracing/processcapture/observe.go @@ -0,0 +1,173 @@ +package processcapture + +import ( + "context" + "fmt" + "path/filepath" + "strconv" + "strings" + "time" + + clickyprocess "github.com/flanksource/clicky/process" + "github.com/flanksource/commons-db/recordstore/metrics" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/groups" + "github.com/flanksource/telltale/pkg/tracing/process" +) + +type MetricRow struct { + Entity string `json:"entity" filter:"exact"` + At time.Time `json:"observedAt"` + Tick int64 `json:"tick"` + CPUPercent *float64 `json:"cpuPercent" pretty:"unit=percent"` + RSSBytes uint64 `json:"rssBytes" pretty:"unit=bytes"` + MemoryPercent float64 `json:"memoryPercent" pretty:"unit=percent"` + DiskReadBytes *uint64 `json:"diskReadBytes" pretty:"unit=bytes"` + DiskWriteBytes *uint64 `json:"diskWriteBytes" pretty:"unit=bytes"` + OpenFiles *int `json:"openFiles"` + ReceivingSockets *int `json:"receivingSockets"` + ConnectionsIn *int `json:"connectionsIn"` + ConnectionsOut *int `json:"connectionsOut"` + ConnectionsUnknown *int `json:"connectionsUnknown"` + DiskReadBps *float64 `json:"diskReadBps" pretty:"unit=Bps"` + DiskWriteBps *float64 `json:"diskWriteBps" pretty:"unit=Bps"` + NetRxBps *float64 `json:"netRxBps" pretty:"unit=Bps"` + NetTxBps *float64 `json:"netTxBps" pretty:"unit=Bps"` + SwapBytes *uint64 `json:"swapBytes" pretty:"unit=bytes"` + CompressedBytes *uint64 `json:"compressedBytes" pretty:"unit=bytes"` +} + +func (r *capture) observe(ctx context.Context) error { + r.mu.Lock() + defer r.mu.Unlock() + snapshot, err := r.top.Snapshot(ctx) + if err != nil { + return err + } + o := Observation{At: snapshot.Frame.SampledAt, Tick: r.store.tick + 1} + keys := map[int]string{} + for _, p := range snapshot.Observed { + keys[p.PID] = process.Key(p.Process) + } + root := r.params.root() + r.rootGone = root != 0 && keys[root] == "" + tags := groupingTags(snapshot) + paths := groupPaths(snapshot) + usage := map[string]groups.Usage{} + for _, p := range snapshot.Processes { + usage[p.Key] = p.Usage + } + var fresh []clickyprocess.Process + var pids []int + for _, p := range snapshot.Observed { + if _, ok := r.store.identities[keys[p.PID]]; !ok { + fresh = append(fresh, p.Process) + pids = append(pids, p.PID) + } + } + startup := clickyprocess.NewSnapshot(fresh) + startup.PopulateWorkingDirectories(ctx, pids) + for _, p := range snapshot.Observed { + key := keys[p.PID] + if _, ok := r.store.identities[key]; !ok { + initial, _ := startup.Get(p.PID) + fields := strings.Fields(p.Command) + if len(fields) == 0 { + return fmt.Errorf("process %s has no command", key) + } + identity := Identity{Key: key, ParentKey: keys[p.PPID], PID: p.PID, PPID: p.PPID, Name: filepath.Base(fields[0]), Command: p.Command, CWD: initial.CWD, FirstObservedAt: o.At, StartedAt: o.At, Tags: tags[key], Groups: paths[key]} + if p.StartedAt != nil { + identity.StartedAt = *p.StartedAt + } + if r.dumpers != nil { + identity.StackTransports = r.dumpers.Transports(p.PID) + } + if r.params.Environment != nil { + environment, err := clickyprocess.Environment(ctx, p.PID, clickyprocess.EnvironmentOptions{Keys: r.params.Environment.Keys, Prefixes: r.params.Environment.Prefixes}) + if err != nil { + identity.EnvironmentError = err.Error() + } else { + identity.Environment = tracing.MaskEnvironment(environment) + } + } + o.Processes = append(o.Processes, identity) + } + values := map[string]any{"rssBytes": p.RSSBytes, "memoryPercent": p.MemoryPercent, "cpuPercent": nil, "diskReadBytes": nil, "diskWriteBytes": nil} + if p.IntervalCPUPercent != nil { + values["cpuPercent"] = *p.IntervalCPUPercent + } + if reading := snapshot.IO[p.PID]; reading.IO != nil { + values["diskReadBytes"], values["diskWriteBytes"] = reading.IO.DiskReadBytes, reading.IO.DiskWriteBytes + } + u := usage[key] + for name, value := range map[string]*float64{"diskReadBps": u.DiskRead, "diskWriteBps": u.DiskWrite, "netRxBps": u.NetRx, "netTxBps": u.NetTx} { + values[name] = nil + if value != nil { + values[name] = *value + } + } + for name, value := range map[string]*uint64{"swapBytes": u.SwapBytes, "compressedBytes": u.CompressedBytes} { + values[name] = nil + if value != nil { + values[name] = *value + } + } + o.Metrics = append(o.Metrics, metrics.Sample{Entity: key, Values: values}) + if len(r.params.Facets) > 0 { + held, complete := snapshot.Descriptors[p.PID] + o.Facets = append(o.Facets, FacetObservation{Process: key, Complete: complete, Descriptors: held, Enabled: r.params.Facets}) + } + } + // A process outside the selected subtree may have been reparented. Only a + // successful host listing can establish that its identity is no longer alive. + all, _, err := r.sampler.Observe(ctx, 0) + if err != nil { + return err + } + alive := map[string]bool{} + for _, p := range all { + alive[process.Key(p.Process)] = true + } + for key, identity := range r.store.identities { + if identity.ExitedAt == nil && !alive[key] { + identity.ExitedAt = &o.At + o.Processes = append(o.Processes, identity) + o.Facets = append(o.Facets, FacetObservation{Process: key, Exited: true}) + } + } + return r.store.Apply(ctx, o) +} + +func groupPaths(snapshot groups.Snapshot) map[string][]GroupInfo { + byKey := map[string]groups.GroupSample{} + for _, g := range snapshot.Frame.Groups { + byKey[g.Key] = g + } + paths := map[string][]GroupInfo{} + for _, p := range snapshot.Processes { + for key := p.GroupKey; key != ""; { + g, ok := byKey[key] + if !ok { + break + } + paths[p.Key] = append(paths[p.Key], GroupInfo{Key: g.Key, ParentKey: g.ParentKey, Name: g.Name, Kind: g.Kind, Target: g.Target, PID: g.PID, Command: g.Command, Context: g.Context, Origin: g.Origin, SkipAncestry: g.SkipAncestry}) + key = g.ParentKey + } + } + return paths +} + +func groupingTags(snapshot groups.Snapshot) map[string]map[string]string { + byGroup := map[string]groups.GroupSample{} + for _, group := range snapshot.Frame.Groups { + byGroup[group.Key] = group + } + result := map[string]map[string]string{} + for _, p := range snapshot.Processes { + g := byGroup[p.GroupKey] + result[p.Key] = map[string]string{"group": g.Key, "groupParent": g.ParentKey, "groupName": g.Name, "groupKind": g.Kind, "groupPID": strconv.Itoa(g.PID), "groupCommand": g.Command, "target": g.Target, + "owner": p.Origin.Owner, "scope": p.Origin.Scope, "source": p.Origin.Source, "service": p.Origin.Service, + "repo": g.Context.Repo, "branch": g.Context.Branch, "cwd": g.Context.CWD, "agent": g.Context.Agent, "session": g.Context.Session, "workspace": g.Context.Workspace, "surface": g.Context.Surface} + } + return result +} diff --git a/pkg/tracing/processcapture/ownership.go b/pkg/tracing/processcapture/ownership.go new file mode 100644 index 0000000..f084e77 --- /dev/null +++ b/pkg/tracing/processcapture/ownership.go @@ -0,0 +1,59 @@ +package processcapture + +import ( + "encoding/json" + "fmt" + "slices" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore/sampleframe" +) + +func ownershipHierarchy(rows []query.Row, metric string) (*sampleframe.Hierarchy, error) { + if metric != "cpuPercent" && metric != "rssBytes" { + return nil, fmt.Errorf("unknown process metric %q", metric) + } + stacks := []sampleframe.WeightedStack{} + for _, row := range rows { + if row[metric] == nil { + continue + } + encoded, err := json.Marshal(row[metric]) + if err != nil { + return nil, err + } + var value float64 + if err := json.Unmarshal(encoded, &value); err != nil { + return nil, err + } + encoded, err = json.Marshal(row["groups"]) + if err != nil { + return nil, err + } + var owners []GroupInfo + if err := json.Unmarshal(encoded, &owners); err != nil { + return nil, err + } + stack := sampleframe.WeightedStack{Value: value} + for _, owner := range slices.Backward(owners) { + stack.Frames = append(stack.Frames, owner.Name) + stack.IDs = append(stack.IDs, owner.Key) + } + stack.Frames = append(stack.Frames, fmt.Sprintf("%v (pid %v)", row["name"], row["pid"])) + stack.IDs = append(stack.IDs, fmt.Sprint(row["entity"])) + stacks = append(stacks, stack) + } + if len(stacks) == 0 { + return nil, nil + } + root, err := sampleframe.BuildHierarchy(stacks) + if err != nil { + return nil, err + } + root.Label = "Recorded process usage" + unit := "percent" + if metric == "rssBytes" { + unit = "bytes" + } + return &sampleframe.Hierarchy{Metric: metric, Unit: unit, Root: root}, nil +} diff --git a/pkg/tracing/processcapture/pretty.go b/pkg/tracing/processcapture/pretty.go new file mode 100644 index 0000000..95ba672 --- /dev/null +++ b/pkg/tracing/processcapture/pretty.go @@ -0,0 +1,58 @@ +package processcapture + +import ( + "fmt" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" +) + +func (r Identity) Pretty() api.Text { + t := clicky.Text(fmt.Sprintf("pid %d", r.PID), "text-muted").Space().Append(r.Name, "text-blue-500 font-bold").Space().Append(r.Command, "font-mono") + if r.ExitedAt != nil { + t = t.Space().Append("exited", "text-yellow-500") + } + if r.EnvironmentError != "" { + t = t.Space().Append(r.EnvironmentError, "text-red-500") + } + return t +} + +func (r Identity) PrettyFull() api.Textable { + return r.Pretty().NewLine().Add(clicky.Map(r.Tags)).NewLine().Add(clicky.Map(map[string]any{"groups": r.Groups, "cwd": r.CWD, "parent": r.ParentKey})) +} + +func (r MetricRow) Pretty() api.Text { + t := clicky.Text(r.Entity, "text-muted") + if r.CPUPercent != nil { + t = t.Space().Append(fmt.Sprintf("CPU %.1f%%", *r.CPUPercent), "text-blue-500") + } + t = t.Space().Append("RSS ", "text-muted").Add(api.HumanizeBytes(int64(r.RSSBytes))) + if r.OpenFiles != nil { + t = t.Space().Append(fmt.Sprintf("%d files", *r.OpenFiles), "text-muted") + } + return t +} + +func (r MetricRow) PrettyFull() api.Textable { + return r.Pretty().NewLine().Add(clicky.Map(map[string]any{ + "CPU %": r.CPUPercent, "RSS": api.HumanizeBytes(int64(r.RSSBytes)), "Files": r.OpenFiles, + "Disk read bytes": r.DiskReadBytes, "Disk write bytes": r.DiskWriteBytes, + "Disk read B/s": r.DiskReadBps, "Disk write B/s": r.DiskWriteBps, + "Network receive B/s": r.NetRxBps, "Network send B/s": r.NetTxBps, + "Connections in": r.ConnectionsIn, "Connections out": r.ConnectionsOut, + })) +} + +func (r FacetEvent) Pretty() api.Text { + name := r.Entity + if r.Resource.Name != "" { + name = r.Resource.Name + } + return clicky.Text(r.Process, "text-muted").Space().Append(r.Facet, "text-blue-500"). + Space().Append(r.Operation, "text-yellow-500").Space().Append(name, "font-mono") +} + +func (r FacetEvent) PrettyFull() api.Textable { + return r.Pretty().NewLine().Add(clicky.Map(map[string]any{"resource": r.Resource, "reason": r.Reason, "directionSource": r.DirectionSource})) +} diff --git a/pkg/tracing/processcapture/projection.go b/pkg/tracing/processcapture/projection.go new file mode 100644 index 0000000..4c98b0a --- /dev/null +++ b/pkg/tracing/processcapture/projection.go @@ -0,0 +1,56 @@ +package processcapture + +import "github.com/flanksource/telltale/pkg/tracing/process" + +// Nodes projects startup identities and numeric history into a current tree. +func Nodes(identities []Identity, metrics []MetricRow) []process.Node { + index := map[string]int{} + nodes := make([]process.Node, 0, len(identities)) + for _, p := range identities { + status := "running" + if p.ExitedAt != nil { + status = "exited" + } + index[p.Key] = len(nodes) + nodes = append(nodes, process.Node{Key: p.Key, ParentKey: p.ParentKey, PID: p.PID, PPID: p.PPID, Name: p.Name, Command: p.Command, CWD: p.CWD, Status: status, + StartedAt: p.StartedAt, ExitedAt: p.ExitedAt, LastSeenAt: p.FirstObservedAt, IsRoot: p.ParentKey == "", Environment: p.Environment, EnvironmentError: p.EnvironmentError, StackTransports: p.StackTransports}) + } + for _, sample := range metrics { + i, ok := index[sample.Entity] + if !ok { + continue + } + n := &nodes[i] + if !sample.At.Before(n.LastSeenAt) { + n.CPUPercent, n.RSSBytes, n.LastSeenAt = sample.CPUPercent, sample.RSSBytes, sample.At + } + n.PeakRSSBytes = max(n.PeakRSSBytes, sample.RSSBytes) + if sample.CPUPercent != nil && (n.PeakCPUPercent == nil || *sample.CPUPercent > *n.PeakCPUPercent) { + n.PeakCPUPercent = sample.CPUPercent + } + } + return nodes +} + +// Samples joins each metric row to its process identity for SDK consumers. +func Samples(identities []Identity, metrics []MetricRow) []process.Sample { + byKey := map[string]Identity{} + for _, p := range identities { + byKey[p.Key] = p + } + samples := make([]process.Sample, 0, len(metrics)) + for _, row := range metrics { + p, ok := byKey[row.Entity] + if !ok { + continue + } + var connections *int + if row.ConnectionsIn != nil && row.ConnectionsOut != nil && row.ConnectionsUnknown != nil { + total := *row.ConnectionsIn + *row.ConnectionsOut + *row.ConnectionsUnknown + connections = &total + } + samples = append(samples, process.Sample{SampledAt: row.At, Tick: row.Tick, Key: row.Entity, PID: p.PID, CPUPercent: row.CPUPercent, MemoryPercent: row.MemoryPercent, RSSBytes: row.RSSBytes, + DiskReadBytes: row.DiskReadBytes, DiskWriteBytes: row.DiskWriteBytes, OpenFiles: row.OpenFiles, Ports: row.ReceivingSockets, Sockets: row.ReceivingSockets, Connections: connections}) + } + return samples +} diff --git a/pkg/tracing/processcapture/projection_test.go b/pkg/tracing/processcapture/projection_test.go new file mode 100644 index 0000000..7fc530a --- /dev/null +++ b/pkg/tracing/processcapture/projection_test.go @@ -0,0 +1,26 @@ +package processcapture + +import ( + "github.com/flanksource/telltale/pkg/tracing/groups" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "time" +) + +var _ = Describe("derived process groups", func() { + It("rebuilds self and descendant totals without storing group snapshots", func() { + at := time.Unix(100, 0) + root := GroupInfo{Key: "tools", Name: "tools", Kind: groups.KindTool} + child := GroupInfo{Key: "tests", ParentKey: root.Key, Name: "tests", Kind: groups.KindTest} + identities := []Identity{{Key: "1", PID: 1, Name: "runner", Groups: []GroupInfo{root}}, {Key: "2", PID: 2, Name: "worker", Groups: []GroupInfo{child, root}}} + cpu := 12.5 + rows := []MetricRow{{Entity: "1", At: at, Tick: 1, RSSBytes: 1024}, {Entity: "2", At: at, Tick: 1, RSSBytes: 2048, CPUPercent: &cpu}} + snapshot, err := GroupSnapshot(identities, rows) + Expect(err).NotTo(HaveOccurred()) + Expect(snapshot.Frame.Groups).To(HaveExactElements( + And(HaveField("Key", root.Key), HaveField("Self", And(HaveField("Processes", 1), HaveField("RSSBytes", uint64(1024)))), HaveField("Total", And(HaveField("Processes", 2), HaveField("RSSBytes", uint64(3072)), HaveField("CPUPercent", Equal(&cpu))))), + And(HaveField("Key", child.Key), HaveField("Total", And(HaveField("Processes", 1), HaveField("RSSBytes", uint64(2048))))), + )) + Expect(snapshot.Processes).To(HaveLen(2)) + }) +}) diff --git a/pkg/tracing/processcapture/store.go b/pkg/tracing/processcapture/store.go new file mode 100644 index 0000000..07036f8 --- /dev/null +++ b/pkg/tracing/processcapture/store.go @@ -0,0 +1,230 @@ +package processcapture + +import ( + "context" + "fmt" + "maps" + "reflect" + "slices" + "sync" + "time" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/commons-db/recordstore/metrics" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/groups" + "github.com/flanksource/telltale/pkg/tracing/process" + sdk "github.com/flanksource/telltale/sdk" +) + +const Kind = "process" +const FacetsTable = "process_facets" +const MetricsTable = "process_metrics" + +type Identity struct { + Key string `json:"key"` + ParentKey string `json:"parentKey,omitempty"` + PID int `json:"pid" filter:"terms"` + PPID int `json:"ppid" filter:"terms"` + Name string `json:"name"` + Command string `json:"command"` + CWD string `json:"cwd,omitempty"` + StartedAt time.Time `json:"startedAt"` + FirstObservedAt time.Time `json:"firstObservedAt"` + ExitedAt *time.Time `json:"exitedAt,omitempty"` + Environment map[string]string `json:"environment,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + EnvironmentError string `json:"environmentError,omitempty"` + Groups []GroupInfo `json:"groups,omitempty"` + StackTransports []process.StackTransport `json:"stackTransports,omitempty"` +} + +type GroupInfo struct { + Key string `json:"key"` + ParentKey string `json:"parentKey,omitempty"` + Name string `json:"name"` + Kind string `json:"kind"` + Target string `json:"target,omitempty"` + PID int `json:"pid"` + Command string `json:"command,omitempty"` + Context groups.Context `json:"context"` + Origin groups.Origin `json:"origin"` + SkipAncestry bool `json:"skipAncestry,omitempty"` +} + +type Observation struct { + At time.Time + Tick int64 + Processes []Identity + Facets []FacetObservation + Metrics []metrics.Sample +} + +type StoreOptions struct { + Backend recordstore.BatchAppender + Stream string +} + +type Store struct { + metricSeq int64 + mu sync.Mutex + backend recordstore.BatchAppender + stream string + metrics *metrics.DynamicMetricTable + facets *FacetState + tick int64 + identities map[string]Identity +} + +func NewStore(options StoreOptions) (*Store, error) { + if options.Backend == nil { + return nil, fmt.Errorf("process capture requires an atomic batch backend") + } + if err := recordstore.ValidateStream(options.Stream); err != nil { + return nil, err + } + table, err := metricTable() + if err != nil { + return nil, err + } + return &Store{backend: options.Backend, stream: options.Stream, metrics: table, facets: NewFacetState(), identities: map[string]Identity{}}, nil +} + +func metricTable() (*metrics.DynamicMetricTable, error) { + return metrics.NewDynamicMetricTable(metrics.TableOptions{Kind: MetricsTable, Metrics: []metrics.Definition{ + {Name: "cpuPercent", Unit: "percent"}, {Name: "rssBytes", Unit: "bytes"}, {Name: "memoryPercent", Unit: "percent"}, + {Name: "diskReadBytes", Unit: "bytes", Counter: true}, {Name: "diskWriteBytes", Unit: "bytes", Counter: true}, + {Name: "openFiles"}, {Name: "receivingSockets"}, {Name: "connectionsIn"}, {Name: "connectionsOut"}, {Name: "connectionsUnknown"}, + {Name: "diskReadBps", Unit: "Bps"}, {Name: "diskWriteBps", Unit: "Bps"}, {Name: "netRxBps", Unit: "Bps"}, {Name: "netTxBps", Unit: "Bps"}, + {Name: "swapBytes", Unit: "bytes"}, {Name: "compressedBytes", Unit: "bytes"}, + }}) +} + +func Schemas() ([]recordstore.KindSchema, error) { + identity, err := query.ColumnsFor(reflect.TypeFor[Identity]()) + if err != nil { + return nil, err + } + facets, err := query.ColumnsFor(reflect.TypeFor[FacetEvent]()) + if err != nil { + return nil, err + } + table, err := metricTable() + if err != nil { + return nil, err + } + return []recordstore.KindSchema{ + {Kind: Kind, Columns: identity, Options: recordstore.KindOptions{Key: "key", OnConflict: recordstore.OnConflictReplace, TimeColumn: "firstObservedAt"}}, + {Kind: FacetsTable, Columns: facets, Options: recordstore.KindOptions{TimeColumn: "observedAt", Indexes: []recordstore.IndexDef{{Columns: []string{"process", "facet", "entity", "tick"}}}}}, + table.Schema(), + }, nil +} + +func (s *Store) Apply(ctx context.Context, o Observation) error { + s.mu.Lock() + defer s.mu.Unlock() + if o.At.IsZero() || o.Tick != s.tick+1 { + return fmt.Errorf("process observation must advance tick %d to %d with a timestamp", s.tick, s.tick+1) + } + schemas, err := Schemas() + if err != nil { + return err + } + batch := recordstore.Batch{ID: fmt.Sprintf("%s-%d", s.stream, o.Tick), Atomic: true, Producer: recordstore.Producer{Instance: s.stream, Seq: o.Tick}, Schemas: schemas[:2]} + identities, rows, err := s.processRows(o.Processes) + if err != nil { + return err + } + batch.Entries = append(batch.Entries, recordstore.BatchEntry{Op: recordstore.BatchAppend, Kind: Kind, Stream: s.stream, Rows: rows}) + next := s.facets.clone() + counts := map[string]map[string]any{} + events := []recordstore.Row{} + for _, facet := range o.Facets { + if _, ok := identities[facet.Process]; !ok { + return fmt.Errorf("facets reference unknown process %q", facet.Process) + } + facet.At, facet.Tick = o.At, o.Tick + changes := next.diff(facet) + if facet.Complete { + counts[facet.Process] = next.counts(facet) + } + for _, change := range changes { + row, err := recordstore.EncodeRow(change) + if err != nil { + return err + } + events = append(events, row) + } + } + batch.Entries = append(batch.Entries, recordstore.BatchEntry{Op: recordstore.BatchAppend, Kind: FacetsTable, Stream: s.stream + "-facets", Rows: events}) + source := metrics.DynamicMetricSource{Table: s.metrics, Stream: s.stream + "-metrics"} + frameSchema, err := sampleframe.Schema(Kind + "_frames") + if err != nil { + return err + } + frame, err := recordstore.EncodeRow(sampleframe.Frame{ID: fmt.Sprintf("%s-%d", s.stream, o.Tick), Source: s.stream, Result: "metrics", Tick: o.Tick, StartedAt: o.At, CompletedAt: o.At, AfterSeq: s.metricSeq, ToSeq: s.metricSeq + int64(len(o.Metrics)), Observed: len(o.Metrics), Retained: len(o.Metrics)}) + if err != nil { + return err + } + batch.Schemas = append(batch.Schemas, frameSchema) + batch.Entries = append(batch.Entries, recordstore.BatchEntry{Op: recordstore.BatchAppend, Kind: frameSchema.Kind, Stream: s.stream + "-frames", Rows: []recordstore.Row{frame}}) + samples := slices.Clone(o.Metrics) + for i := range samples { + if _, ok := identities[samples[i].Entity]; !ok { + return fmt.Errorf("metrics reference unknown process %q", samples[i].Entity) + } + samples[i].At, samples[i].Tick = o.At, o.Tick + values := map[string]any{} + maps.Copy(values, samples[i].Values) + samples[i].Values = values + maps.Copy(samples[i].Values, counts[samples[i].Entity]) + } + if _, err := source.Append(ctx, metrics.AppendOptions{Backend: s.backend, Batch: batch, Samples: samples}); err != nil { + return err + } + s.identities, s.facets, s.tick = identities, next, o.Tick + s.metricSeq += int64(len(samples)) + return nil +} + +func (s *Store) processRows(processes []Identity) (map[string]Identity, []recordstore.Row, error) { + identities := maps.Clone(s.identities) + rows := []recordstore.Row{} + for _, process := range processes { + if process.Key == "" || process.PID <= 0 || process.FirstObservedAt.IsZero() { + return nil, nil, fmt.Errorf("process identity requires key, pid and first observation time") + } + if previous, ok := identities[process.Key]; ok { + if previous.PID != process.PID { + return nil, nil, fmt.Errorf("process identity %s changed pid", process.Key) + } + if previous.ExitedAt != nil || process.ExitedAt == nil { + continue + } + previous.ExitedAt = process.ExitedAt + process = previous + } + process.Command = sdk.Mask(process.Command) + process.Environment = tracing.MaskEnvironment(process.Environment) + process.Groups = slices.Clone(process.Groups) + process.StackTransports = slices.Clone(process.StackTransports) + if process.ExitedAt != nil { + exited := *process.ExitedAt + process.ExitedAt = &exited + } + for i := range process.Groups { + process.Groups[i].Command = sdk.Mask(process.Groups[i].Command) + } + row, err := recordstore.EncodeRow(process) + if err != nil { + return nil, nil, err + } + process.Environment = maps.Clone(process.Environment) + process.Tags = maps.Clone(process.Tags) + identities[process.Key] = process + rows = append(rows, row) + } + return identities, rows, nil +} diff --git a/pkg/tracing/processcapture/store_test.go b/pkg/tracing/processcapture/store_test.go new file mode 100644 index 0000000..01ac0cf --- /dev/null +++ b/pkg/tracing/processcapture/store_test.go @@ -0,0 +1,72 @@ +package processcapture + +import ( + "context" + "fmt" + clickyprocess "github.com/flanksource/clicky/process" + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/commons-db/recordstore/metrics" + "github.com/flanksource/commons-db/recordstore/sqlite" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "path/filepath" + "time" +) + +type refusingBackend struct { + recordstore.BatchAppender + refuse bool +} + +func (b *refusingBackend) AppendBatch(ctx context.Context, batch recordstore.Batch) (recordstore.BatchResult, error) { + if b.refuse { + return recordstore.BatchResult{}, fmt.Errorf("storage unavailable") + } + return b.BatchAppender.AppendBatch(ctx, batch) +} + +var _ = Describe("process observation storage", func() { + It("stores independent tables atomically and retries deltas without advancing state after failure", func() { + ctx := context.Background() + backend, err := sqlite.Open(sqlite.Options{Path: filepath.Join(GinkgoT().TempDir(), "records.sqlite"), Schema: func(kind string) (recordstore.KindSchema, error) { + schemas, err := Schemas() + if err != nil { + return recordstore.KindSchema{}, err + } + for _, schema := range schemas { + if schema.Kind == kind { + return schema, nil + } + } + return recordstore.KindSchema{}, fmt.Errorf("undeclared %s", kind) + }, SweepInterval: time.Hour}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(backend.Close) + writer := &refusingBackend{BatchAppender: backend, refuse: true} + store, err := NewStore(StoreOptions{Backend: writer, Stream: "capture"}) + Expect(err).NotTo(HaveOccurred()) + at := time.Unix(1, 0) + o := Observation{At: at, Tick: 1, Processes: []Identity{{Key: "p", PID: 1, FirstObservedAt: at, StartedAt: at, Command: "worker", Environment: map[string]string{"MODE": "test"}}}, Facets: []FacetObservation{{Process: "p", Complete: true, Descriptors: []clickyprocess.Descriptor{{FD: "1", Type: "REG", Name: "data"}}}}, Metrics: []metrics.Sample{{Entity: "p", Values: map[string]any{"rssBytes": uint64(1024), "customMetric": 2}}}} + Expect(store.Apply(ctx, o)).NotTo(Succeed()) + Expect(store.tick).To(BeZero()) + Expect(store.identities).To(BeEmpty()) + writer.refuse = false + Expect(store.Apply(ctx, o)).To(Succeed()) + for _, stream := range []string{"capture", "capture-facets", "capture-metrics"} { + meta, err := backend.Meta(ctx, stream) + Expect(err).NotTo(HaveOccurred()) + Expect(meta.Total).To(Equal(int64(1))) + } + o.Tick = 2 + o.At = at.Add(time.Second) + o.Processes[0].Command = "must not rewrite startup metadata" + Expect(store.Apply(ctx, o)).To(Succeed()) + meta, err := backend.Meta(ctx, "capture-facets") + Expect(err).NotTo(HaveOccurred()) + Expect(meta.Total).To(Equal(int64(1))) + Expect(backend.Scan(ctx, "capture", 0, func(_ int64, row recordstore.Row) error { Expect(row["command"]).To(Equal("worker")); return nil })).To(Succeed()) + meta, err = backend.Meta(ctx, "capture-metrics") + Expect(err).NotTo(HaveOccurred()) + Expect(meta.Total).To(Equal(int64(2))) + }) +}) diff --git a/pkg/tracing/processcapture/table.go b/pkg/tracing/processcapture/table.go new file mode 100644 index 0000000..47536e0 --- /dev/null +++ b/pkg/tracing/processcapture/table.go @@ -0,0 +1,25 @@ +package processcapture + +import ( + "github.com/flanksource/clicky/api" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/telltale/pkg/tracing" +) + +func topSpec() tracing.TopSpec { + return tracing.TopSpec{Profile: sampleProfile, Hierarchy: ownershipHierarchy, Metrics: []tracing.TopMetric{{Key: "cpuPercent", Label: "Process CPU", Unit: "percent", View: "flamegraph"}, {Key: "rssBytes", Label: "Process resident memory", Unit: "bytes", View: "treemap"}}, Result: "metrics", Identity: []string{"entity"}, Order: query.Order{{Column: "cpuPercent", Desc: true}, {Column: "entity"}}} +} +func (Identity) Columns() []api.ColumnDef { + return query.TableColumnsFor[Identity]("pid", "name", "command") +} +func (r Identity) Row() map[string]any { return query.TableRowFor(r) } +func (r Identity) RowDetail() api.Textable { return r.PrettyFull() } +func (MetricRow) Columns() []api.ColumnDef { + return query.TableColumnsFor[MetricRow]("entity", "cpuPercent", "rssBytes", "openFiles", "diskReadBps", "diskWriteBps") +} +func (r MetricRow) Row() map[string]any { return query.TableRowFor(r) } +func (r MetricRow) RowDetail() api.Textable { return r.PrettyFull() } +func (ProcessRow) Columns() []api.ColumnDef { + return query.TableColumnsFor[ProcessRow]("pid", "name", "cpuPercent", "rssBytes", "command") +} +func (r ProcessRow) Row() map[string]any { return query.TableRowFor(r) } diff --git a/pkg/tracing/processcapture/top.go b/pkg/tracing/processcapture/top.go new file mode 100644 index 0000000..dc318dd --- /dev/null +++ b/pkg/tracing/processcapture/top.go @@ -0,0 +1,66 @@ +package processcapture + +import ( + "context" + "slices" + "strings" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/query/profilestore" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/flanksource/telltale/pkg/tracing" +) + +func sampleProfile(ctx context.Context, runtime *tracing.Runtime, ref tracing.TraceRef, _ sampleframe.Frame) (query.Profile, error) { + profile, err := runtime.Results.Registry.Get(ctx, "traces/"+MetricsTable) + if err != nil { + return query.Profile{}, err + } + identities, err := runtime.Results.Registry.Get(ctx, "traces/"+Kind) + if err != nil { + return query.Profile{}, err + } + stream := strings.TrimSuffix(ref.Stream, "-metrics") + params := (tracing.TraceRef{Kind: Kind, Stream: stream}).Params() + release, err := runtime.Results.Registry.BeforeExecute(ctx, []profilestore.ReadRequest{{Profile: identities, Params: params}}) + if err != nil { + return query.Profile{}, err + } + defer release() + identitySQL := strings.NewReplacer("{{.params.stream}}", "'"+strings.ReplaceAll(stream, "'", "''")+"'", "{{.params.afterSeq}}", "0", "{{.params.toSeq}}", "9223372036854775807", "{{.params.q}}", "''", "{{.params.rootsOnly}}", "0").Replace(identities.Query) + profile.Query = `SELECT metrics.*, identities.pid, identities.name, identities.command, identities.parentKey, identities.groups FROM (` + profile.Query + `) metrics JOIN (` + identitySQL + `) identities ON identities.key = metrics.entity` + columns := []query.ColumnDef{} + for _, name := range []string{"pid", "name", "cpuPercent", "rssBytes", "command", "parentKey", "groups"} { + for _, c := range append(slices.Clone(identities.Columns), profile.Columns...) { + if c.Name == name { + columns = append(columns, c) + break + } + } + } + for _, c := range profile.Columns { + if !slices.ContainsFunc(columns, func(existing query.ColumnDef) bool { return existing.Name == c.Name }) { + columns = append(columns, c) + } + } + profile.Columns, profile.Presenter = columns, nil + profile.Query = "SELECT * FROM (" + profile.Query + ") WHERE {{.params.q}} = '' OR instr(lower(name || ' ' || command), lower({{.params.q}})) > 0" + profile.Params = append(profile.Params, query.ParamDef{Name: "q", Label: "Search", Role: query.ParamRoleSearch, Default: ""}) + presented, err := query.ClickyColumns(profile) + if err != nil { + return query.Profile{}, err + } + for i := range presented { + presented[i].DefaultHidden = i >= 5 + } + profile.Presenter = samplePresenter{columns: presented} + return profile, nil +} + +type samplePresenter struct{ columns []api.ColumnDef } + +func (p samplePresenter) Columns() []api.ColumnDef { return p.columns } +func (samplePresenter) Present(row query.Row) (map[string]any, error) { return row, nil } +func (samplePresenter) Detail(row query.Row) (api.Textable, error) { return clicky.Map(row), nil } diff --git a/pkg/tracing/profile_publication_internal_test.go b/pkg/tracing/profile_publication_internal_test.go new file mode 100644 index 0000000..a3cc8db --- /dev/null +++ b/pkg/tracing/profile_publication_internal_test.go @@ -0,0 +1,111 @@ +package tracing + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "strings" + "time" + + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/commons-db/recordstore/recordstoretest" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/flanksource/commons-db/recordstore/sqlite" + "github.com/flanksource/telltale/pkg/otlp/profilepublication" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type failingProfileBackend struct { + *sqlite.Backend + fail bool +} + +func (b *failingProfileBackend) AppendBatch(ctx context.Context, batch recordstore.Batch) (recordstore.BatchResult, error) { + if b.fail { + return recordstore.BatchResult{}, errors.New("profile append unavailable") + } + return b.Backend.AppendBatch(ctx, batch) +} + +func publicationFrame(sequence int64, source string) sampleframe.Frame { + at := time.Date(2026, 11, 15, 10, 0, 0, 0, time.UTC) + return sampleframe.Frame{Source: source + "/capture", Tick: sequence, StartedAt: at, CompletedAt: at, Observed: 1, + Summary: map[string]any{"profileId": fmt.Sprintf("00000000-0000-4000-8000-%012d", sequence), "captureId": "capture", "digest": fmt.Sprintf("%064x", sequence), "complete": true, "receivedAt": "first-received"}} +} + +var _ = Describe("direct profile publication", func() { + var backend *failingProfileBackend + var output *emitter[recordstore.Row] + ctx := context.Background() + newOutput := func() *emitter[recordstore.Row] { + schema, err := recordstoretest.Schema(recordstoretest.Kind) + Expect(err).NotTo(HaveOccurred()) + return &emitter[recordstore.Row]{source: newSource("profile-publication", 10, defaultBufferBytes, "", false), process: pipeline(nil), top: &TopSpec{}, frames: &sampleframe.Writer{Backend: backend, Schema: schema, Stream: "profile-publication"}} + } + frames := func() []recordstore.Row { + var rows []recordstore.Row + Expect(backend.Scan(ctx, "profile-publication-frames", 0, func(_ int64, row recordstore.Row) error { rows = append(rows, row); return nil })).To(Succeed()) + return rows + } + BeforeEach(func() { + store, err := sqlite.Open(sqlite.Options{Path: filepath.Join(GinkgoT().TempDir(), "profiles.sqlite"), Schema: func(kind string) (recordstore.KindSchema, error) { + if strings.HasSuffix(kind, "_frames") { + return sampleframe.Schema(kind) + } + if kind == profilepublication.IdentityKind { + return profilepublication.ResultType().KindSchema() + } + return recordstoretest.Schema(kind) + }, TTL: time.Hour, SweepInterval: time.Hour}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(store.Close) + backend = &failingProfileBackend{Backend: store} + output = newOutput() + }) + + It("retains reordered history and deduplicates old retries after a writer restart without refreshing freshness", func() { + for _, sequence := range []int64{7, 9, 8} { + Expect(output.EmitFrame(ctx, publicationFrame(sequence, "java-a"), recordstoretest.SampleRows(int(sequence), int(sequence)))).To(Succeed()) + } + retry := publicationFrame(7, "java-a") + retry.Summary["receivedAt"] = "retried-later" + Expect(newOutput().EmitFrame(ctx, retry, recordstoretest.SampleRows(7, 7))).To(Succeed()) + stored := frames() + Expect(stored).To(HaveLen(3)) + Expect(stored[0]).To(HaveKeyWithValue("id", publicationFrame(7, "java-a").Summary["profileId"])) + Expect(stored[0]["summary"]).To(HaveKeyWithValue("receivedAt", "first-received")) + for index, sequence := range []int64{7, 9, 8} { + Expect(stored[index]).To(HaveKeyWithValue("tick", BeNumerically("==", sequence))) + Expect(stored[index]["summary"]).To(HaveKeyWithValue("complete", true)) + } + meta, err := backend.Meta(ctx, "profile-publication") + Expect(err).NotTo(HaveOccurred()) + Expect(meta.Total).To(Equal(int64(3))) + }) + + It("rejects UUID reuse and source-sequence conflicts before publishing rows or frames", func() { + first := publicationFrame(3, "java-a") + Expect(output.EmitFrame(ctx, first, recordstoretest.SampleRows(3, 3))).To(Succeed()) + uuidConflict := publicationFrame(4, "java-b") + uuidConflict.Summary["profileId"] = first.Summary["profileId"] + Expect(output.EmitFrame(ctx, uuidConflict, recordstoretest.SampleRows(4, 4))).To(MatchError(ContainSubstring("conflicting"))) + sequenceConflict := publicationFrame(3, "java-a") + sequenceConflict.Summary["profileId"] = publicationFrame(5, "java-a").Summary["profileId"] + sequenceConflict.Summary["digest"] = fmt.Sprintf("%064x", 5) + Expect(output.EmitFrame(ctx, sequenceConflict, recordstoretest.SampleRows(5, 5))).To(MatchError(ContainSubstring("conflicting"))) + Expect(frames()).To(HaveLen(1)) + meta, err := backend.Meta(ctx, "profile-publication") + Expect(err).NotTo(HaveOccurred()) + Expect(meta.Total).To(Equal(int64(1))) + }) + + It("admits a retry after failed atomic publication", func() { + backend.fail = true + Expect(output.EmitFrame(ctx, publicationFrame(6, "java-a"), recordstoretest.SampleRows(6, 6))).To(MatchError(ContainSubstring("profile append unavailable"))) + backend.fail = false + Expect(output.EmitFrame(ctx, publicationFrame(6, "java-a"), recordstoretest.SampleRows(6, 6))).To(Succeed()) + Expect(frames()).To(HaveLen(1)) + }) +}) diff --git a/pkg/tracing/prometheus/client.go b/pkg/tracing/prometheus/client.go index 0a459f0..ee080df 100644 --- a/pkg/tracing/prometheus/client.go +++ b/pkg/tracing/prometheus/client.go @@ -5,10 +5,8 @@ import ( "encoding/json" "fmt" "io" - "math" "net/http" "net/url" - "strconv" "strings" "time" @@ -30,16 +28,16 @@ func newClient(server Server) *client { func (c *client) get(ctx context.Context, path string, query url.Values, data any) error { request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(c.server.URL, "/")+path+"?"+query.Encode(), nil) if err != nil { - return fmt.Errorf("prometheus %s%s: %w", c.server.URL, path, err) + return fmt.Errorf("VictoriaMetrics %s%s: %w", c.server.URL, path, err) } response, err := c.http.Do(request) if err != nil { - return fmt.Errorf("prometheus %s is unreachable: %w", c.server.URL, err) + return fmt.Errorf("VictoriaMetrics %s is unreachable: %w", c.server.URL, err) } defer func() { _ = response.Body.Close() }() body, err := io.ReadAll(io.LimitReader(response.Body, 64<<20)) if err != nil { - return fmt.Errorf("prometheus %s%s: %w", c.server.URL, path, err) + return fmt.Errorf("VictoriaMetrics %s%s: %w", c.server.URL, path, err) } var answer struct { Status string `json:"status"` @@ -47,50 +45,37 @@ func (c *client) get(ctx context.Context, path string, query url.Values, data an Error string `json:"error"` Data json.RawMessage `json:"data"` } - if err := json.Unmarshal(body, &answer); err != nil || answer.Status != "success" { - return fmt.Errorf("prometheus %s%s answered %s: %s %s %.200s", c.server.URL, path, response.Status, answer.ErrorType, answer.Error, body) + if err := json.Unmarshal(body, &answer); err != nil || answer.Status != "success" || response.StatusCode != http.StatusOK { + return fmt.Errorf("VictoriaMetrics %s%s answered %s: %s %s %.200s", c.server.URL, path, response.Status, answer.ErrorType, answer.Error, body) } if err := json.Unmarshal(answer.Data, data); err != nil { - return fmt.Errorf("prometheus %s%s data: %w", c.server.URL, path, err) + return fmt.Errorf("VictoriaMetrics %s%s data: %w", c.server.URL, path, err) } return nil } -// ready waits up to wait for the server's /-/ready. -func (c *client) ready(ctx context.Context, wait time.Duration) error { - deadline := time.Now().Add(wait) - var last string - for { - request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(c.server.URL, "/")+"/-/ready", nil) - if err != nil { - return err - } - response, err := c.http.Do(request) - if err == nil { - _ = response.Body.Close() - if response.StatusCode == http.StatusOK { - return nil - } - last = response.Status - } else { - last = err.Error() - } - if time.Now().After(deadline) { - return fmt.Errorf("prometheus %s is not ready after %s: %s", c.server.URL, wait, last) - } - if err := sleep(ctx, 200*time.Millisecond); err != nil { - return err - } +func (c *client) health(ctx context.Context) error { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(c.server.URL, "/")+"/health", nil) + if err != nil { + return err } + response, err := c.http.Do(request) + if err != nil { + return fmt.Errorf("VictoriaMetrics %s is unreachable: %w", c.server.URL, err) + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK { + return fmt.Errorf("VictoriaMetrics %s is not ready: %s", c.server.URL, response.Status) + } + return nil } // scrapeTarget is one target of /api/v1/targets. type scrapeTarget struct { - Labels map[string]string `json:"labels"` - Health string `json:"health"` - LastError string `json:"lastError"` - LastScrape time.Time `json:"lastScrape"` - LastScrapeDuration float64 `json:"lastScrapeDuration"` + Labels map[string]string `json:"labels"` + Health string `json:"health"` + LastError string `json:"lastError"` + LastScrape time.Time `json:"lastScrape"` } // targets are the active targets of the server's job labelled with stream. @@ -110,43 +95,6 @@ func (c *client) targets(ctx context.Context, stream string) ([]scrapeTarget, er return targets, nil } -// point is one sample of a series. -type point struct { - at time.Time - value float64 -} - -// series are the samples of a range query by each series' endpoint label. -func (c *client) series(ctx context.Context, promql string) (map[string][]point, error) { - var data struct { - ResultType string `json:"resultType"` - Result []struct { - Metric map[string]string `json:"metric"` - Values [][2]any `json:"values"` - } `json:"result"` - } - if err := c.get(ctx, "/api/v1/query", url.Values{"query": {promql}}, &data); err != nil { - return nil, err - } - if data.ResultType != "matrix" { - return nil, fmt.Errorf("prometheus %s query %s: want a matrix, got %q", c.server.URL, promql, data.ResultType) - } - series := map[string][]point{} - for _, result := range data.Result { - endpoint := result.Metric[LabelEndpoint] - for _, value := range result.Values { - at, ok := value[0].(float64) - text, isText := value[1].(string) - parsed, err := strconv.ParseFloat(text, 64) - if !ok || !isText || err != nil { - return nil, fmt.Errorf("prometheus %s query %s: unreadable sample %v", c.server.URL, promql, value) - } - series[endpoint] = append(series[endpoint], point{at: time.UnixMilli(int64(math.Round(at * 1000))), value: parsed}) - } - } - return series, nil -} - func sleep(ctx context.Context, d time.Duration) error { select { case <-ctx.Done(): diff --git a/pkg/tracing/prometheus/endpoints.go b/pkg/tracing/prometheus/endpoints.go new file mode 100644 index 0000000..6a2af36 --- /dev/null +++ b/pkg/tracing/prometheus/endpoints.go @@ -0,0 +1,70 @@ +package prometheus + +import ( + "context" + "errors" + "fmt" + "net/url" + "sync" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/kubernetes" + "github.com/flanksource/commons-db/models" +) + +type endpoint struct { + original string + url *url.URL +} + +func resolveEndpoints(ctx dbcontext.Context, requested []string) (_ []endpoint, _ func() error, err error) { + forwards := kubernetes.NewForwardManager(0) + var relay scrapeRelay + release := sync.OnceValue(func() error { + err := relay.close() + forwards.CloseAll() + return err + }) + defer func() { + if err != nil { + err = errors.Join(err, release()) + } + }() + resolveCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + ctx = ctx.Wrap(resolveCtx).WithPortForwardManager(forwards) + endpoints := make([]endpoint, 0, len(requested)) + for _, raw := range requested { + u, err := parseEndpoint(raw) + if err != nil { + return nil, nil, err + } + if u.Scheme == portForwardScheme || u.Scheme == proxyScheme { + u, err = resolveWorkloadEndpoint(ctx, u, &relay) + if err != nil { + return nil, nil, fmt.Errorf("resolve metrics endpoint %q: %w", raw, err) + } + } + endpoints = append(endpoints, endpoint{original: raw, url: u}) + } + return endpoints, release, nil +} + +func resolveWorkloadEndpoint(ctx dbcontext.Context, u *url.URL, relay *scrapeRelay) (*url.URL, error) { + resolved, err := ctx.ResolveConnectionURL(u.String(), models.ConnectionTypeHTTP, "") + if err != nil { + return nil, err + } + target, err := parseEndpoint(resolved) + if err != nil { + return nil, err + } + if target.Scheme != "http" && target.Scheme != "https" { + return nil, errors.New("the Kubernetes workload did not resolve to an HTTP(S) endpoint") + } + if u.Scheme == proxyScheme { + return relay.route(target) + } + return target, nil +} diff --git a/pkg/tracing/prometheus/filesd.go b/pkg/tracing/prometheus/filesd.go index b55f423..5600a16 100644 --- a/pkg/tracing/prometheus/filesd.go +++ b/pkg/tracing/prometheus/filesd.go @@ -1,10 +1,12 @@ package prometheus import ( + "cmp" "encoding/json" "errors" "fmt" "maps" + "net" "os" "path/filepath" ) @@ -22,30 +24,28 @@ type targetGroup struct { // endpoint, carrying its scheme, path, query params, scrape interval and // timeout as the labels Prometheus reads them from, then the params' labels // and telltale's own. -func targetGroups(params Params, s session) ([]byte, error) { +func targetGroups(params Params, s session, endpoints []endpoint) ([]byte, error) { interval, timeout, err := params.timing() if err != nil { return nil, err } - groups := make([]targetGroup, 0, len(params.Endpoints)) - for _, raw := range params.Endpoints { - e, err := parseEndpoint(raw) - if err != nil { - return nil, err - } + groups := make([]targetGroup, 0, len(endpoints)) + for _, endpoint := range endpoints { + u := endpoint.url + port := cmp.Or(u.Port(), map[string]string{"http": "80", "https": "443"}[u.Scheme]) labels := maps.Clone(params.Labels) if labels == nil { labels = map[string]string{} } maps.Copy(labels, map[string]string{ - "__scheme__": e.scheme, "__metrics_path__": e.path, + "__scheme__": u.Scheme, "__metrics_path__": cmp.Or(u.EscapedPath(), "/"), "__scrape_interval__": promDuration(interval), "__scrape_timeout__": promDuration(timeout), - LabelEndpoint: e.url, LabelRun: s.run, LabelStream: s.stream, + LabelEndpoint: endpoint.original, LabelRun: s.run, LabelStream: s.stream, }) - for name, value := range e.params { - labels["__param_"+name] = value + for name, values := range u.Query() { + labels["__param_"+name] = values[0] } - groups = append(groups, targetGroup{Targets: []string{e.address}, Labels: labels}) + groups = append(groups, targetGroup{Targets: []string{net.JoinHostPort(u.Hostname(), port)}, Labels: labels}) } rendered, err := json.MarshalIndent(groups, "", " ") if err != nil { diff --git a/pkg/tracing/prometheus/filesd_internal_test.go b/pkg/tracing/prometheus/filesd_internal_test.go index 8a0a075..9655f34 100644 --- a/pkg/tracing/prometheus/filesd_internal_test.go +++ b/pkg/tracing/prometheus/filesd_internal_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" + dbcontext "github.com/flanksource/commons-db/context" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -14,7 +15,10 @@ var _ = Describe("file_sd target groups", func() { Endpoints: []string{"http://localhost:8080/metrics", "https://exporter.example.com:9115/probe?module=http_2xx"}, Interval: "5s", Timeout: "2s", Labels: map[string]string{"env": "dev"}, }} - rendered, err := targetGroups(params, session{run: "run-2026-10-31T10-00-00Z", stream: "STREAM1"}) + endpoints, release, err := resolveEndpoints(dbcontext.New(), params.Endpoints) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(release) + rendered, err := targetGroups(params, session{run: "run-2026-10-31T10-00-00Z", stream: "STREAM1"}, endpoints) Expect(err).ToNot(HaveOccurred()) Expect(string(rendered)).To(Equal(`[ { @@ -53,7 +57,11 @@ var _ = Describe("file_sd target groups", func() { }) It("defaults the port by scheme, the path to /, and the timeout to the shorter of 10s and the interval", func() { - rendered, err := targetGroups(Params{Options: Options{Endpoints: []string{"https://example.com"}, Interval: "3s"}}, session{run: "run-1", stream: "S"}) + params := Params{Options: Options{Endpoints: []string{"https://example.com"}, Interval: "3s"}} + endpoints, release, err := resolveEndpoints(dbcontext.New(), params.Endpoints) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(release) + rendered, err := targetGroups(params, session{run: "run-1", stream: "S"}, endpoints) Expect(err).ToNot(HaveOccurred()) Expect(string(rendered)).To(ContainSubstring(`"example.com:443"`)) Expect(string(rendered)).To(ContainSubstring(`"__metrics_path__": "/"`)) diff --git a/pkg/tracing/prometheus/handle_spec_test.go b/pkg/tracing/prometheus/handle_spec_test.go index b59c69d..0127298 100644 --- a/pkg/tracing/prometheus/handle_spec_test.go +++ b/pkg/tracing/prometheus/handle_spec_test.go @@ -13,7 +13,6 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gstruct" "github.com/flanksource/commons-db/query" "github.com/flanksource/telltale/pkg/tracing" @@ -23,14 +22,12 @@ import ( // scrapeState is what the fake Prometheus reports of one endpoint's last scrape. type scrapeState struct { - at time.Time - health string - err string - duration float64 - samples int + at time.Time + health string + err string } -// fakePrometheus serves the targets and query APIs over the target groups it +// fakePrometheus serves the targets API over the target groups it // reads from the file_sd dir, reporting the scrape each endpoint is given. type fakePrometheus struct { dir string @@ -42,9 +39,7 @@ type fakePrometheus struct { func newFakePrometheus(dir string) *fakePrometheus { fake := &fakePrometheus{dir: dir, scrapes: map[string]scrapeState{}} mux := http.NewServeMux() - mux.HandleFunc("GET /-/ready", func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("ready")) }) mux.HandleFunc("GET /api/v1/targets", fake.targets) - mux.HandleFunc("GET /api/v1/query", fake.query) fake.server = httptest.NewServer(mux) DeferCleanup(fake.server.Close) return fake @@ -89,29 +84,12 @@ func (f *fakePrometheus) targets(w http.ResponseWriter, r *http.Request) { active = append(active, map[string]any{ "labels": map[string]string{"job": "file_sd", "telltale_endpoint": labels["telltale_endpoint"], "telltale_stream": labels["telltale_stream"]}, "scrapePool": "file_sd", "health": state.health, "lastError": state.err, - "lastScrape": state.at.UTC().Format(time.RFC3339Nano), "lastScrapeDuration": state.duration, + "lastScrape": state.at.UTC().Format(time.RFC3339Nano), }) } writeJSON(w, map[string]any{"status": "success", "data": map[string]any{"activeTargets": active, "droppedTargets": []any{}}}) } -func (f *fakePrometheus) query(w http.ResponseWriter, _ *http.Request) { - f.mu.Lock() - defer f.mu.Unlock() - result := []map[string]any{} - for endpoint, state := range f.scrapes { - if state.at.IsZero() { - continue - } - at := float64(state.at.UnixMilli()) / 1000 - result = append(result, map[string]any{ - "metric": map[string]string{"telltale_endpoint": endpoint}, - "values": []any{[]any{at, fmt.Sprint(state.samples)}}, - }) - } - writeJSON(w, map[string]any{"status": "success", "data": map[string]any{"resultType": "matrix", "result": result}}) -} - func writeJSON(w http.ResponseWriter, value any) { w.Header().Set("Content-Type", "application/json") Expect(json.NewEncoder(w).Encode(value)).To(Succeed()) @@ -138,41 +116,28 @@ var _ = Describe("prometheus capture", func() { return env.Start(prometheus.Kind, fmt.Sprintf(`{"endpoints": [%q, %q], "interval": "1s"}`, healthy, failing)) } - It("records one scrape each time a target's last scrape advances, an error target's with its error", func() { + It("reports successful and failed targets without emitting scrape records", func() { session := start() tracingtest.Running(session) first := time.Now().Add(-100 * time.Millisecond).Truncate(time.Millisecond) - fake.scraped(healthy, scrapeState{at: first, health: "up", duration: 0.0042, samples: 7}) - fake.scraped(failing, scrapeState{at: first, health: "down", err: "connection refused", duration: 0.001}) - Eventually(func() int64 { return session.Snapshot().EventCount }).WithTimeout(5 * time.Second).Should(Equal(int64(2))) - Consistently(func() int64 { return session.Snapshot().EventCount }).WithTimeout(1200 * time.Millisecond).Should(Equal(int64(2))) - - fake.scraped(healthy, scrapeState{at: first.Add(time.Second), health: "up", duration: 0.003, samples: 9}) - Eventually(func() int64 { return session.Snapshot().EventCount }).WithTimeout(5 * time.Second).Should(Equal(int64(3))) + fake.scraped(healthy, scrapeState{at: first, health: "up"}) + fake.scraped(failing, scrapeState{at: first, health: "down", err: "connection refused"}) + Eventually(func() string { return string(session.Snapshot().Summary) }).WithTimeout(5 * time.Second). + Should(And(ContainSubstring(`"up":1`), ContainSubstring(`"down":1`), ContainSubstring("connection refused"))) session.Stop("spec") info := tracingtest.Ended(session) Expect(info.Error).To(BeEmpty()) - - scrape := func(endpoint, health string, samples int, scrapeErr string) OmegaMatcher { - return MatchKeys(IgnoreExtras, Keys{ - "endpoint": Equal(endpoint), "health": Equal(health), "samples": BeNumerically("==", samples), "error": Equal(scrapeErr), - }) - } - Expect(env.Rows(info)).To(ConsistOf( - scrape(healthy, "up", 7, ""), - scrape(failing, "down", 0, "connection refused"), - scrape(healthy, "up", 9, ""), - )) + Expect(info.EventCount).To(BeZero()) Expect(info.Handle).To(Equal(fake.server.URL)) Expect(filepath.Glob(filepath.Join(fake.dir, "*.json"))).To(BeEmpty(), "release removes the session's file_sd file") }) - It("fails the capture loudly once Prometheus is unreachable", func() { + It("fails the capture once VictoriaMetrics is unreachable", func() { session := start() tracingtest.Running(session) fake.server.Close() info := tracingtest.Ended(session) Expect(info.State).To(Equal(query.SessionFailed)) - Expect(info.Error).To(ContainSubstring("prometheus")) + Expect(info.Error).To(ContainSubstring("VictoriaMetrics")) }) }) diff --git a/pkg/tracing/prometheus/instance.go b/pkg/tracing/prometheus/instance.go new file mode 100644 index 0000000..7a04fac --- /dev/null +++ b/pkg/tracing/prometheus/instance.go @@ -0,0 +1,221 @@ +package prometheus + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "runtime" + "strconv" + "time" + + "github.com/flanksource/clicky/exec" + "github.com/flanksource/deps" + "github.com/flanksource/telltale/internal/lifetime" + "github.com/flanksource/telltale/pkg/tracing/process/starttime" + "github.com/gofrs/flock" + psprocess "github.com/shirou/gopsutil/v4/process" +) + +type instanceState struct { + Port int `json:"port"` + PID int `json:"pid"` + StartedAt time.Time `json:"startedAt"` +} + +func (s instanceState) address() string { + return net.JoinHostPort("127.0.0.1", strconv.Itoa(s.Port)) +} + +func (s instanceState) server(paths ManagedPaths) Server { + return Server{URL: "http://" + s.address(), Job: Job, FileSDDir: paths.FileSD} +} + +func (s instanceState) running(ctx context.Context) (bool, error) { + exists, err := psprocess.PidExistsWithContext(ctx, int32(s.PID)) + if err != nil || !exists { + return false, err + } + startedAt, err := starttime.Of(s.PID) + if errors.Is(err, psprocess.ErrorProcessNotRunning) || errors.Is(err, os.ErrNotExist) { + return false, nil + } + return s.StartedAt.Equal(startedAt), err +} + +func stateFile(paths ManagedPaths) string { return filepath.Join(paths.serviceDir(), "state.json") } + +func readInstance(paths ManagedPaths) (*instanceState, error) { + path := stateFile(paths) + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read VictoriaMetrics state %s: %w", path, err) + } + var state instanceState + if err := json.Unmarshal(data, &state); err != nil { + return nil, fmt.Errorf("decode VictoriaMetrics state %s: %w", path, err) + } + if state.Port < 1 || state.Port > 65535 || state.PID < 1 || state.PID > 1<<31-1 || state.StartedAt.IsZero() { + return nil, fmt.Errorf("invalid VictoriaMetrics process identity in %s", path) + } + return &state, nil +} + +func (s instanceState) save(paths ManagedPaths) error { + file, err := os.CreateTemp(paths.serviceDir(), ".state-*.tmp") + if err != nil { + return err + } + writeErr := json.NewEncoder(file).Encode(s) + if err := errors.Join(writeErr, file.Close()); err != nil { + return errors.Join(err, os.Remove(file.Name())) + } + if err := os.Rename(file.Name(), stateFile(paths)); err != nil { + return errors.Join(err, os.Remove(file.Name())) + } + return nil +} + +func lockInstance(ctx context.Context, paths ManagedPaths) (func() error, error) { + lock := flock.New(filepath.Join(paths.serviceDir(), "instance.lock")) + locked, err := lock.TryLockContext(ctx, 100*time.Millisecond) + if err != nil || !locked { + return nil, fmt.Errorf("lock VictoriaMetrics at %s: %w", paths.serviceDir(), errors.Join(err, ctx.Err(), lock.Close())) + } + return lock.Close, nil +} + +func openInstance(ctx context.Context, paths ManagedPaths) (*lease, error) { + state, err := readInstance(paths) + if err != nil { + return nil, err + } + if state != nil { + running, err := state.running(ctx) + if err != nil { + return nil, fmt.Errorf("check VictoriaMetrics process %d: %w", state.PID, err) + } + if running { + server := state.server(paths) + if err := newClient(server).health(ctx); err != nil { + return nil, err + } + return &lease{paths: paths, server: server}, nil + } + } else { + state = &instanceState{} + } + return startInstance(ctx, paths, *state) +} + +func installVictoriaMetrics(ctx context.Context, dir string) (_ string, err error) { + bin := filepath.Join(dir, "bin", Version, "victoria-metrics") + if info, err := os.Stat(bin); err == nil { + if !info.Mode().IsRegular() || info.Mode()&0o111 == 0 { + return "", fmt.Errorf("VictoriaMetrics binary is not executable: %s", bin) + } + return bin, ctx.Err() + } else if !errors.Is(err, os.ErrNotExist) { + return "", fmt.Errorf("check VictoriaMetrics binary: %w", err) + } + if err := os.MkdirAll(filepath.Dir(bin), 0o700); err != nil { + return "", err + } + staging, err := os.MkdirTemp(dir, ".install-*") + if err != nil { + return "", err + } + defer func() { err = errors.Join(err, os.RemoveAll(staging)) }() + asset := fmt.Sprintf("victoria-metrics-%s-%s-%s.tar.gz", runtime.GOOS, runtime.GOARCH, Version) + _, err = deps.InstallWithContext(ctx, "VictoriaMetrics/VictoriaMetrics#"+asset, Version, + deps.WithStrictChecksum(true), deps.WithTimeout(2*time.Minute), deps.WithBinDir(filepath.Join(staging, "bin")), + deps.WithAppDir(filepath.Join(staging, "apps")), deps.WithTmpDir(filepath.Join(staging, "tmp")), + deps.WithCacheDir(filepath.Join(dir, "downloads"))) + if err != nil { + return "", fmt.Errorf("install VictoriaMetrics %s (%s): %w", Version, asset, err) + } + if err := os.Rename(filepath.Join(staging, "bin", "victoria-metrics"), bin); err != nil { + return "", fmt.Errorf("publish VictoriaMetrics binary: %w", err) + } + return bin, ctx.Err() +} + +func startInstance(ctx context.Context, paths ManagedPaths, state instanceState) (_ *lease, err error) { + if err := os.MkdirAll(paths.FileSD, 0o700); err != nil { + return nil, fmt.Errorf("create VictoriaMetrics target directory: %w", err) + } + config := filepath.Join(paths.serviceDir(), "scrape.yaml") + scrapes := fmt.Sprintf("scrape_configs:\n - job_name: %s\n file_sd_configs:\n - files: [%q]\n", Job, filepath.Join(paths.FileSD, "*.json")) + if err := os.WriteFile(config, []byte(scrapes), 0o600); err != nil { + return nil, fmt.Errorf("write VictoriaMetrics scrape configuration: %w", err) + } + binary, err := installVictoriaMetrics(ctx, paths.serviceDir()) + if err != nil { + return nil, err + } + listener, err := net.Listen("tcp", state.address()) + if err != nil { + return nil, fmt.Errorf("allocate VictoriaMetrics listener at %s: %w", state.address(), err) + } + state.Port = listener.Addr().(*net.TCPAddr).Port + if err := listener.Close(); err != nil { + return nil, err + } + server := state.server(paths) + processCtx, cancel := context.WithCancel(context.WithoutCancel(ctx)) + process := exec.NewExec(binary, "-httpListenAddr="+state.address(), "-storageDataPath="+filepath.Join(paths.serviceDir(), "data"), + "-retentionPeriod=15d", "-promscrape.config="+config, "-promscrape.fileSDCheckInterval=1s", "-search.latencyOffset=0s"). + WithContext(processCtx).WithProcessGroup().WithCaptureLimit(64 << 10) + run := lifetime.Start("VictoriaMetrics", func() error { return process.Run().Err }) + stop := func() error { + cancel() + err := run.Wait() + if !errors.Is(err, context.Canceled) { + return err + } + return nil + } + defer func() { + if err != nil { + err = errors.Join(err, stop()) + } + }() + if err := awaitReady(ctx, server, process, run); err != nil { + return nil, err + } + state.PID = process.Pid() + state.StartedAt, err = starttime.Of(state.PID) + if err != nil { + return nil, fmt.Errorf("identify VictoriaMetrics process %d: %w", state.PID, err) + } + if err := state.save(paths); err != nil { + return nil, fmt.Errorf("publish VictoriaMetrics state: %w", err) + } + return &lease{paths: paths, server: server, stop: stop}, nil +} + +func awaitReady(ctx context.Context, server Server, process *exec.Process, run *lifetime.Service) error { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + client := newClient(server) + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-run.Done(): + return fmt.Errorf("VictoriaMetrics exited during startup: %v\n%s", run.Wait(), process.GetStderr()) + case <-ctx.Done(): + return fmt.Errorf("VictoriaMetrics did not become ready at %s: %w\n%s", server.URL, ctx.Err(), process.GetStderr()) + case <-ticker.C: + if process.IsRunning() && client.health(ctx) == nil && process.IsRunning() { + return nil + } + } + } +} diff --git a/pkg/tracing/prometheus/managed.go b/pkg/tracing/prometheus/managed.go index a0d3896..1cc74a7 100644 --- a/pkg/tracing/prometheus/managed.go +++ b/pkg/tracing/prometheus/managed.go @@ -4,66 +4,46 @@ import ( "context" "errors" "fmt" - "io/fs" - "net" "os" "path/filepath" "sync" + "time" "github.com/flanksource/commons-db/models" - "github.com/flanksource/deps/start" "github.com/google/uuid" ) -// Version is the Prometheus release the managed Prometheus runs: a 3.x, whose -// targets take their scrape interval and timeout from their own labels. -const Version = "3.5.0" - -// Job is the managed Prometheus's job that scrapes the file_sd targets. +const Version = "v1.156.0" const Job = "file_sd" - -// ConnectionName names the managed Prometheus's connection. const ConnectionName = "telltale-prometheus" -const service = "prometheus" - -// ManagedPaths are where a data dir keeps its managed Prometheus: the deps -// service state (its TSDB under /prometheus/data), and the file_sd -// target files its captures write. type ManagedPaths struct { State string FileSD string } -// Paths are the managed Prometheus's absolute paths under dataDir: deps takes -// only an absolute file_sd dir, and the state path keys the connection's ID. func Paths(dataDir string) (ManagedPaths, error) { root, err := filepath.Abs(dataDir) if err != nil { return ManagedPaths{}, fmt.Errorf("resolve the data dir %q: %w", dataDir, err) } - return ManagedPaths{State: filepath.Join(root, "services"), FileSD: filepath.Join(root, "prometheus", "targets")}, nil + return ManagedPaths{State: filepath.Join(root, "services"), FileSD: filepath.Join(root, "victoriametrics", "targets")}, nil } -// Managed is the Prometheus of a data dir, one per data dir: the first -// capture of a run starts it through deps start, and every later one reuses -// it with its TSDB. deps supervises a binary in the process that starts it, -// so it is telltale's sub-process: Release stops it once no run of this -// process holds it, and the next capture starts it again on the same TSDB and -// port. A Prometheus another process supervises is reused and left running. +func (p ManagedPaths) serviceDir() string { return filepath.Join(p.State, "victoriametrics") } + +// Managed holds the data directory's VictoriaMetrics for a run or HTTP server. type Managed struct { dataDir string mu sync.Mutex held *lease } -// lease is one data dir's Prometheus, shared by every run of this process -// that captures into that data dir. type lease struct { - state string - instance *start.Instance - owned bool - holders int + paths ManagedPaths + server Server + stop func() error + holders int } var leases = struct { @@ -71,49 +51,39 @@ var leases = struct { byState map[string]*lease }{byState: map[string]*lease{}} -// NewManaged is the managed Prometheus of dataDir, for one run. func NewManaged(dataDir string) *Managed { return &Managed{dataDir: dataDir} } -// Server is the run's Prometheus, started on the run's first call. func (m *Managed) Server(ctx context.Context) (Server, error) { - held, paths, err := m.hold(ctx) + if m.dataDir == "" { + return Server{}, errors.New("managed VictoriaMetrics requires a telltale data directory") + } + paths, err := Paths(m.dataDir) if err != nil { return Server{}, err } - return Server{URL: held.instance.Connection.URL, Job: Job, FileSDDir: paths.FileSD}, nil + m.mu.Lock() + defer m.mu.Unlock() + if m.held == nil { + m.held, err = acquire(ctx, paths) + if err != nil { + return Server{}, err + } + } + return m.held.server, nil } -// Connection is the run's Prometheus as the read-only connection its samples -// are queried through, started on the run's first call as Server is, so a run -// that only queries it holds it too. func (m *Managed) Connection(ctx context.Context) (*models.Connection, error) { - held, paths, err := m.hold(ctx) + server, err := m.Server(ctx) if err != nil { return nil, err } - return managedConnection(held.instance.Connection, paths), nil -} - -func (m *Managed) hold(ctx context.Context) (*lease, ManagedPaths, error) { - if m.dataDir == "" { - return nil, ManagedPaths{}, errors.New("a managed prometheus lives in telltale's data dir, and none is set") - } paths, err := Paths(m.dataDir) if err != nil { - return nil, ManagedPaths{}, err - } - m.mu.Lock() - defer m.mu.Unlock() - if m.held == nil { - if m.held, err = acquire(ctx, paths); err != nil { - return nil, ManagedPaths{}, err - } + return nil, err } - return m.held, paths, nil + return managedConnection(server.URL, paths), nil } -// Release ends the run's hold on its Prometheus, stopping a Prometheus this -// process supervises once no other run holds it. func (m *Managed) Release(ctx context.Context) error { m.mu.Lock() defer m.mu.Unlock() @@ -127,97 +97,70 @@ func (m *Managed) Release(ctx context.Context) error { if held.holders--; held.holders > 0 { return nil } - delete(leases.byState, held.state) - if !held.owned { + delete(leases.byState, held.paths.State) + if held.stop == nil { return nil } - if err := held.instance.Stop(ctx); err != nil { - return fmt.Errorf("stop the managed prometheus of %s: %w", held.state, err) + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + unlock, err := lockInstance(ctx, held.paths) + if err != nil { + return errors.Join(err, held.stop()) } - return nil + return errors.Join(held.stop(), unlock()) } -func acquire(ctx context.Context, paths ManagedPaths) (*lease, error) { +func acquire(ctx context.Context, paths ManagedPaths) (_ *lease, err error) { leases.Lock() defer leases.Unlock() - if held, ok := leases.byState[paths.State]; ok { + if held := leases.byState[paths.State]; held != nil { + if err := newClient(held.server).health(ctx); err != nil { + return nil, err + } held.holders++ return held, nil } - instance, err := startManaged(ctx, paths) - if err != nil { + if err := os.MkdirAll(paths.serviceDir(), 0o700); err != nil { return nil, err } - held := &lease{state: paths.State, instance: instance, owned: instance.State.SupervisorPID == os.Getpid(), holders: 1} - leases.byState[paths.State] = held - return held, nil -} - -func startManaged(ctx context.Context, paths ManagedPaths) (*start.Instance, error) { - if err := os.MkdirAll(paths.FileSD, 0o755); err != nil { - return nil, fmt.Errorf("create the file_sd dir: %w", err) - } - options := []start.Option{ - start.WithStateDir(paths.State), start.WithRuntime(start.RuntimeBinary), start.WithVersion(Version), - start.WithParameters(map[string]string{"file-sd-dir": paths.FileSD}), - } - _, err := start.Get(ctx, service, start.WithStateDir(paths.State)) - switch { - case errors.Is(err, fs.ErrNotExist): - // The first start takes a free port; later ones keep the port its state recorded. - port, err := freePort() - if err != nil { - return nil, err - } - options = append(options, start.WithPort(port)) - case err != nil: - return nil, fmt.Errorf("read the managed prometheus state in %s: %w", paths.State, err) + unlock, err := lockInstance(ctx, paths) + if err != nil { + return nil, err } - return start.Start(ctx, service, options...) -} - -func freePort() (int, error) { - listener, err := net.Listen("tcp", "127.0.0.1:0") + defer func() { err = errors.Join(err, unlock()) }() + held, err := openInstance(ctx, paths) if err != nil { - return 0, fmt.Errorf("find a free port for the managed prometheus: %w", err) + return nil, err } - defer func() { _ = listener.Close() }() - return listener.Addr().(*net.TCPAddr).Port, nil + held.holders = 1 + leases.byState[paths.State] = held + return held, nil } -// ManagedConnection is the connection of dataDir's managed Prometheus, as a -// read-only connection its samples are queried through; nil when no capture -// has started one. It answers while a capture's run holds that Prometheus. -func ManagedConnection(ctx context.Context, dataDir string) (*models.Connection, error) { +// ManagedConnection looks up the virtual connection without starting the service. +func ManagedConnection(_ context.Context, dataDir string) (*models.Connection, error) { paths, err := Paths(dataDir) if err != nil { return nil, err } - instance, err := start.Get(ctx, service, start.WithStateDir(paths.State)) - if errors.Is(err, fs.ErrNotExist) { - return nil, nil - } - if err != nil { - return nil, fmt.Errorf("read the managed prometheus state in %s: %w", paths.State, err) + state, err := readInstance(paths) + if err != nil || state == nil { + return nil, err } - return managedConnection(instance.Connection, paths), nil + return managedConnection(state.server(paths).URL, paths), nil } -func managedConnection(connection models.Connection, paths ManagedPaths) *models.Connection { - connection.ID = ManagedConnectionID(paths) - connection.Name, connection.Namespace = ConnectionName, "" - connection.Source = filepath.Join(paths.State, service, "state.yaml") - connection.ReadOnly, connection.Virtual = true, true - return &connection +func managedConnection(url string, paths ManagedPaths) *models.Connection { + return &models.Connection{ + ID: ManagedConnectionID(paths), Name: ConnectionName, Type: models.ConnectionTypePrometheus, + URL: url, Source: stateFile(paths), ReadOnly: true, Virtual: true, + } } -// ManagedConnectionID is the id of the managed connection kept at paths. func ManagedConnectionID(paths ManagedPaths) uuid.UUID { return uuid.NewSHA1(uuid.NameSpaceURL, []byte("telltale://managed/prometheus/"+paths.State)) } -// NamesManaged reports whether reference names the managed connection kept at -// paths: by its id, its name or connection://. func NamesManaged(paths ManagedPaths, reference string) bool { return reference == ManagedConnectionID(paths).String() || reference == ConnectionName || reference == "connection://"+ConnectionName } diff --git a/pkg/tracing/prometheus/managed_spec_test.go b/pkg/tracing/prometheus/managed_spec_test.go index 405e54e..d5d2f11 100644 --- a/pkg/tracing/prometheus/managed_spec_test.go +++ b/pkg/tracing/prometheus/managed_spec_test.go @@ -11,7 +11,7 @@ import ( ) var _ = Describe("managed prometheus paths", func() { - It("roots a relative data dir at the working directory, because deps takes only an absolute file_sd dir", func() { + It("roots the VictoriaMetrics service and target files under an absolute data directory", func() { relative := filepath.Join(".tmp", "data") cwd, err := os.Getwd() Expect(err).NotTo(HaveOccurred()) @@ -21,7 +21,7 @@ var _ = Describe("managed prometheus paths", func() { Expect(err).NotTo(HaveOccurred()) Expect(paths).To(Equal(prometheus.ManagedPaths{ State: filepath.Join(cwd, relative, "services"), - FileSD: filepath.Join(cwd, relative, "prometheus", "targets"), + FileSD: filepath.Join(cwd, relative, "victoriametrics", "targets"), })) }) }) diff --git a/pkg/tracing/prometheus/params.go b/pkg/tracing/prometheus/params.go index 754a07f..85586f3 100644 --- a/pkg/tracing/prometheus/params.go +++ b/pkg/tracing/prometheus/params.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "maps" - "net" "net/url" "regexp" "slices" @@ -12,6 +11,7 @@ import ( "time" "github.com/flanksource/telltale/pkg/tracing/target" + "k8s.io/apimachinery/pkg/util/validation" ) // DefaultInterval is how often an endpoint is scraped unless asked otherwise, @@ -22,6 +22,11 @@ const DefaultInterval = 15 * time.Second // caps it. const defaultTimeout = 10 * time.Second +const ( + portForwardScheme = "portforward" + proxyScheme = "proxy" +) + // The labels telltale gives every target it scrapes, which params may not set. const ( LabelRun = "telltale_run" @@ -34,7 +39,7 @@ var labelName = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) // Options choose the endpoints a capture scrapes, how often, and the labels // their samples are stored with. type Options struct { - Endpoints []string `json:"endpoints" clicky:"title=Endpoints,desc=Prometheus metrics URLs to scrape such as http://localhost:8080/metrics"` + Endpoints []string `json:"endpoints" clicky:"title=Endpoints,desc=Metrics URLs using http:// or https:// or Kubernetes portforward:// (also port-forward://) or proxy://"` Interval string `json:"interval,omitempty" clicky:"title=Interval,desc=How often each endpoint is scraped such as 15s (at least 1s)"` Timeout string `json:"timeout,omitempty" clicky:"title=Timeout,desc=How long one scrape may take such as 5s (at most the interval; default the shorter of 10s and the interval)"` Labels map[string]string `json:"labels,omitempty" clicky:"title=Labels,desc=Extra labels every scraped sample is stored with such as env=dev"` @@ -115,42 +120,45 @@ func parseMillis(name, value string) (time.Duration, error) { // promDuration is d as Prometheus reads a duration. func promDuration(d time.Duration) string { return fmt.Sprintf("%dms", d.Milliseconds()) } -// endpoint is a metrics URL as a scrape target: its scheme, host:port, path -// and query params. -type endpoint struct { - url, scheme, address, path string - params map[string]string -} - -func parseEndpoint(raw string) (endpoint, error) { +func parseEndpoint(raw string) (*url.URL, error) { parsed, err := url.Parse(raw) - if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Hostname() == "" { - return endpoint{}, fmt.Errorf("endpoint %q: want an absolute http(s) URL such as http://localhost:8080/metrics", raw) + if err != nil { + return nil, fmt.Errorf("endpoint %q: %w", raw, err) + } + if parsed.Scheme == "port-forward" { + parsed.Scheme = portForwardScheme + } + workload := parsed.Scheme == portForwardScheme || parsed.Scheme == proxyScheme + if !workload && ((parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Hostname() == "") { + return nil, fmt.Errorf("endpoint %q: want an absolute http(s) URL or a Kubernetes portforward:// or proxy:// URL", raw) } if parsed.User != nil { - return endpoint{}, fmt.Errorf("endpoint %q carries credentials, which a scrape target cannot", raw) + return nil, fmt.Errorf("endpoint %q carries credentials, which a scrape target cannot", raw) } if parsed.Fragment != "" { - return endpoint{}, fmt.Errorf("endpoint %q has a fragment, which a scrape never sends", raw) + return nil, fmt.Errorf("endpoint %q has a fragment, which a scrape never sends", raw) } query, err := url.ParseQuery(parsed.RawQuery) if err != nil { - return endpoint{}, fmt.Errorf("endpoint %q query: %w", raw, err) + return nil, fmt.Errorf("endpoint %q query: %w", raw, err) } - params := map[string]string{} for name, values := range query { if len(values) > 1 { - return endpoint{}, fmt.Errorf("endpoint %q: query param %q is given more than once, which a scrape target cannot carry", raw, name) + return nil, fmt.Errorf("endpoint %q: query param %q is given more than once, which a scrape target cannot carry", raw, name) } - params[name] = values[0] } - port := parsed.Port() - if port == "" { - port = map[string]string{"http": "80", "https": "443"}[parsed.Scheme] + if workload && (parsed.Opaque != "" || (parsed.Hostname() == "" && query.Get("selector") == "")) { + return nil, fmt.Errorf("endpoint %q: a Kubernetes workload name or selector is required", raw) } - path := parsed.EscapedPath() - if path == "" { - path = "/" + if parsed.Scheme == proxyScheme { + for segment := range strings.SplitSeq(parsed.Path, "/") { + if segment == "." || segment == ".." { + return nil, fmt.Errorf("endpoint %q: a Kubernetes proxy path cannot contain a dot segment", raw) + } + } + if port := query.Get("port"); port != "" && len(validation.IsDNS1123Label(port)) != 0 { + return nil, fmt.Errorf("endpoint %q: invalid port %q", raw, port) + } } - return endpoint{url: raw, scheme: parsed.Scheme, address: net.JoinHostPort(parsed.Hostname(), port), path: path, params: params}, nil + return parsed, nil } diff --git a/pkg/tracing/prometheus/params_spec_test.go b/pkg/tracing/prometheus/params_spec_test.go index fb650ce..6d3d2b5 100644 --- a/pkg/tracing/prometheus/params_spec_test.go +++ b/pkg/tracing/prometheus/params_spec_test.go @@ -28,6 +28,8 @@ var _ = Describe("prometheus params", func() { Entry("a reserved label", prometheus.Options{Endpoints: []string{endpoint}, Labels: map[string]string{"__address__": "x"}}, `label "__address__" is reserved`), Entry("telltale's own label", prometheus.Options{Endpoints: []string{endpoint}, Labels: map[string]string{"telltale_stream": "x"}}, `label "telltale_stream" is reserved`), Entry("an invalid label name", prometheus.Options{Endpoints: []string{endpoint}, Labels: map[string]string{"app-name": "x"}}, `label "app-name"`), + Entry("a proxy path that escapes the service", prometheus.Options{Endpoints: []string{"proxy://exporter.monitoring/%2e%2e/metrics"}}, "dot segment"), + Entry("a proxy port that inserts a path", prometheus.Options{Endpoints: []string{"proxy://exporter.monitoring/metrics?port=http%2Fproxy"}}, "invalid port"), ) It("accepts endpoints with a query, an interval, a timeout within it and labels", func() { @@ -36,4 +38,13 @@ var _ = Describe("prometheus params", func() { Interval: "5s", Timeout: "5s", Labels: map[string]string{"env": "dev"}, }.Validate()).To(Succeed()) }) + + It("validates direct and inline Kubernetes endpoints together without contacting Kubernetes", func() { + Expect(prometheus.Options{Endpoints: []string{ + endpoint, + "port-forward://exporter.monitoring:9100/metrics", + "portforward://exporter-0.monitoring:9100/metrics?kind=pod", + "proxy://exporter.monitoring/probe?port=http&module=http_2xx", + }}.Validate()).To(Succeed()) + }) }) diff --git a/pkg/tracing/prometheus/portforward_internal_test.go b/pkg/tracing/prometheus/portforward_internal_test.go new file mode 100644 index 0000000..2f5ba6e --- /dev/null +++ b/pkg/tracing/prometheus/portforward_internal_test.go @@ -0,0 +1,93 @@ +package prometheus + +import ( + "bufio" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + dbkubernetes "github.com/flanksource/commons-db/kubernetes" + "github.com/flanksource/commons/logger" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/portforward" + "k8s.io/streaming/pkg/httpstream" + "k8s.io/streaming/pkg/httpstream/spdy" +) + +var _ = Describe("Kubernetes port-forward scrape endpoints", func() { + It("forwards to the pod and closes its tunnel independently of another capture", func(ctx SpecContext) { + const endpoint = "port-forward://exporter.monitoring:9100/metrics?kind=pod&module=fixture" + api := httptest.NewServer(http.HandlerFunc(portForwardFixture)) + DeferCleanup(api.Close) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: "monitoring", Name: "exporter"}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "exporter", Ports: []corev1.ContainerPort{{ContainerPort: 9100}}}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}}, + } + client := dbkubernetes.NewKubeClient(logger.GetLogger("test"), fake.NewSimpleClientset(pod), &rest.Config{Host: api.URL}) + capture := dbcontext.NewContext(ctx).WithLocalKubernetes(client) + first, releaseFirst, err := resolveEndpoints(capture, []string{endpoint}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(releaseFirst) + second, releaseSecond, err := resolveEndpoints(capture, []string{endpoint}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(releaseSecond) + Expect(first[0].original).To(Equal(endpoint)) + Expect(first[0].url.Host).NotTo(Equal(second[0].url.Host)) + Expect(releaseFirst()).To(Succeed()) + + Eventually(func() error { + conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", first[0].url.Host) + if conn != nil { + _ = conn.Close() + } + return err + }).Should(HaveOccurred()) + request, err := http.NewRequestWithContext(ctx, http.MethodGet, second[0].url.String(), nil) + Expect(err).NotTo(HaveOccurred()) + response, err := http.DefaultClient.Do(request) + Expect(err).NotTo(HaveOccurred()) + body, err := io.ReadAll(response.Body) + Expect(response.Body.Close()).To(Succeed()) + Expect(err).NotTo(HaveOccurred()) + Expect(string(body)).To(Equal("fixture_metric 1\n")) + }, SpecTimeout(10*time.Second)) +}) + +func portForwardFixture(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/v1/namespaces/monitoring/pods/exporter/portforward" { + http.NotFound(w, r) + return + } + if _, err := httpstream.Handshake(r, w, []string{portforward.PortForwardProtocolV1Name}); err != nil { + return + } + conn := spdy.NewResponseUpgrader().UpgradeResponse(w, r, func(stream httpstream.Stream, ready <-chan struct{}) error { + go func() { + <-ready + defer func() { _ = stream.Close() }() + if stream.Headers().Get(corev1.StreamType) != corev1.StreamTypeData { + return + } + request, err := http.ReadRequest(bufio.NewReader(stream)) + if err != nil || request.URL.RequestURI() != "/metrics?module=fixture" || stream.Headers().Get(corev1.PortHeader) != "9100" { + return + } + const body = "fixture_metric 1\n" + _, _ = fmt.Fprintf(stream, "HTTP/1.1 200 OK\r\nContent-Length: %d\r\nConnection: close\r\n\r\n%s", len(body), body) + }() + return nil + }) + if conn != nil { + <-conn.CloseChan() + } +} diff --git a/pkg/tracing/prometheus/pretty.go b/pkg/tracing/prometheus/pretty.go new file mode 100644 index 0000000..8a63d7d --- /dev/null +++ b/pkg/tracing/prometheus/pretty.go @@ -0,0 +1,21 @@ +package prometheus + +import ( + "time" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" + "github.com/flanksource/telltale/pkg/tracing" +) + +func (r Scrape) Pretty() api.Text { + t := clicky.Text(r.Health, tracing.SeverityStyle(r.Health)).Space().Append(r.Endpoint, "text-blue-500"). + Space().Add(api.HumanNumber(r.Samples)).Append(" samples", "text-muted").Space(). + Add(api.Human(time.Duration(r.DurationMs*float64(time.Millisecond)), "text-muted")) + if r.Error != "" { + t = t.Space().Append(r.Error, "text-red-500") + } + return t +} + +func (r Scrape) PrettyFull() api.Textable { return r.Pretty() } diff --git a/pkg/tracing/prometheus/prometheus.go b/pkg/tracing/prometheus/prometheus.go index 93ad092..afb6380 100644 --- a/pkg/tracing/prometheus/prometheus.go +++ b/pkg/tracing/prometheus/prometheus.go @@ -1,8 +1,4 @@ -// Package prometheus is a trace kind that scrapes Prometheus metrics endpoints -// into a Prometheus telltale manages: the capture hands its endpoints to that -// Prometheus as file_sd targets, labelled with the run and stream, and records -// one row per scrape Prometheus completes. The samples themselves stay in -// Prometheus's TSDB, queryable with PromQL after the capture ends. +// Package prometheus configures VictoriaMetrics to scrape Prometheus exporters. package prometheus import ( @@ -23,14 +19,9 @@ import ( // Kind is the name a host registers the kind as. const Kind = "prometheus" -// How long a start waits for Prometheus to be ready and to discover the -// session's targets: file_sd changes reach the scrape manager at most every 5s. -const ( - readyWait = 30 * time.Second - discoveryWait = 20 * time.Second -) +const discoveryWait = 20 * time.Second -// Scrape is the record the kind stores: one scrape of one endpoint. +// Scrape is the record format of historical scrape captures. type Scrape struct { TakenAt time.Time `json:"takenAt" pretty:"label=Scraped" sort:"takenAt"` Endpoint string `json:"endpoint" filter:"terms"` @@ -40,7 +31,7 @@ type Scrape struct { Error string `json:"error"` } -// Server is a Prometheus a capture's targets are handed to: its URL, the job +// Server is where a capture hands off its targets: its URL, the job // whose file_sd files it reads, and their directory. type Server struct { URL string @@ -49,7 +40,7 @@ type Server struct { } // KindOptions are what the kind's captures share: the run they record into, and -// the Prometheus they scrape with, started or found running on each start. +// the VictoriaMetrics instance that scrapes them. type KindOptions struct { RunID string Server func(context.Context) (Server, error) @@ -69,23 +60,28 @@ func (kind) Schema() recordresults.ResultType[Scrape] { type clientKey struct{} -// Prepare hands the session's endpoints to Prometheus and waits until it -// scrapes them, refusing the start when Prometheus is not ready or never -// discovers them. -func (k kind) Prepare(ctx dbcontext.Context, params Params, out tracing.Emitter[Scrape]) (dbcontext.Context, func() error, error) { +// Prepare hands the session's endpoints to VictoriaMetrics and waits for discovery. +func (k kind) Prepare(ctx dbcontext.Context, params Params, out tracing.Emitter[Scrape]) (_ dbcontext.Context, _ func() error, err error) { if k.options.RunID == "" || k.options.Server == nil { return ctx, nil, errors.New("a prometheus capture records into a telltale run, which gives it its run id and data dir") } stream := tracing.StreamOf(ctx) + endpoints, closeEndpoints, err := resolveEndpoints(ctx, params.Endpoints) + if err != nil { + return ctx, nil, err + } + release := closeEndpoints + defer func() { + if err != nil { + err = errors.Join(err, release()) + } + }() server, err := k.options.Server(ctx) if err != nil { - return ctx, nil, fmt.Errorf("start the managed prometheus: %w", err) + return ctx, nil, fmt.Errorf("start managed VictoriaMetrics: %w", err) } c := newClient(server) - if err := c.ready(ctx, readyWait); err != nil { - return ctx, nil, err - } - groups, err := targetGroups(params, session{run: k.options.RunID, stream: stream}) + groups, err := targetGroups(params, session{run: k.options.RunID, stream: stream}, endpoints) if err != nil { return ctx, nil, err } @@ -93,9 +89,9 @@ func (k kind) Prepare(ctx dbcontext.Context, params Params, out tracing.Emitter[ if err != nil { return ctx, nil, err } - release := func() error { return os.Remove(path) } - if err := awaitTargets(ctx, c, stream, len(params.Endpoints)); err != nil { - return ctx, nil, errors.Join(err, release()) + release = func() error { return errors.Join(os.Remove(path), closeEndpoints()) } + if err := awaitTargets(ctx, c, stream, len(endpoints)); err != nil { + return ctx, nil, err } out.Describe(server.URL, query.SessionMetadata{Name: "job", Label: "Job", Value: server.Job}, @@ -105,9 +101,10 @@ func (k kind) Prepare(ctx dbcontext.Context, params Params, out tracing.Emitter[ return ctx.WithValue(clientKey{}, c), release, nil } -// awaitTargets waits until Prometheus lists the session's want targets. +// awaitTargets waits until VictoriaMetrics lists all of the session's targets. func awaitTargets(ctx context.Context, c *client, stream string, want int) error { - deadline := time.Now().Add(discoveryWait) + ctx, cancel := context.WithTimeout(ctx, discoveryWait) + defer cancel() for { targets, err := c.targets(ctx, stream) if err != nil { @@ -116,42 +113,31 @@ func awaitTargets(ctx context.Context, c *client, stream string, want int) error if len(targets) >= want { return nil } - if time.Now().After(deadline) { - return fmt.Errorf("prometheus %s lists %d of the session's %d targets in job %s after %s", c.server.URL, len(targets), want, c.server.Job, discoveryWait) - } if err := sleep(ctx, 200*time.Millisecond); err != nil { - return err + return fmt.Errorf("VictoriaMetrics %s discovered %d of %d targets in %s: %w", c.server.URL, len(targets), want, c.server.Job, err) } } } -// Handle records each scrape Prometheus completes until the capture stops, -// polling twice an interval so no scrape goes unseen. +// Handle reports target health while VictoriaMetrics stores the samples. func (kind) Handle(ctx dbcontext.Context, params Params, out tracing.Emitter[Scrape], _ tracing.Records[Scrape]) error { interval, _, err := params.timing() if err != nil { return err } - w := &watcher{client: ctx.Value(clientKey{}).(*client), stream: tracing.StreamOf(ctx), interval: interval, last: map[string]time.Time{}} - ticker := time.NewTicker(interval / 2) + client := ctx.Value(clientKey{}).(*client) + stream := tracing.StreamOf(ctx) + ticker := time.NewTicker(min(interval/2, time.Second)) defer ticker.Stop() for { - scrapes, summary, err := w.poll(ctx) + targets, err := client.targets(ctx, stream) if ctx.Err() != nil { return nil } if err != nil { return err } - out.Report(summary) - for _, scrape := range scrapes { - if err := out.Emit(ctx, scrape); err != nil { - if ctx.Err() != nil { - return nil - } - return err - } - } + out.Report(targetHealth(client.server, targets)) select { case <-ctx.Done(): return nil @@ -160,7 +146,37 @@ func (kind) Handle(ctx dbcontext.Context, params Params, out tracing.Emitter[Scr } } +func targetHealth(server Server, targets []scrapeTarget) Summary { + summary := Summary{Prometheus: server.URL, Job: server.Job, Targets: len(targets)} + for _, target := range targets { + if !target.LastScrape.After(time.Unix(0, 0)) { + continue + } + switch target.Health { + case "up": + summary.Up++ + case "down": + summary.Down++ + if summary.Errors == nil { + summary.Errors = make(map[string]string) + } + summary.Errors[target.Labels[LabelEndpoint]] = target.LastError + } + } + return summary +} + +// Summary reports the current status of the capture's targets. +type Summary struct { + Prometheus string `json:"prometheus"` + Job string `json:"job"` + Targets int `json:"targets"` + Up int `json:"up"` + Down int `json:"down"` + Errors map[string]string `json:"errors,omitempty"` +} + // NewKind is the prometheus kind over options. func NewKind(options KindOptions) *tracing.Handler[Params, Scrape] { - return tracing.NewHandler[Params, Scrape](kind{options: options}, tracing.Capabilities{Live: true}, target.Host) + return tracing.NewHandler[Params, Scrape](kind{options: options}, tracing.Capabilities{Live: true}, target.Host).WithTop(topSpec()) } diff --git a/pkg/tracing/prometheus/prometheus_live_test.go b/pkg/tracing/prometheus/prometheus_live_test.go index ad5538e..f0e302f 100644 --- a/pkg/tracing/prometheus/prometheus_live_test.go +++ b/pkg/tracing/prometheus/prometheus_live_test.go @@ -1,112 +1,84 @@ -// A live spec of the prometheus kind against the managed Prometheus it starts -// through deps: an endpoint's gauge is scraped into its TSDB with the -// session's labels. - package prometheus_test import ( "context" - "encoding/json" "fmt" "net/http" "net/http/httptest" - "net/url" "os" + "path/filepath" "time" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/flanksource/deps/start" - "github.com/flanksource/deps/start/state" + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" + _ "github.com/flanksource/commons-db/query/providers" "github.com/flanksource/telltale/pkg/tracing" "github.com/flanksource/telltale/pkg/tracing/prometheus" "github.com/flanksource/telltale/pkg/tracing/tracingtest" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) -// Set TELLTALE_PROMETHEUS_LIVE=1 to run it: the first run downloads -// Prometheus prometheus.Version into ~/.deps. var _ = Describe("prometheus trace kind (live)", func() { - const gauge, value = "telltale_live_spec_gauge", 42 - BeforeEach(func() { if os.Getenv("TELLTALE_PROMETHEUS_LIVE") == "" { - Skip("TELLTALE_PROMETHEUS_LIVE is not set; this spec downloads and starts Prometheus " + prometheus.Version) + Skip("set TELLTALE_PROMETHEUS_LIVE to install and run VictoriaMetrics") } }) - It("scrapes an endpoint's gauge into the managed Prometheus with the session's labels", func() { + It("stores labelled gauge samples in VictoriaMetrics and reads them after restart", func(ctx SpecContext) { + const gauge, value = "telltale_live_spec_gauge", 42 metrics := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = fmt.Fprintf(w, "# TYPE %s gauge\n%s %d\n", gauge, gauge, value) })) DeferCleanup(metrics.Close) endpoint := metrics.URL + "/metrics" - dataDir := GinkgoT().TempDir() - paths, err := prometheus.Paths(dataDir) - Expect(err).NotTo(HaveOccurred()) + oldState := filepath.Join(dataDir, "services", "prometheus", "state.yaml") + Expect(os.MkdirAll(filepath.Dir(oldState), 0o700)).To(Succeed()) + Expect(os.WriteFile(oldState, []byte("previous Prometheus state"), 0o600)).To(Succeed()) managed := prometheus.NewManaged(dataDir) - DeferCleanup(func() { - Expect(managed.Release(context.Background())).To(Succeed()) - Expect(start.Status(context.Background(), "prometheus", start.WithStateDir(paths.State))). - To(Equal(state.StatusStopped), "the last release stops the Prometheus this process supervises") - }) + DeferCleanup(managed.Release, context.Background()) env := tracingtest.NewEnv(map[string]tracing.TracePlugin{ prometheus.Kind: prometheus.NewKind(prometheus.KindOptions{RunID: "run-live", Server: managed.Server}), }) + from := time.Now().UTC().Truncate(time.Second) session := env.Start(prometheus.Kind, fmt.Sprintf(`{"endpoints": [%q], "interval": "1s", "labels": {"env": "dev"}}`, endpoint)) tracingtest.Running(session) - Eventually(func() int64 { return session.Snapshot().EventCount }).WithTimeout(30 * time.Second).Should(BeNumerically(">=", 2)) server := session.Snapshot().Handle - - var gauges struct { - Result []struct { - Metric map[string]string `json:"metric"` - Value [2]any `json:"value"` - } `json:"result"` - } - apiGet(server, "/api/v1/query", url.Values{"query": {gauge + `{telltale_run="run-live"}`}}, &gauges) - var targets struct { - ActiveTargets []struct { - Labels map[string]string `json:"labels"` - ScrapeInterval string `json:"scrapeInterval"` - } `json:"activeTargets"` + profile := query.Profile{Name: "captured-metrics", Query: gauge + `{telltale_run="run-live"}`, + Provider: query.ProviderConfig{Type: "prometheus", Options: map[string]any{"url": server}}} + checkSample := func(g Gomega) { + result, err := query.Execute(dbcontext.NewContext(ctx), profile) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(result.Rows).To(ContainElement(And( + HaveKeyWithValue("value", BeNumerically("==", value)), HaveKeyWithValue("env", "dev"), + HaveKeyWithValue("telltale_run", "run-live"), HaveKeyWithValue("telltale_endpoint", endpoint), + HaveKeyWithValue("telltale_stream", Not(BeEmpty())), + ))) } - apiGet(server, "/api/v1/targets", url.Values{"scrapePool": {prometheus.Job}}, &targets) + Eventually(checkSample).WithTimeout(20 * time.Second).Should(Succeed()) session.Stop("spec") info := tracingtest.Ended(session) Expect(info.Error).To(BeEmpty()) - - Expect(gauges.Result).To(HaveLen(1)) - Expect(gauges.Result[0].Metric).To(Equal(map[string]string{ - "__name__": gauge, "job": prometheus.Job, "instance": metrics.Listener.Addr().String(), "env": "dev", - "telltale_run": "run-live", "telltale_stream": info.Events.Stream, "telltale_endpoint": endpoint, - })) - Expect(gauges.Result[0].Value[1]).To(Equal(fmt.Sprint(value))) - var intervals []string - for _, t := range targets.ActiveTargets { - if t.Labels["telltale_stream"] == info.Events.Stream { - intervals = append(intervals, t.ScrapeInterval) - } + Expect(info.EventCount).To(BeZero()) + Expect(env.Rows(info)).To(BeEmpty()) + frames, err := env.Results.Backend.Meta(ctx, info.Events.Stream+"-frames") + Expect(err).NotTo(HaveOccurred()) + Expect(frames.HighSeq).To(BeZero()) + paths, err := prometheus.Paths(dataDir) + Expect(err).NotTo(HaveOccurred()) + Expect(filepath.Glob(filepath.Join(paths.FileSD, "*.json"))).To(BeEmpty()) + to := time.Now().UTC().Add(time.Second).Truncate(time.Second) + Expect(managed.Release(ctx)).To(Succeed()) + Expect(http.Get(server + "/health")).Error().To(HaveOccurred()) + restarted, err := managed.Server(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(restarted.URL).To(Equal(server)) + profile.Provider.Options["range"] = map[string]any{ + "start": from.Format(time.RFC3339), "end": to.Format(time.RFC3339), "step": "1s", } - Expect(intervals).To(Equal([]string{"1000ms"}), "the target's own __scrape_interval__ label, not the global 15s, sets its interval") - rows := env.Rows(info) - Expect(rows[0]).To(HaveKeyWithValue("health", "up")) - Expect(rows[0]).To(HaveKeyWithValue("samples", BeNumerically(">", 0))) + checkSample(Default) + Expect(os.ReadFile(oldState)).To(Equal([]byte("previous Prometheus state"))) }) }) - -// apiGet decodes the data of a Prometheus API answer into data. -func apiGet(server, path string, query url.Values, data any) { - GinkgoHelper() - response, err := http.Get(server + path + "?" + query.Encode()) - Expect(err).ToNot(HaveOccurred()) - defer func() { _ = response.Body.Close() }() - var answer struct { - Status string `json:"status"` - Data json.RawMessage `json:"data"` - } - Expect(json.NewDecoder(response.Body).Decode(&answer)).To(Succeed()) - Expect(answer.Status).To(Equal("success")) - Expect(json.Unmarshal(answer.Data, data)).To(Succeed()) -} diff --git a/pkg/tracing/prometheus/prometheus_suite_test.go b/pkg/tracing/prometheus/prometheus_suite_test.go index e36d4d5..776f215 100644 --- a/pkg/tracing/prometheus/prometheus_suite_test.go +++ b/pkg/tracing/prometheus/prometheus_suite_test.go @@ -1,5 +1,4 @@ -// Runs the prometheus kind's specs: params, file_sd target groups, the scrape -// records read from a fake Prometheus API, and one live managed Prometheus. +// Runs the prometheus capture's parameter, target, and VictoriaMetrics specs. package prometheus_test diff --git a/pkg/tracing/prometheus/relay.go b/pkg/tracing/prometheus/relay.go new file mode 100644 index 0000000..1d03883 --- /dev/null +++ b/pkg/tracing/prometheus/relay.go @@ -0,0 +1,90 @@ +package prometheus + +import ( + "crypto/rand" + "errors" + "fmt" + "net" + "net/http" + "net/http/httputil" + "net/url" + "time" + + "github.com/flanksource/commons-db/kubernetes" + "github.com/flanksource/telltale/internal/lifetime" +) + +type scrapeRelay struct { + server *http.Server + service *lifetime.Service + mux *http.ServeMux + address string +} + +func (r *scrapeRelay) route(upstream *url.URL) (*url.URL, error) { + identity, err := kubernetes.APIServerProxyFor(upstream) + if err != nil { + return nil, err + } + if identity == nil { + return nil, errors.New("no Kubernetes identity registered for the resolved API server proxy URL") + } + transport, err := identity.Transport() + if err != nil { + return nil, err + } + if r.server == nil { + if err := r.start(); err != nil { + return nil, err + } + } + path := "/scrape/" + rand.Text() + proxy := &httputil.ReverseProxy{ + Transport: transport, + Rewrite: func(request *httputil.ProxyRequest) { + fixed := *upstream + request.Out.URL, request.Out.Host = &fixed, fixed.Host + request.Out.Header = make(http.Header) + for _, name := range []string{"Accept", "Accept-Encoding", "User-Agent", "X-Prometheus-Scrape-Timeout-Seconds"} { + for _, value := range request.In.Header.Values(name) { + request.Out.Header.Add(name, value) + } + } + request.Out.Body, request.Out.ContentLength = nil, 0 + request.Out.TransferEncoding, request.Out.Trailer = nil, nil + }, + } + r.mux.HandleFunc(path, func(w http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodGet { + w.Header().Set("Allow", http.MethodGet) + http.Error(w, "only GET is allowed", http.StatusMethodNotAllowed) + return + } + proxy.ServeHTTP(w, request) + }) + return &url.URL{Scheme: "http", Host: r.address, Path: path}, nil +} + +func (r *scrapeRelay) start() error { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return fmt.Errorf("listen for Kubernetes proxy scrapes: %w", err) + } + r.address = listener.Addr().String() + r.mux = http.NewServeMux() + r.server = &http.Server{Handler: r.mux, ReadHeaderTimeout: 5 * time.Second} + r.service = lifetime.Start("Kubernetes scrape relay", func() error { + if err := r.server.Serve(listener); !errors.Is(err, http.ErrServerClosed) { + return err + } + return nil + }) + return nil +} + +func (r *scrapeRelay) close() error { + if r.server == nil { + return nil + } + return errors.Join(r.server.Close(), r.service.Wait()) +} diff --git a/pkg/tracing/prometheus/relay_internal_test.go b/pkg/tracing/prometheus/relay_internal_test.go new file mode 100644 index 0000000..18f87cf --- /dev/null +++ b/pkg/tracing/prometheus/relay_internal_test.go @@ -0,0 +1,160 @@ +package prometheus + +import ( + "context" + "encoding/json" + "encoding/pem" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + dbkubernetes "github.com/flanksource/commons-db/kubernetes" + "github.com/flanksource/commons/logger" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/tracingtest" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" +) + +const proxyExporter = "proxy://exporter.monitoring/probe%3Fapp?port=http&module=http_2xx" +const proxyMetrics = "fixture_metric 1\n" + +var _ = Describe("Kubernetes scrape endpoints", func() { + It("relays only the configured GET with the Kubernetes identity and closes on release", func(ctx SpecContext) { + endpoints, release, err := resolveEndpoints(kubernetesProxyFixture(ctx), []string{proxyExporter}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(release) + Expect(endpoints).To(HaveLen(1)) + Expect(endpoints[0].original).To(Equal(proxyExporter)) + relay := endpoints[0].url.String() + request, err := http.NewRequestWithContext(ctx, http.MethodGet, relay+"?module=other", nil) + Expect(err).NotTo(HaveOccurred()) + request.Header.Set("Authorization", "Bearer caller") + request.Header.Set("Impersonate-User", "caller") + response, err := http.DefaultClient.Do(request) + Expect(err).NotTo(HaveOccurred()) + Expect(response.StatusCode).To(Equal(http.StatusOK)) + body, err := io.ReadAll(response.Body) + Expect(response.Body.Close()).To(Succeed()) + Expect(err).NotTo(HaveOccurred()) + Expect(string(body)).To(Equal(proxyMetrics)) + + for _, attempt := range []struct { + method, address string + status int + }{ + {http.MethodPost, relay, http.StatusMethodNotAllowed}, + {http.MethodGet, relay + "/other", http.StatusNotFound}, + } { + request, err := http.NewRequestWithContext(ctx, attempt.method, attempt.address, nil) + Expect(err).NotTo(HaveOccurred()) + response, err := http.DefaultClient.Do(request) + Expect(err).NotTo(HaveOccurred()) + Expect(response.Body.Close()).To(Succeed()) + Expect(response.StatusCode).To(Equal(attempt.status)) + } + Expect(release()).To(Succeed()) + response, err = http.DefaultClient.Do(request) + if response != nil { + Expect(response.Body.Close()).To(Succeed()) + } + Expect(err).To(HaveOccurred()) + }) + + It("removes the scrape target file and closes the relay when discovery fails", func(ctx SpecContext) { + dir := GinkgoT().TempDir() + relay := make(chan string, 1) + scraper := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/-/ready" { + w.WriteHeader(http.StatusOK) + return + } + files, err := filepath.Glob(filepath.Join(dir, "*.json")) + if err == nil && len(files) == 1 { + groups, err := os.ReadFile(files[0]) + var targets []targetGroup + if err == nil && json.Unmarshal(groups, &targets) == nil && len(targets) == 1 && len(targets[0].Targets) == 1 { + relay <- "http://" + targets[0].Targets[0] + targets[0].Labels["__metrics_path__"] + } + } + http.Error(w, "discovery unavailable", http.StatusServiceUnavailable) + })) + DeferCleanup(scraper.Close) + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{ + Kind: NewKind(KindOptions{RunID: "proxy-failure", Server: func(context.Context) (Server, error) { + return Server{URL: scraper.URL, Job: Job, FileSDDir: dir}, nil + }}), + }) + _, err := env.Runtime.Start(kubernetesProxyFixture(ctx), proxyCaptureRequest()) + Expect(err).To(MatchError(ContainSubstring("503"))) + Expect(os.ReadDir(dir)).To(BeEmpty()) + var address string + Expect(relay).To(Receive(&address)) + response, err := http.Get(address) + if response != nil { + Expect(response.Body.Close()).To(Succeed()) + } + Expect(err).To(HaveOccurred()) + }) + + It("stores samples from the authenticated proxy with the original label (live)", func(ctx SpecContext) { + if os.Getenv("TELLTALE_PROMETHEUS_LIVE") == "" { + Skip("set TELLTALE_PROMETHEUS_LIVE to run the managed scraper") + } + managed := NewManaged(GinkgoT().TempDir()) + DeferCleanup(managed.Release, context.Background()) + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{ + Kind: NewKind(KindOptions{RunID: "proxy-live", Server: managed.Server}), + }) + capture, err := env.Runtime.Start(kubernetesProxyFixture(ctx), proxyCaptureRequest()) + Expect(err).NotTo(HaveOccurred()) + session := capture.Session() + tracingtest.Running(session) + client := newClient(Server{URL: session.Snapshot().Handle, Job: Job}) + Eventually(func(g Gomega) { + var series struct { + Result []struct{ Metric map[string]string } + } + g.Expect(client.get(ctx, "/api/v1/query", url.Values{"query": {`fixture_metric{telltale_run="proxy-live"}`}}, &series)).To(Succeed()) + g.Expect(series.Result).To(ContainElement(HaveField("Metric", HaveKeyWithValue(LabelEndpoint, proxyExporter)))) + }).WithTimeout(20 * time.Second).Should(Succeed()) + session.Stop("spec") + Expect(tracingtest.Ended(session).Error).To(BeEmpty()) + }) +}) + +func proxyCaptureRequest() tracing.StartRequest { + params, err := json.Marshal(Options{Endpoints: []string{proxyExporter}, Interval: "1s"}) + Expect(err).NotTo(HaveOccurred()) + return tracing.StartRequest{Kind: Kind, Params: params} +} + +func kubernetesProxyFixture(ctx SpecContext) dbcontext.Context { + const credential = "fixture-identity" + const path = "/api/v1/namespaces/monitoring/services/http:exporter:http/proxy/probe%3Fapp" + upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer "+credential || r.Header.Get("Impersonate-User") != "" { + http.Error(w, "invalid identity", http.StatusUnauthorized) + return + } + if r.Method != http.MethodGet || r.URL.EscapedPath() != path || r.URL.RawQuery != "module=http_2xx" { + http.Error(w, "incorrect scrape request", http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "text/plain; version=0.0.4") + _, _ = io.WriteString(w, proxyMetrics) + })) + DeferCleanup(upstream.Close) + config := &rest.Config{Host: upstream.URL, BearerToken: credential, TLSClientConfig: rest.TLSClientConfig{ + CAData: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: upstream.Certificate().Raw}), + }} + client := dbkubernetes.NewKubeClient(logger.GetLogger("test"), fake.NewSimpleClientset(), config) + return dbcontext.NewContext(ctx).WithLocalKubernetes(client) +} diff --git a/pkg/tracing/prometheus/summary.go b/pkg/tracing/prometheus/summary.go new file mode 100644 index 0000000..8e0b85d --- /dev/null +++ b/pkg/tracing/prometheus/summary.go @@ -0,0 +1,25 @@ +package prometheus + +import "iter" + +func (kind) Summarize(rows iter.Seq[Scrape]) (map[string]any, error) { + latest := map[string]Scrape{} + for row := range rows { + if previous, ok := latest[row.Endpoint]; !ok || row.TakenAt.After(previous.TakenAt) { + latest[row.Endpoint] = row + } + } + up, down := 0, 0 + var samples int64 + duration := 0.0 + for _, row := range latest { + if row.Health == "up" { + up++ + } else { + down++ + } + samples += row.Samples + duration += row.DurationMs + } + return map[string]any{"targets": len(latest), "up": up, "down": down, "latestSamples": samples, "latestDurationMs": duration}, nil +} diff --git a/pkg/tracing/prometheus/summary_test.go b/pkg/tracing/prometheus/summary_test.go new file mode 100644 index 0000000..afb2220 --- /dev/null +++ b/pkg/tracing/prometheus/summary_test.go @@ -0,0 +1,17 @@ +package prometheus + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "slices" + "time" +) + +var _ = Describe("scrape monitor summary", func() { + It("counts latest target health instead of summing historical gauges", func() { + rows := []Scrape{{Endpoint: "a", TakenAt: time.Unix(1, 0), Health: "down", Samples: 3}, {Endpoint: "a", TakenAt: time.Unix(2, 0), Health: "up", Samples: 7}, {Endpoint: "b", TakenAt: time.Unix(2, 0), Health: "down", Samples: 2}} + summary, err := (kind{}).Summarize(slices.Values(rows)) + Expect(err).NotTo(HaveOccurred()) + Expect(summary).To(Equal(map[string]any{"targets": 2, "up": 1, "down": 1, "latestSamples": int64(9), "latestDurationMs": float64(0)})) + }) +}) diff --git a/pkg/tracing/prometheus/table.go b/pkg/tracing/prometheus/table.go new file mode 100644 index 0000000..a1c1fc7 --- /dev/null +++ b/pkg/tracing/prometheus/table.go @@ -0,0 +1,17 @@ +package prometheus + +import ( + "github.com/flanksource/clicky/api" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/telltale/pkg/tracing" +) + +func (Scrape) Columns() []api.ColumnDef { + return query.TableColumnsFor[Scrape]("endpoint", "health", "durationMs", "samples", "error") +} +func (s Scrape) Row() map[string]any { return query.TableRowFor(s) } +func (s Scrape) RowDetail() api.Textable { return s.PrettyFull() } + +func topSpec() tracing.TopSpec { + return tracing.TopSpec{PerSource: true, Identity: []string{"endpoint"}, Order: query.Order{{Column: "health"}, {Column: "durationMs", Desc: true}, {Column: "endpoint"}}} +} diff --git a/pkg/tracing/prometheus/watch.go b/pkg/tracing/prometheus/watch.go deleted file mode 100644 index 6a698bb..0000000 --- a/pkg/tracing/prometheus/watch.go +++ /dev/null @@ -1,121 +0,0 @@ -package prometheus - -import ( - "context" - "fmt" - "slices" - "time" -) - -// commitWait bounds how long a scrape Prometheus reports may go without its -// scrape_samples_scraped sample: it reports the scrape before it commits it. -const commitWait = 5 * time.Second - -// Summary is the kind's report of its capture. -type Summary struct { - Prometheus string `json:"prometheus"` - Job string `json:"job"` - Targets int `json:"targets"` - Up int `json:"up"` - Down int `json:"down"` - Scrapes int `json:"scrapes"` -} - -// watcher turns a session's targets into one Scrape each time a target's -// last scrape advances. -type watcher struct { - client *client - stream string - interval time.Duration - last map[string]time.Time - scrapes int -} - -// poll reads the session's targets and the scrapes completed since the last -// poll, with what the capture has seen so far. -func (w *watcher) poll(ctx context.Context) ([]Scrape, Summary, error) { - targets, err := w.client.targets(ctx, w.stream) - if err != nil { - return nil, Summary{}, err - } - summary := Summary{Prometheus: w.client.server.URL, Job: w.client.server.Job, Targets: len(targets)} - var advanced []scrapeTarget - for _, t := range targets { - switch t.Health { - case "up": - summary.Up++ - case "down": - summary.Down++ - } - if t.LastScrape.After(w.last[t.Labels[LabelEndpoint]]) { - advanced = append(advanced, t) - } - } - if len(advanced) == 0 { - summary.Scrapes = w.scrapes - return nil, summary, nil - } - samples, err := w.samplesScraped(ctx, advanced) - if err != nil { - return nil, Summary{}, err - } - scrapes := make([]Scrape, 0, len(advanced)) - for index, t := range advanced { - endpoint := t.Labels[LabelEndpoint] - w.last[endpoint] = t.LastScrape - scrapes = append(scrapes, Scrape{ - TakenAt: t.LastScrape.UTC(), Endpoint: endpoint, Health: t.Health, - DurationMs: t.LastScrapeDuration * 1000, Samples: int64(samples[index]), Error: t.LastError, - }) - } - w.scrapes += len(scrapes) - summary.Scrapes = w.scrapes - return scrapes, summary, nil -} - -// samplesScraped is the scrape_samples_scraped sample of each scrape, the one -// nearest its start within half an interval, waiting up to commitWait for -// Prometheus to commit those it reported before committing them. -func (w *watcher) samplesScraped(ctx context.Context, scraped []scrapeTarget) ([]float64, error) { - promql := fmt.Sprintf(`scrape_samples_scraped{%s=%q}[%dms]`, LabelStream, w.stream, (2*w.interval + commitWait).Milliseconds()) - deadline := time.Now().Add(commitWait) - for { - series, err := w.client.series(ctx, promql) - if err != nil { - return nil, err - } - samples, missing := matchScrapes(series, scraped, w.interval/2) - if missing == nil { - return samples, nil - } - if time.Now().After(deadline) { - return nil, fmt.Errorf("prometheus %s has no scrape_samples_scraped sample for the scrape of %s at %s after %s", - w.client.server.URL, missing.Labels[LabelEndpoint], missing.LastScrape.Format(time.RFC3339Nano), commitWait) - } - if err := sleep(ctx, 100*time.Millisecond); err != nil { - return nil, err - } - } -} - -// matchScrapes is the value series holds for each scrape, the point of its -// endpoint nearest its last scrape within tolerance, or the first scrape that -// has none. -func matchScrapes(series map[string][]point, scraped []scrapeTarget, tolerance time.Duration) ([]float64, *scrapeTarget) { - samples := make([]float64, len(scraped)) - for index, t := range scraped { - points := series[t.Labels[LabelEndpoint]] - distance := func(p point) time.Duration { return max(p.at.Sub(t.LastScrape), t.LastScrape.Sub(p.at)) } - nearest := slices.IndexFunc(points, func(p point) bool { return distance(p) <= tolerance }) - if nearest < 0 { - return nil, &scraped[index] - } - for candidate, p := range points { - if distance(p) < distance(points[nearest]) { - nearest = candidate - } - } - samples[index] = points[nearest].value - } - return samples, nil -} diff --git a/pkg/tracing/query/params.go b/pkg/tracing/query/params.go new file mode 100644 index 0000000..649e253 --- /dev/null +++ b/pkg/tracing/query/params.go @@ -0,0 +1,223 @@ +package query + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + dbquery "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/query/datetime" + "github.com/flanksource/commons-db/query/providers" + "github.com/flanksource/commons-db/query/providers/sqlite" +) + +// Params choose what a capture reads, a saved profile or a provider read in +// place, and the window and pace it reads at. +type Params struct { + // Profile names a saved query profile, resolved through the host's + // ProfileResolver. + Profile string `json:"profile,omitempty" clicky:"title=Profile,desc=Saved query profile to read,order=0"` + // Provider reads a commons-db query provider in place of a saved profile. + Provider *Provider `json:"provider,omitempty" clicky:"title=Provider,order=1"` + // Params are the profile's declared params by name. + Params map[string]any `json:"params,omitempty" clicky:"title=Params,desc=The profile's params by name,order=2"` + // From and To bound the window read, as a time or date math such as + // now-1h. To is empty for now; a window needs a From. + From string `json:"from,omitempty" clicky:"title=From,desc=Start of the window; a time or date math such as now-1h,order=3"` + To string `json:"to,omitempty" clicky:"title=To,desc=End of the window; empty is now,order=4"` + // Every re-reads the window from the last row kept up to now, at this + // interval, until the capture stops. + Every string `json:"every,omitempty" clicky:"title=Every,desc=Re-read the rows written since the last read at this interval; e.g. 15s,order=5"` + // UntilStopped reads once, when the capture stops, over the whole capture. + UntilStopped bool `json:"untilStopped,omitempty" clicky:"title=Until stopped,desc=Read once when the capture stops; up to that moment,order=6"` + // Limit is the most rows one read keeps; 0 keeps every row. + Limit int `json:"limit,omitempty" clicky:"title=Limit,desc=Most rows one read keeps; 0 keeps all,order=7"` +} + +// Provider is a provider read in place: the commons-db provider type, its +// connection and query, and the typed options of that one provider. +type Provider struct { + // Type is the registered provider key, such as sql, prometheus or opensearch. + Type string `json:"type" clicky:"required,title=Type"` + Connection string `json:"connection,omitempty" clicky:"title=Connection"` + Query string `json:"query,omitempty" clicky:"title=Query"` + + SQL *sqlite.SQLOptions `json:"sql,omitempty"` + Prometheus *providers.PrometheusOptions `json:"prometheus,omitempty"` + OpenSearch *providers.OpenSearchOptions `json:"opensearch,omitempty"` + HTTP *providers.HTTPOptions `json:"http,omitempty"` + PostgREST *providers.PostgRESTOptions `json:"postgrest,omitempty"` + BigQuery *providers.BigQueryOptions `json:"bigquery,omitempty"` + Jaeger *providers.JaegerOptions `json:"jaeger,omitempty"` + OpenTelemetry *providers.OpenTelemetryOptions `json:"opentelemetry,omitempty"` + + CloudWatchMetrics *providers.CloudWatchMetricsOptions `json:"cloudwatchmetrics,omitempty"` +} + +// typedOptions is one provider's options field: the provider types it +// configures, whether it is set, and its options decoded and checked from a +// provider config's free-form map. +type typedOptions struct { + field string + types []string + value any + set bool + check func(options map[string]any) error +} + +func checked[T interface{ Validate() error }](options map[string]any) error { + decoded, err := dbquery.DecodeOptions[T](options) + if err != nil { + return err + } + return decoded.Validate() +} + +func (p Provider) typed() []typedOptions { + return []typedOptions{ + {"sql", []string{"sql", "sqlite", "postgres", "mysql", "sqlserver", "clickhouse"}, p.SQL, p.SQL != nil, checked[sqlite.SQLOptions]}, + {"prometheus", []string{"prometheus"}, p.Prometheus, p.Prometheus != nil, checked[providers.PrometheusOptions]}, + {"opensearch", []string{"opensearch"}, p.OpenSearch, p.OpenSearch != nil, checked[providers.OpenSearchOptions]}, + {"http", []string{"http"}, p.HTTP, p.HTTP != nil, checked[providers.HTTPOptions]}, + {"postgrest", []string{"postgrest"}, p.PostgREST, p.PostgREST != nil, checked[providers.PostgRESTOptions]}, + {"bigquery", []string{"bigquery"}, p.BigQuery, p.BigQuery != nil, checked[providers.BigQueryOptions]}, + {"jaeger", []string{"jaeger"}, p.Jaeger, p.Jaeger != nil, checked[providers.JaegerOptions]}, + {"opentelemetry", []string{"opentelemetry"}, p.OpenTelemetry, p.OpenTelemetry != nil, checked[providers.OpenTelemetryOptions]}, + {"cloudwatchmetrics", []string{"cloudwatchmetrics"}, p.CloudWatchMetrics, p.CloudWatchMetrics != nil, checked[providers.CloudWatchMetricsOptions]}, + } +} + +// options finds the options field Type reads, refusing a type the kind does +// not read and options set for any other provider. +func (p Provider) options() (typedOptions, error) { + var found *typedOptions + var known []string + for _, candidate := range p.typed() { + known = append(known, candidate.types...) + reads := false + for _, kind := range candidate.types { + reads = reads || kind == p.Type + } + if reads { + found = &candidate + } else if candidate.set { + return typedOptions{}, fmt.Errorf("provider type %q takes no %s options", p.Type, candidate.field) + } + } + if found == nil { + return typedOptions{}, fmt.Errorf("provider type %q is not one this kind reads; one of %s", p.Type, strings.Join(known, ", ")) + } + if p.Type == "sqlite" && p.SQL != nil && p.SQL.Type != "" && p.SQL.Type != "sqlite" { + return typedOptions{}, fmt.Errorf("provider sqlite always reads sqlite, so sql.type %q is never used; use provider sql", p.SQL.Type) + } + return *found, nil +} + +// profile is the in-place profile the provider reads. +func (p Provider) profile() (dbquery.Profile, error) { + typed, err := p.options() + if err != nil { + return dbquery.Profile{}, err + } + profile := dbquery.Profile{Provider: dbquery.ProviderConfig{Type: p.Type, Connection: p.Connection}, Query: p.Query} + if typed.set { + encoded, err := json.Marshal(typed.value) + if err != nil { + return dbquery.Profile{}, fmt.Errorf("%s options: %w", typed.field, err) + } + if err := json.Unmarshal(encoded, &profile.Provider.Options); err != nil { + return dbquery.Profile{}, fmt.Errorf("%s options: %w", typed.field, err) + } + } + return profile, nil +} + +// checkOptions checks a (windowed) in-place profile's options as its provider +// will read them. +func (p Provider) checkOptions(profile dbquery.Profile) error { + typed, err := p.options() + if err != nil { + return err + } + return typed.check(profile.Provider.Options) +} + +func (p Params) Validate() error { + if (p.Profile == "") == (p.Provider == nil) { + return errors.New("name exactly one of profile or provider: a saved profile, or a provider read in place") + } + if p.Limit < 0 { + return fmt.Errorf("limit %d is negative", p.Limit) + } + if _, err := p.every(); err != nil { + return err + } + if _, err := p.window(time.Now()); err != nil { + return err + } + if p.Provider == nil { + return nil + } + profile, err := p.Provider.profile() + if err != nil { + return err + } + if p.windowed() { + now := time.Now() + if profile, _, err = dbquery.WindowProfile(profile, p.Params, dbquery.ReadWindow{Start: now, End: now}); err != nil { + return err + } + } + return p.Provider.checkOptions(profile) +} + +// windowed reports whether a read is bounded by a window: one with a From, +// read every interval, or read at the stop. +func (p Params) windowed() bool { return p.From != "" || p.Every != "" || p.UntilStopped } + +func (p Params) every() (time.Duration, error) { + if p.Every == "" { + return 0, nil + } + if p.UntilStopped { + return 0, errors.New("every re-reads while the capture runs and untilStopped reads once at its stop; pick one") + } + if p.To != "" { + return 0, errors.New("every follows the rows up to now, so it takes no to") + } + every, err := time.ParseDuration(p.Every) + if err != nil || every <= 0 { + return 0, fmt.Errorf("every %q is not a positive duration such as 15s", p.Every) + } + return every, nil +} + +// bounds are a capture's window bounds evaluated at its start: a zero start +// is the capture's start, a zero end is the moment each read runs. +type bounds struct{ start, end time.Time } + +func (p Params) window(now time.Time) (bounds, error) { + var window bounds + if p.To != "" && p.UntilStopped { + return window, errors.New("untilStopped ends the window when the capture stops, so it takes no to") + } + if p.To != "" && p.From == "" { + return window, fmt.Errorf("to %q needs a from to start the window", p.To) + } + for _, bound := range []struct { + name, value string + into *time.Time + }{{"from", p.From, &window.start}, {"to", p.To, &window.end}} { + if bound.value == "" { + continue + } + parsed, err := datetime.Parse(bound.value, now) + if err != nil { + return window, fmt.Errorf("%s: %w", bound.name, err) + } + *bound.into = parsed.Time.UTC() + } + return window, nil +} diff --git a/pkg/tracing/query/pretty.go b/pkg/tracing/query/pretty.go new file mode 100644 index 0000000..46422be --- /dev/null +++ b/pkg/tracing/query/pretty.go @@ -0,0 +1,19 @@ +package query + +import ( + "maps" + "slices" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" +) + +func (r Row) Pretty() api.Text { + t := clicky.Text(r.Profile, "text-blue-500 font-bold").Space().Append(r.Provider, "text-muted") + for _, key := range slices.Sorted(maps.Keys(r.Values)) { + t = t.Space().Append(key+"=", "text-muted").Add(clicky.Human(r.Values[key])) + } + return t +} + +func (r Row) PrettyFull() api.Textable { return r.Pretty().NewLine().Add(clicky.Map(r.Values)) } diff --git a/pkg/tracing/query/pretty_test.go b/pkg/tracing/query/pretty_test.go new file mode 100644 index 0000000..885e0dd --- /dev/null +++ b/pkg/tracing/query/pretty_test.go @@ -0,0 +1,23 @@ +package query_test + +import ( + "encoding/json" + "strings" + + tracequery "github.com/flanksource/telltale/pkg/tracing/query" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("query record presentation", func() { + It("sorts dynamic fields without changing their stored values", func() { + r := tracequery.Row{Profile: "request-rate", Provider: "prometheus", Values: map[string]any{"z": 9, "a": 2}} + before, err := json.Marshal(r) + Expect(err).NotTo(HaveOccurred()) + Expect(strings.Join(strings.Fields(r.Pretty().String()), " ")).To(Equal("request-rate prometheus a=2 z=9")) + Expect(r.PrettyFull().String()).To(ContainSubstring("request-rate")) + after, err := json.Marshal(r) + Expect(err).NotTo(HaveOccurred()) + Expect(after).To(Equal(before)) + }) +}) diff --git a/pkg/tracing/query/query.go b/pkg/tracing/query/query.go new file mode 100644 index 0000000..80a9931 --- /dev/null +++ b/pkg/tracing/query/query.go @@ -0,0 +1,116 @@ +// Package query is the query trace kind: a commons-db query profile, saved or +// read in place through a provider's typed options, read once over a window, +// once when the capture stops, or every interval, each row a record. +package query + +import ( + "context" + "errors" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + dbquery "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/query/profilestore" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/target" +) + +// ProfileResolver resolves a saved profile by name, imports applied, as the +// host's profile store holds it. +type ProfileResolver interface { + Resolve(ctx context.Context, name string) (dbquery.Profile, error) +} + +// Options are the host seams the kind reads through. +type Options struct { + // Sampled requires an emitter that atomically publishes complete read windows. + Sampled bool + // Profiles resolves a params' saved profile; nil refuses every capture + // that names one. + Profiles ProfileResolver + // BeforeExecute prepares, and may refuse, every read before it runs, as + // every other read of a profile's data is prepared. + BeforeExecute profilestore.BeforeExecuteFunc +} + +// Row is one row a read returned. +type Row struct { + // Time is the row's time column, time or timestamp; a row of a read that + // has none is timed by the moment it was read. + Time time.Time `json:"time" pretty:"label=Time" sort:"time"` + // Profile is the saved profile read; empty for a provider read in place. + Profile string `json:"profile,omitempty" filter:"terms"` + Provider string `json:"provider" filter:"terms"` + // Row is the provider's row as it returned it. + Values map[string]any `json:"row" pretty:"hide"` +} + +// Reads is the kind's trace handler: a host whose params carry more than +// Params wraps it in its own handler, as pkg/traces does with its target. +type Reads struct{ options Options } + +// NewReads is the kind's handler over options' seams. +func NewReads(options Options) (Reads, error) { + if options.BeforeExecute == nil { + return Reads{}, errors.New("the query kind requires a BeforeExecute to prepare each read") + } + return Reads{options: options}, nil +} + +func (Reads) Params() Params { return Params{} } + +func (Reads) Schema() recordresults.ResultType[Row] { + return recordresults.ResultType[Row]{Title: "Query profile reads", TimeColumn: "time"} +} + +type readerKey struct{} + +// Prepare resolves the profile and checks it can be windowed as the params +// ask, so a capture that could not read refuses its start. +func (r Reads) Prepare(ctx dbcontext.Context, params Params, _ tracing.Emitter[Row]) (dbcontext.Context, func() error, error) { + read, err := r.reader(ctx, params, time.Now().UTC()) + if err != nil { + return ctx, nil, err + } + return ctx.WithValue(readerKey{}, read), nil, nil +} + +// Handle reads once and ends; with UntilStopped it reads when the capture +// stops, and with Every it reads at each interval until it stops. +func (Reads) Handle(ctx dbcontext.Context, params Params, records tracing.Emitter[Row], _ tracing.Records[Row]) error { + read, ok := ctx.Value(readerKey{}).(*reader) + if !ok { + return errors.New("query capture handled without the reader its Prepare set up") + } + every, err := params.every() + if err != nil { + return err + } + switch { + case every > 0: + return read.follow(ctx, every, records) + case params.UntilStopped: + <-ctx.Done() + // A stopped capture still takes what its handler emits before it + // returns, so the read outlives the stop that triggered it. + stopped := ctx.Wrap(context.WithoutCancel(ctx)) + return read.once(stopped, time.Now().UTC(), records) + default: + return read.once(ctx, time.Now().UTC(), records) + } +} + +// Capabilities are what a query capture can do: read a past window, and follow +// with Every. +var Capabilities = tracing.Capabilities{Historical: true, Follow: true} + +// Kind is the query trace kind over options' seams, armed on the host: a read +// runs where the runtime does. +func Kind(options Options) (*tracing.Handler[Params, Row], error) { + handler, err := NewReads(options) + if err != nil { + return nil, err + } + return tracing.NewHandler[Params, Row](handler, Capabilities, target.Host), nil +} diff --git a/pkg/tracing/query/query_spec_test.go b/pkg/tracing/query/query_spec_test.go new file mode 100644 index 0000000..06b177d --- /dev/null +++ b/pkg/tracing/query/query_spec_test.go @@ -0,0 +1,251 @@ +// Specs for the query trace kind: a saved profile and a typed provider entry +// read once, a prometheus range windowed onto the read, every appending only +// rows past the last one it kept, untilStopped reading at the stop, and the +// params a capture could not read with refused. + +package query_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strconv" + "sync" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + dbcontext "github.com/flanksource/commons-db/context" + dbquery "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/query/profilestore" + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/query" + "github.com/flanksource/telltale/pkg/tracing/tracingtest" +) + +// savedProfiles is the host's profile store, by name. +type savedProfiles map[string]dbquery.Profile + +func (s savedProfiles) Resolve(_ context.Context, name string) (dbquery.Profile, error) { + if profile, ok := s[name]; ok { + return profile, nil + } + return dbquery.Profile{}, fmt.Errorf("profile %q not found", name) +} + +// gate records every read the kind prepares. +type gate struct { + mu sync.Mutex + reads []profilestore.ReadRequest +} + +func (g *gate) BeforeExecute(_ context.Context, reads []profilestore.ReadRequest) (func(), error) { + g.mu.Lock() + defer g.mu.Unlock() + g.reads = append(g.reads, reads...) + return func() {}, nil +} + +func (g *gate) Reads() []profilestore.ReadRequest { + g.mu.Lock() + defer g.mu.Unlock() + return append([]profilestore.ReadRequest(nil), g.reads...) +} + +// stamp is how the specs' SQL rows spell a time: what SQLite's julianday reads. +func stamp(t time.Time) string { return t.UTC().Format("2006-01-02T15:04:05.000Z") } + +// windowedRows is a SQL read of rows at the given times, bounded by the window +// inclusively at both ends, so a re-read returns the row it ended on again. +func windowedRows(rows map[string]time.Time) string { + values := "" + for id, at := range rows { + if values != "" { + values += ", " + } + values += fmt.Sprintf("('%s', '%s')", id, stamp(at)) + } + return fmt.Sprintf(`SELECT column1 AS id, column2 AS timestamp FROM (VALUES %s) + WHERE julianday(column2) >= julianday({{.params.from}}) AND julianday(column2) <= julianday({{.params.to}})`, values) +} + +func sqlEntry(statement string, extra string) string { + encoded, err := json.Marshal(statement) + Expect(err).ToNot(HaveOccurred()) + return fmt.Sprintf(`{"provider": {"type": "sql", "sql": {"type": "sqlite", "url": ":memory:"}, "query": %s}%s}`, encoded, extra) +} + +// recorded is the provider row a record carries. +func recorded(row recordstore.Row) map[string]any { + switch value := row["row"].(type) { + case map[string]any: + return value + case string: + var decoded map[string]any + Expect(json.Unmarshal([]byte(value), &decoded)).To(Succeed()) + return decoded + } + Fail(fmt.Sprintf("record row is %T", row["row"])) + return nil +} + +func ids(rows []recordstore.Row) []string { + var out []string + for _, row := range rows { + out = append(out, fmt.Sprint(recorded(row)["id"])) + } + return out +} + +var _ = Describe("query trace kind", func() { + var env tracingtest.Env + var reads *gate + saved := savedProfiles{ + "work-items": { + Name: "work-items", + Provider: dbquery.ProviderConfig{Type: "sqlite", Options: map[string]any{"url": ":memory:"}}, + Query: `SELECT column1 AS id, column2 AS time FROM (VALUES ('w1', '2026-10-05T09:00:00Z'), ('w2', '2026-10-05T09:01:00Z'))`, + }, + } + + BeforeEach(func() { + reads = &gate{} + kind, err := query.Kind(query.Options{Profiles: saved, BeforeExecute: reads.BeforeExecute}) + Expect(err).ToNot(HaveOccurred()) + env = tracingtest.NewEnv(map[string]tracing.TracePlugin{"query": kind}) + }) + + It("reads a saved profile through the host's resolver, behind BeforeExecute", func() { + info := tracingtest.Ended(env.Start("query", `{"profile": "work-items"}`)) + Expect(info.Error).To(BeEmpty()) + Expect(info.State).To(Equal(dbquery.SessionCompleted)) + rows := env.Rows(info) + Expect(ids(rows)).To(Equal([]string{"w1", "w2"})) + Expect(rows[0]["profile"]).To(Equal("work-items")) + Expect(rows[0]["provider"]).To(Equal("sqlite")) + Expect(fmt.Sprint(rows[1]["time"])).To(ContainSubstring("2026-10-05")) + Expect(reads.Reads()).To(HaveLen(1)) + Expect(reads.Reads()[0].Profile.Name).To(Equal("work-items")) + }) + + It("reads a typed sql entry in place", func() { + info := tracingtest.Ended(env.Start("query", sqlEntry(`SELECT 'a' AS id`, ""))) + Expect(info.Error).To(BeEmpty()) + rows := env.Rows(info) + Expect(ids(rows)).To(Equal([]string{"a"})) + Expect(rows[0]["provider"]).To(Equal("sql")) + }) + + It("windows a prometheus range onto the read", func() { + var asked []string + var mu sync.Mutex + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer GinkgoRecover() + Expect(r.ParseForm()).To(Succeed()) + mu.Lock() + asked = append(asked, r.URL.Path, r.Form.Get("start"), r.Form.Get("end"), r.Form.Get("step")) + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprint(w, `{"status":"success","data":{"resultType":"matrix","result":[`+ + `{"metric":{"pod":"a"},"values":[[1790000000,"1"],[1790000060,"2"]]}]}}`) + })) + DeferCleanup(server.Close) + params := fmt.Sprintf(`{"provider": {"type": "prometheus", "prometheus": {"url": %q, "range": {"step": "1m"}}, "query": "up"}, "from": "now-1h"}`, server.URL) + info := tracingtest.Ended(env.Start("query", params)) + Expect(info.Error).To(BeEmpty()) + Expect(env.Rows(info)).To(HaveLen(2)) + + mu.Lock() + defer mu.Unlock() + Expect(asked).To(HaveLen(4)) + Expect(asked[0]).To(Equal("/api/v1/query_range")) + start, end := unixSeconds(asked[1]), unixSeconds(asked[2]) + Expect(end - start).To(BeNumerically("~", time.Hour.Seconds(), 5)) + Expect(end).To(BeNumerically("~", float64(time.Now().Unix()), 10)) + Expect(asked[3]).To(Equal("60")) + }) + + It("re-reads every interval, appending only rows past the last one it kept", func() { + now := time.Now() + statement := windowedRows(map[string]time.Time{ + "past": now.Add(-500 * time.Millisecond), "next": now.Add(300 * time.Millisecond), "later": now.Add(700 * time.Millisecond), + }) + session := env.Start("query", sqlEntry(statement, `, "from": "now-1s", "every": "50ms"`)) + Eventually(func() int64 { return session.Snapshot().EventCount }).WithTimeout(5 * time.Second).Should(Equal(int64(3))) + Consistently(func() int64 { return session.Snapshot().EventCount }).WithTimeout(300 * time.Millisecond).Should(Equal(int64(3))) + session.Stop("spec") + info := tracingtest.Ended(session) + Expect(info.State).To(Equal(dbquery.SessionStopped)) + Expect(ids(env.Rows(info))).To(Equal([]string{"past", "next", "later"})) + Expect(len(reads.Reads())).To(BeNumerically(">", 3), "every read is gated") + }) + + It("reads once when stopped, over the whole capture", func() { + now := time.Now() + statement := windowedRows(map[string]time.Time{ + "before": now.Add(-time.Hour), "during": now.Add(150 * time.Millisecond), "after": now.Add(time.Hour), + }) + session := env.Start("query", sqlEntry(statement, `, "untilStopped": true`)) + tracingtest.Running(session) + time.Sleep(300 * time.Millisecond) + Expect(reads.Reads()).To(BeEmpty(), "nothing is read before the stop") + session.Stop("spec") + info := tracingtest.Ended(session) + Expect(info.Error).To(BeEmpty()) + Expect(ids(env.Rows(info))).To(Equal([]string{"during"})) + }) + + It("fails a capture read every interval whose rows carry no time column", func() { + session := env.Start("query", sqlEntry(`SELECT 'a' AS id WHERE {{.params.from}} IS NOT NULL AND {{.params.to}} IS NOT NULL`, `, "every": "50ms"`)) + info := tracingtest.Ended(session) + Expect(info.State).To(Equal(dbquery.SessionFailed)) + Expect(info.Error).To(ContainSubstring("neither time nor timestamp")) + }) + + DescribeTable("refuses params it could not read with", + func(raw, message string) { + kind, err := query.Kind(query.Options{Profiles: saved, BeforeExecute: (&gate{}).BeforeExecute}) + Expect(err).ToNot(HaveOccurred()) + Expect(kind.ValidateParams(json.RawMessage(raw))).To(MatchError(ContainSubstring(message))) + }, + Entry("neither a profile nor a provider", `{}`, "exactly one of profile or provider"), + Entry("both a profile and a provider", `{"profile": "p", "provider": {"type": "sql"}}`, "exactly one of profile or provider"), + Entry("every and untilStopped", `{"profile": "p", "every": "1s", "untilStopped": true}`, "every"), + Entry("an every that is no duration", `{"profile": "p", "every": "often"}`, "every"), + Entry("a to with no from", `{"profile": "p", "to": "now"}`, "to"), + Entry("a provider type it does not read", `{"provider": {"type": "loki"}}`, `provider type "loki"`), + Entry("options of another provider", `{"provider": {"type": "prometheus", "sql": {"type": "sqlite"}}}`, "sql options"), + Entry("the sqlite key with another engine", `{"provider": {"type": "sqlite", "sql": {"type": "postgres"}}}`, "provider sql"), + Entry("invalid typed options", `{"provider": {"type": "http", "http": {"method": "FETCH"}}}`, "unsupported http method"), + Entry("cloudwatch metrics with no range", `{"provider": {"type": "cloudwatchmetrics", "cloudwatchmetrics": {"metrics": [{"name": "CPUUtilization", "namespace": "AWS/EC2"}]}}}`, "requires range"), + Entry("an unbounded in-place read every interval", `{"provider": {"type": "sql", "query": "SELECT 1"}, "every": "1s"}`, "neither params.from nor params.to"), + ) + + It("reads cloudwatch metrics through their typed options", func() { + kind, err := query.Kind(query.Options{Profiles: saved, BeforeExecute: (&gate{}).BeforeExecute}) + Expect(err).ToNot(HaveOccurred()) + Expect(kind.ValidateParams(json.RawMessage(`{"provider": {"type": "cloudwatchmetrics", "connection": "connection://aws", + "cloudwatchmetrics": {"region": "eu-west-1", "range": {"start": "now-1h", "end": "now", "step": "60s"}, "metrics": [{"name": "CPUUtilization", "namespace": "AWS/EC2"}]}}, + "every": "1m"}`))).To(Succeed()) + }) + + It("refuses at start a saved profile the host does not hold", func() { + _, err := env.Runtime.Start(dbcontext.New(), tracing.StartRequest{Kind: "query", Params: json.RawMessage(`{"profile": "missing"}`)}) + Expect(err).To(MatchError(ContainSubstring(`profile "missing" not found`))) + }) + + It("refuses a kind with no BeforeExecute to gate its reads", func() { + _, err := query.Kind(query.Options{Profiles: saved}) + Expect(err).To(MatchError(ContainSubstring("BeforeExecute"))) + }) +}) + +func unixSeconds(value string) float64 { + parsed, err := strconv.ParseFloat(value, 64) + Expect(err).ToNot(HaveOccurred()) + return parsed +} diff --git a/pkg/tracing/query/query_suite_test.go b/pkg/tracing/query/query_suite_test.go new file mode 100644 index 0000000..9a67618 --- /dev/null +++ b/pkg/tracing/query/query_suite_test.go @@ -0,0 +1,16 @@ +// Runs the query trace kind's specs: saved and in-place profile reads, once, +// at stop and every interval, into a real results store. + +package query_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestQueryKind(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Query Trace Kind Suite") +} diff --git a/pkg/tracing/query/reader.go b/pkg/tracing/query/reader.go new file mode 100644 index 0000000..0ac9a7e --- /dev/null +++ b/pkg/tracing/query/reader.go @@ -0,0 +1,254 @@ +package query + +import ( + "context" + "fmt" + "sort" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + dbquery "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/query/datetime" + "github.com/flanksource/commons-db/query/profilestore" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/flanksource/telltale/pkg/tracing" +) + +// timeColumns are the row keys a read's rows are timed by, in preference order. +var timeColumns = []string{"time", "timestamp"} + +// reader is one capture's read: the profile and params it executes, and the +// window it binds onto them. +type reader struct { + columns []dbquery.ColumnDef + sampled bool + profile dbquery.Profile + params map[string]any + // name is the saved profile read, empty for a provider read in place. + name string + // windowed binds [start, end] onto each read; a zero end is the moment + // each read runs. + windowed bool + start, end time.Time + limit int + // timed refuses rows that carry no time column, which a read every + // interval needs to know what it has kept. + timed bool + before profilestore.BeforeExecuteFunc +} + +func (r Reads) reader(ctx dbcontext.Context, params Params, now time.Time) (*reader, error) { + profile, name, err := r.profile(ctx, params) + if err != nil { + return nil, err + } + window, err := params.window(now) + if err != nil { + return nil, err + } + read := &reader{ + sampled: r.options.Sampled, + profile: profile, params: params.Params, name: name, windowed: params.windowed(), + start: window.start, end: window.end, limit: params.Limit, timed: params.Every != "", before: r.options.BeforeExecute, + } + if read.params == nil { + read.params = map[string]any{} + } + if !read.windowed { + return read, nil + } + if read.start.IsZero() { + read.start = now + } + if _, _, err := dbquery.WindowProfile(profile, read.params, dbquery.ReadWindow{Start: read.start, End: read.start}); err != nil { + return nil, err + } + return read, nil +} + +// profile is the profile params read: the provider's in-place profile, or the +// saved one the host resolves. +func (r Reads) profile(ctx context.Context, params Params) (dbquery.Profile, string, error) { + if params.Provider != nil { + profile, err := params.Provider.profile() + return profile, "", err + } + if r.options.Profiles == nil { + return dbquery.Profile{}, "", fmt.Errorf("profile %q cannot be read: the host serves no saved profiles", params.Profile) + } + profile, err := r.options.Profiles.Resolve(ctx, params.Profile) + if err != nil { + return dbquery.Profile{}, "", fmt.Errorf("resolve profile %q: %w", params.Profile, err) + } + return profile, params.Profile, nil +} + +func (r *reader) label() string { + if r.name != "" { + return fmt.Sprintf("profile %q", r.name) + } + return fmt.Sprintf("provider %s", r.profile.Provider.Type) +} + +// read executes the profile over [start, end] when windowed, behind the +// host's BeforeExecute, and returns its rows ordered by time when they carry one. +func (r *reader) read(ctx dbcontext.Context, start, end time.Time) ([]Row, error) { + profile, params := r.profile, r.params + if r.windowed { + var err error + if profile, params, err = dbquery.WindowProfile(r.profile, r.params, dbquery.ReadWindow{Start: start, End: end}); err != nil { + return nil, err + } + } + release, err := r.before(ctx, []profilestore.ReadRequest{{Profile: profile, Params: params}}) + if err != nil { + return nil, fmt.Errorf("prepare the read of %s: %w", r.label(), err) + } + defer release() + result, err := dbquery.Execute(ctx, profile, params) + if err != nil { + return nil, fmt.Errorf("read %s: %w", r.label(), err) + } + return r.records(result.Rows, end) +} + +// records times each row by its time column, or by readAt when the read has +// none, and orders them by it. +func (r *reader) records(rows []dbquery.Row, readAt time.Time) ([]Row, error) { + column := "" + if len(rows) > 0 { + for _, candidate := range timeColumns { + if _, ok := rows[0][candidate]; ok { + column = candidate + break + } + } + } + if column == "" && r.timed && len(rows) > 0 { + return nil, fmt.Errorf("%s is read every interval, but its rows carry neither time nor timestamp to tell new rows from read ones", r.label()) + } + records := make([]Row, 0, len(rows)) + for index, row := range rows { + at := readAt + if column != "" { + var err error + if at, err = rowTime(row[column]); err != nil { + return nil, fmt.Errorf("%s row %d %s: %w", r.label(), index, column, err) + } + } + records = append(records, Row{Time: at, Profile: r.name, Provider: r.profile.Provider.Type, Values: row}) + } + if r.timed { + sort.SliceStable(records, func(i, j int) bool { return records[i].Time.Before(records[j].Time) }) + } + return records, nil +} + +// rowTime reads a time column's value: a time, or text in a date format. +func rowTime(value any) (time.Time, error) { + switch typed := value.(type) { + case time.Time: + return typed.UTC(), nil + case string: + parsed, err := datetime.Parse(typed, time.Time{}) + if err != nil { + return time.Time{}, err + } + return parsed.Time.UTC(), nil + } + return time.Time{}, fmt.Errorf("%v (%T) is neither a time nor a date", value, value) +} + +// kept cuts rows to the read's limit, warning the capture of what it cut. +func (r *reader) kept(rows []Row, records tracing.Emitter[Row]) []Row { + if r.limit == 0 || len(rows) <= r.limit { + return rows + } + records.Warn(fmt.Sprintf("%s returned %d rows; kept the first %d", r.label(), len(rows), r.limit)) + return rows[:r.limit] +} + +// once reads the window up to end, or up to the window's own end, and emits +// every row it keeps. +func (r *reader) once(ctx dbcontext.Context, end time.Time, records tracing.Emitter[Row]) error { + if !r.end.IsZero() { + end = r.end + } + rows, err := r.read(ctx, r.start, end) + if err != nil { + return err + } + return r.emit(ctx, r.start, end, rows, records) +} + +// follow reads [last, now] every interval, emitting only the rows timed past +// the last row it kept, until the capture stops. +func (r *reader) follow(ctx dbcontext.Context, every time.Duration, records tracing.Emitter[Row]) error { + emitCtx := context.WithoutCancel(ctx) + ticker := time.NewTicker(every) + defer ticker.Stop() + last, seen := r.start, false + for { + end := time.Now().UTC() + rows, err := r.read(ctx, last, end) + if err != nil && ctx.Err() == nil { + return err + } + var fresh []Row + for _, row := range rows { + if !seen || row.Time.After(last) { + fresh = append(fresh, row) + } + } + if err == nil { + if err := r.emit(emitCtx, last, end, fresh, records); err != nil { + return err + } + } + retained := fresh + if r.limit > 0 && len(retained) > r.limit { + retained = retained[:r.limit] + } + for _, row := range retained { + last, seen = row.Time, true + } + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + } +} + +func (r *reader) emit(ctx context.Context, start, end time.Time, rows []Row, records tracing.Emitter[Row]) error { + kept := r.kept(rows, records) + if r.sampled { + publisher, ok := records.(interface { + EmitFrame(context.Context, sampleframe.Frame, []Row) error + }) + if !ok { + return fmt.Errorf("sampled query requires an atomic frame emitter") + } + if start.IsZero() { + start = end + } + columns := r.profile.Columns + if len(columns) == 0 { + values := make([]dbquery.Row, len(rows)) + for i, row := range rows { + values[i] = row.Values + } + if len(values) > 0 { + r.columns = dbquery.InferSampleColumns(values) + } + columns = r.columns + } + return publisher.EmitFrame(ctx, sampleframe.Frame{Source: r.label(), StartedAt: start, CompletedAt: end, Observed: len(rows), Limit: r.limit, Summary: map[string]any{"columns": columns, "order": r.profile.Order}}, kept) + } + for _, row := range kept { + if err := records.Emit(ctx, row); err != nil { + return err + } + } + return nil +} diff --git a/pkg/tracing/query/table.go b/pkg/tracing/query/table.go new file mode 100644 index 0000000..53b14a0 --- /dev/null +++ b/pkg/tracing/query/table.go @@ -0,0 +1,12 @@ +package query + +import ( + "github.com/flanksource/clicky/api" + dbquery "github.com/flanksource/commons-db/query" +) + +func (Row) Columns() []api.ColumnDef { + return dbquery.TableColumnsFor[Row]("profile", "provider", "time", "row") +} +func (r Row) Row() map[string]any { return dbquery.TableRowFor(r) } +func (r Row) RowDetail() api.Textable { return r.PrettyFull() } diff --git a/pkg/tracing/ranking.go b/pkg/tracing/ranking.go new file mode 100644 index 0000000..b531aba --- /dev/null +++ b/pkg/tracing/ranking.go @@ -0,0 +1,25 @@ +package tracing + +import ( + "cmp" + "slices" +) + +// RankedValues returns the largest weights within a single sample unit. +func RankedValues(values map[string]int64) []map[string]any { + names := make([]string, 0, len(values)) + for name := range values { + names = append(names, name) + } + slices.SortFunc(names, func(a, b string) int { + if order := cmp.Compare(values[b], values[a]); order != 0 { + return order + } + return cmp.Compare(a, b) + }) + rows := make([]map[string]any, 0, min(len(names), 10)) + for _, name := range names[:min(len(names), 10)] { + rows = append(rows, map[string]any{"function": name, "value": values[name]}) + } + return rows +} diff --git a/pkg/tracing/results.go b/pkg/tracing/results.go new file mode 100644 index 0000000..440e80d --- /dev/null +++ b/pkg/tracing/results.go @@ -0,0 +1,101 @@ +package tracing + +import ( + "context" + "fmt" + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/commons-db/recordstore/recordresults" +) + +type captureBackendKey struct{} + +// CaptureBackend is the atomic writer of a capture with named result tables. +func CaptureBackend(ctx context.Context) (recordstore.BatchAppender, error) { + backend, ok := ctx.Value(captureBackendKey{}).(recordstore.BatchAppender) + if !ok { + return nil, fmt.Errorf("capture backend does not support atomic batches") + } + return backend, nil +} + +// ResultTable is a named, read-only result of a capture. It is advertised with +// its owning kind, and registered with the same query/export infrastructure. +type ResultTable struct { + Name string `json:"name"` + plugin TracePlugin +} + +type resultParams struct{} + +func (resultParams) Validate() error { + return fmt.Errorf("result tables cannot be started independently") +} + +type resultReader[R any] struct{ result recordresults.ResultType[R] } + +func (r resultReader[R]) Params() resultParams { return resultParams{} } +func (r resultReader[R]) Schema() recordresults.ResultType[R] { return r.result } +func (resultReader[R]) Handle(dbcontext.Context, resultParams, Emitter[R], Records[R]) error { + return fmt.Errorf("result tables cannot be started independently") +} + +func NewResultTable[R any](name string, result recordresults.ResultType[R]) ResultTable { + return ResultTable{Name: name, plugin: NewHandler[resultParams, R](resultReader[R]{result: result}, Capabilities{Live: true}, "self")} +} + +func (h *Handler[P, R]) WithResults(tables ...ResultTable) *Handler[P, R] { + h.tables = append(h.tables, tables...) + return h +} +func (h *Handler[P, R]) ResultTables() []ResultTable { return h.tables } + +// ResultOf identifies the capture and named result behind a storage kind. +func (k *Kinds) ResultOf(storage string) (kind, result string) { + k.mu.RLock() + defer k.mu.RUnlock() + for name, plugin := range k.plugins { + for _, table := range plugin.ResultTables() { + if storage == name+"_"+table.Name { + return name, table.Name + } + } + } + return storage, "" +} + +// CaptureRefs returns the primary and all named results of a capture. +func (k *Kinds) CaptureRefs(ref TraceRef) []TraceRef { + k.mu.RLock() + defer k.mu.RUnlock() + refs := []TraceRef{ref} + if plugin := k.plugins[ref.Kind]; plugin != nil { + for _, table := range plugin.ResultTables() { + refs = append(refs, TraceRef{Kind: ref.Kind + "_" + table.Name, Stream: ref.Stream + "-" + table.Name}) + } + } + return refs +} + +// Schemas includes each kind's primary table and its named result tables. +func (k *Kinds) Schemas() ([]recordstore.KindSchema, error) { + k.mu.RLock() + defer k.mu.RUnlock() + schemas := []recordstore.KindSchema{} + for _, name := range k.names() { + plugin := k.plugins[name] + schema, err := plugin.Schema(name) + if err != nil { + return nil, err + } + schemas = append(schemas, schema) + for _, result := range plugin.ResultTables() { + schema, err := result.plugin.Schema(name + "_" + result.Name) + if err != nil { + return nil, err + } + schemas = append(schemas, schema) + } + } + return schemas, nil +} diff --git a/pkg/tracing/seam/blob_internal_test.go b/pkg/tracing/seam/blob_internal_test.go new file mode 100644 index 0000000..3aca105 --- /dev/null +++ b/pkg/tracing/seam/blob_internal_test.go @@ -0,0 +1,60 @@ +package seam + +import ( + "context" + "encoding/json" + "errors" + "time" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/tracing" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type frameOutput struct { + frames []sampleframe.Frame + rows [][]string +} + +func (*frameOutput) Emit(context.Context, string) error { + return errors.New("unexpected individual row") +} +func (*frameOutput) TryEmit(string) bool { return false } +func (*frameOutput) Report(any) {} +func (*frameOutput) Warn(string) {} +func (*frameOutput) Describe(string, ...query.SessionMetadata) {} +func (*frameOutput) Export(string, string) {} +func (o *frameOutput) EmitFrame(_ context.Context, frame sampleframe.Frame, rows []string) error { + o.frames = append(o.frames, frame) + o.rows = append(o.rows, rows) + return nil +} + +var _ = Describe("decoded agent frames", func() { + It("preserves producer frame metadata including an empty snapshot", func() { + started := time.Date(2026, 11, 14, 10, 0, 0, 0, time.UTC) + frame := sampleframe.Frame{Source: "jvm-instance/http", Tick: 9, StartedAt: started, CompletedAt: started.Add(time.Millisecond), Observed: 0, Limit: 50, Summary: map[string]any{"complete": true}} + out := &frameOutput{} + s := AgentSeam[struct{}, string]{DecodeBlob: func(agentclient.Blob, map[string]string) (DecodedBlob[string], error) { + return DecodedBlob[string]{Frame: &frame, Rows: []string{}}, nil + }} + Expect(s.blob(context.Background(), agentclient.Blob{}, map[string]string{"process.pid": "1"}, out)).To(Succeed()) + Expect(out.frames).To(Equal([]sampleframe.Frame{frame})) + Expect(out.rows).To(Equal([][]string{{}})) + }) + It("delegates named-result blobs without requiring a row decoder", func() { + out := &frameOutput{} + var received json.RawMessage + s := AgentSeam[struct{}, string]{HandleBlob: func(_ context.Context, blob agentclient.Blob, resource map[string]string, emitter tracing.Emitter[string]) error { + Expect(resource).To(HaveKeyWithValue("source", "jvm")) + Expect(emitter).To(BeIdenticalTo(out)) + received = blob.Data + return nil + }} + Expect(s.blob(context.Background(), agentclient.Blob{Data: []byte("snapshot")}, map[string]string{"source": "jvm"}, out)).To(Succeed()) + Expect(string(received)).To(Equal("snapshot")) + }) +}) diff --git a/pkg/tracing/seam/completion_internal_test.go b/pkg/tracing/seam/completion_internal_test.go new file mode 100644 index 0000000..0234686 --- /dev/null +++ b/pkg/tracing/seam/completion_internal_test.go @@ -0,0 +1,56 @@ +package seam + +import ( + "context" + "encoding/json" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/tracing/target" + "github.com/flanksource/telltale/sdk/protocol" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type completionEmitter struct{ values []string } + +func (e *completionEmitter) Emit(_ context.Context, value string) error { + e.values = append(e.values, value) + return nil +} +func (e *completionEmitter) TryEmit(value string) bool { + e.values = append(e.values, value) + return true +} +func (*completionEmitter) Report(any) {} +func (*completionEmitter) Warn(string) {} +func (*completionEmitter) Describe(string, ...query.SessionMetadata) {} +func (*completionEmitter) Export(string, string) {} + +var _ = Describe("agent capture completion", func() { + DescribeTable("drains records and distinguishes completion from failure", func(end protocol.Ended, failure string) { + agent := &agentclient.Agent{Hello: protocol.Hello{AgentID: "fixture"}} + session := &agentclient.Session{Records: make(chan protocol.Records, 1), Ended: make(chan protocol.Ended, 1)} + session.Records <- protocol.Records{Items: []json.RawMessage{json.RawMessage(`"final event"`)}} + session.Ended <- end + state := &capture{sessions: map[*agentclient.Agent]*agentclient.Session{agent: session}, dropped: make(map[*agentclient.Agent]uint64), report: make(map[string]any)} + base, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + ctx := dbcontext.New().Wrap(base).WithValue(stateKey{}, state) + s := AgentSeam[target.Target, string]{Target: func(t target.Target) target.Target { return t }} + out := &completionEmitter{} + err := s.Handle(ctx, target.Target{Agent: "fixture"}, out) + if failure == "" { + Expect(err).NotTo(HaveOccurred()) + } else { + Expect(err).To(MatchError(ContainSubstring(failure))) + } + Expect(out.values).To(Equal([]string{"final event"})) + }, + Entry("event budget", protocol.Ended{Complete: true}, ""), + Entry("unexpected stop", protocol.Ended{}, "ended unexpectedly"), + Entry("failed cleanup", protocol.Ended{Complete: true, Error: "restore failed"}, "restore failed"), + ) +}) diff --git a/pkg/tracing/seam/resolve_spec_test.go b/pkg/tracing/seam/resolve_spec_test.go new file mode 100644 index 0000000..4e6ceae --- /dev/null +++ b/pkg/tracing/seam/resolve_spec_test.go @@ -0,0 +1,49 @@ +package seam_test + +import ( + "context" + "errors" + "path/filepath" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/tracing/seam" + "github.com/flanksource/telltale/pkg/tracing/target" + "github.com/flanksource/telltale/sdk/protocol" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("agent seam resolvers", func() { + const remote = "remote/orders-0" + + // registryResolving answers ids under remote/ by adding an agent with + // probe, or by failing with err; resolved counts the answers. + registryResolving := func(probe string, err error, resolved *int) *agentclient.Registry { + registry := agentclient.NewRegistry() + registry.SetResolver("remote/", func(_ context.Context, registry *agentclient.Registry, id string) error { + *resolved++ + if err != nil { + return err + } + return registry.Add(&agentclient.Agent{Hello: protocol.Hello{AgentID: id, Probes: []protocol.ProbeSpec{{Name: probe}}}}) + }) + return registry + } + + It("connects a resolver's agent once, without local discovery", func() { + var resolved int + s := seam.AgentSeam[struct{}, string]{Probe: "jvm_trace", Dir: filepath.Join(GinkgoT().TempDir(), "absent"), Registry: registryResolving("jvm_trace", nil, &resolved)} + for range 2 { + Expect(s.Available("jvm_trace", target.Target{Agent: remote})).To(BeTrue()) + } + Expect(resolved).To(Equal(1)) + Expect(s.Available("jvm_jfr", target.Target{Agent: remote})).To(BeFalse()) + }) + + It("fails the capture when its resolver cannot connect the agent", func() { + var resolved int + s := seam.AgentSeam[struct{}, string]{Probe: "jvm_trace", Registry: registryResolving("jvm_trace", errors.New("port-forward refused"), &resolved)} + _, err := s.Available("jvm_trace", target.Target{Agent: remote}) + Expect(err).To(MatchError(ContainSubstring(`connect agent "remote/orders-0": port-forward refused`))) + }) +}) diff --git a/pkg/tracing/seam/seam.go b/pkg/tracing/seam/seam.go index b717f6f..c1ced24 100644 --- a/pkg/tracing/seam/seam.go +++ b/pkg/tracing/seam/seam.go @@ -12,12 +12,19 @@ import ( "time" dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/recordstore/sampleframe" "github.com/flanksource/telltale/pkg/agentclient" "github.com/flanksource/telltale/pkg/tracing" "github.com/flanksource/telltale/pkg/tracing/target" telltale "github.com/flanksource/telltale/sdk" + "github.com/flanksource/telltale/sdk/protocol" ) +type DecodedBlob[R any] struct { + Rows []R + Frame *sampleframe.Frame +} + type AgentSeam[P, R any] struct { Probe string Dir string @@ -25,8 +32,13 @@ type AgentSeam[P, R any] struct { Target func(P) target.Target ArmParams func(P) (json.RawMessage, error) Mutates func(P) bool + // Compatible refuses params the agent's advertised probe cannot honour, + // before anything is armed; nil accepts every agent advertising the probe. + Compatible func(P, protocol.ProbeSpec) error DecodeRecord func(json.RawMessage) (R, error) - DecodeBlob func(agentclient.Blob, map[string]string) ([]R, error) + DecodeBlob func(agentclient.Blob, map[string]string) (DecodedBlob[R], error) + HandleBlob func(context.Context, agentclient.Blob, map[string]string, tracing.Emitter[R]) error + CommitBlob func(agentclient.Blob) error } type stateKey struct{} @@ -65,6 +77,17 @@ func (s AgentSeam[P, R]) Available(probe string, selection target.Target) (bool, if agent != nil && agent.HasProbe(probe) { return true, nil } + // A resolver connects a remote agent, such as a deployed target over + // a port-forward, which takes longer than finding a local socket. + resolveCtx, cancelResolve := context.WithTimeout(context.Background(), time.Minute) + defer cancelResolve() + if claimed, err := registry.Resolve(resolveCtx, selection.Agent); claimed { + if err != nil { + return false, err + } + agent := registry.ByID(selection.Agent) + return agent != nil && agent.HasProbe(probe), nil + } } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -157,7 +180,25 @@ func (s AgentSeam[P, R]) Prepare(ctx dbcontext.Context, params P, out tracing.Em return ctx.WithValue(stateKey{}, state), func() error { return s.release(state) }, nil } +func (s AgentSeam[P, R]) compatible(agent *agentclient.Agent, params P) error { + if s.Compatible == nil { + return nil + } + for _, spec := range agent.Hello.Probes { + if spec.Name == s.Probe { + if err := s.Compatible(params, spec); err != nil { + return fmt.Errorf("arm %s on %s: %w", s.Probe, agent.Hello.AgentID, err) + } + return nil + } + } + return fmt.Errorf("agent %s does not advertise probe %s", agent.Hello.AgentID, s.Probe) +} + func (s AgentSeam[P, R]) arm(ctx context.Context, state *capture, agent *agentclient.Agent, params P) error { + if err := s.compatible(agent, params); err != nil { + return err + } data := json.RawMessage(`{}`) var err error if s.ArmParams != nil { @@ -221,7 +262,7 @@ func (s AgentSeam[P, R]) Checkpoint(ctx context.Context, raw json.RawMessage, ou } } for _, blob := range result.Blobs { - if err := s.blob(ctx, blob, agent.Hello.Resource, out); err != nil { + if err := s.blob(ctx, blob, sampleResource(agent), out); err != nil { return err } } @@ -246,7 +287,7 @@ func (s AgentSeam[P, R]) Window(ctx context.Context, params P, raw json.RawMessa select { case blob := <-session.Blobs: blob.CheckpointParams = raw - err = s.blob(ctx, blob, agent.Hello.Resource, out) + err = s.blob(ctx, blob, sampleResource(agent), out) case ended := <-session.Ended: err = fmt.Errorf("agent profile ended: %s", ended.Error) case <-ctx.Done(): @@ -263,14 +304,26 @@ func (s AgentSeam[P, R]) Window(ctx context.Context, params P, raw json.RawMessa } func (s AgentSeam[P, R]) blob(ctx context.Context, blob agentclient.Blob, resource map[string]string, out tracing.Emitter[R]) error { + if s.HandleBlob != nil { + return s.HandleBlob(ctx, blob, resource, out) + } if s.DecodeBlob == nil { return fmt.Errorf("probe %s has no blob decoder", s.Probe) } - rows, err := s.DecodeBlob(blob, resource) + decoded, err := s.DecodeBlob(blob, resource) if err != nil { return err } - for _, row := range rows { + if decoded.Frame != nil { + if err := tracing.PublishFrame(ctx, out, *decoded.Frame, decoded.Rows); err != nil { + return err + } + if s.CommitBlob != nil { + return s.CommitBlob(blob) + } + return nil + } + for _, row := range decoded.Rows { if err := out.Emit(ctx, row); err != nil { return err } @@ -278,6 +331,15 @@ func (s AgentSeam[P, R]) blob(ctx context.Context, blob agentclient.Blob, resour return nil } +func sampleResource(agent *agentclient.Agent) map[string]string { + resource := maps.Clone(agent.Hello.Resource) + if resource == nil { + resource = map[string]string{} + } + resource["telltale.agent.id"] = agent.Hello.AgentID + "/" + agent.Hello.Started.UTC().Format(time.RFC3339Nano) + return resource +} + func (s AgentSeam[P, R]) emit(ctx context.Context, raw json.RawMessage, resource map[string]string, out tracing.Emitter[R]) error { var row R var err error @@ -374,6 +436,16 @@ func (s AgentSeam[P, R]) Handle(ctx dbcontext.Context, params P, out tracing.Emi return fmt.Errorf("discover late agents: %w", err) } case failure := <-errs: + if failure.err == nil { + state.mu.Lock() + delete(state.sessions, failure.agent) + remaining := len(state.sessions) + state.mu.Unlock() + if remaining == 0 && len(s.Target(params).Selector) == 0 { + return nil + } + continue + } if !failure.departed || len(s.Target(params).Selector) == 0 { return failure.err } @@ -437,7 +509,7 @@ func (s AgentSeam[P, R]) read(ctx context.Context, state *capture, agent *agentc } state.reportDropped(agent, batch.Dropped, out.Report) case blob := <-session.Blobs: - if err := s.blob(ctx, blob, agent.Hello.Resource, out); err != nil { + if err := s.blob(ctx, blob, sampleResource(agent), out); err != nil { fail(err, false) return } @@ -452,7 +524,7 @@ func (s AgentSeam[P, R]) read(ctx context.Context, state *capture, agent *agentc for { select { case blob := <-session.Blobs: - if err := s.blob(ctx, blob, agent.Hello.Resource, out); err != nil { + if err := s.blob(ctx, blob, sampleResource(agent), out); err != nil { fail(err, false) return } @@ -464,6 +536,13 @@ func (s AgentSeam[P, R]) read(ctx context.Context, state *capture, agent *agentc } } state.reportDropped(agent, batch.Dropped, out.Report) + case report := <-session.Reports: + // The agent reports after the blob it describes; one sent with + // the final blob is still queued when the capture ends. + if err := state.mergeReport(report.Summary, out.Report); err != nil { + fail(err, false) + return + } default: goto drained } @@ -472,6 +551,8 @@ func (s AgentSeam[P, R]) read(ctx context.Context, state *capture, agent *agentc state.reportDropped(agent, end.Dropped, out.Report) if end.Error != "" { fail(fmt.Errorf("agent %s ended: %s", agent.Hello.AgentID, end.Error), true) + } else if end.Complete { + fail(nil, true) } else { fail(fmt.Errorf("agent %s ended unexpectedly", agent.Hello.AgentID), true) } diff --git a/pkg/tracing/seam/seam_test.go b/pkg/tracing/seam/seam_test.go index 978a0cc..67b5712 100644 --- a/pkg/tracing/seam/seam_test.go +++ b/pkg/tracing/seam/seam_test.go @@ -42,6 +42,16 @@ func (refusingProbe) Arm(context.Context, json.RawMessage, telltale.Sink) (func( return nil, errors.New("mutation not allowed by agent") } +type versionedProbe struct{ arms atomic.Int32 } + +func (*versionedProbe) Spec() protocol.ProbeSpec { + return protocol.ProbeSpec{Name: "versioned-arm", Version: 9} +} +func (p *versionedProbe) Arm(context.Context, json.RawMessage, telltale.Sink) (func() error, error) { + p.arms.Add(1) + return func() error { return nil }, nil +} + func testRegistry() *agentclient.Registry { registry := agentclient.NewRegistry() registry.Dir = GinkgoT().TempDir() @@ -63,6 +73,25 @@ func (*emitter) Describe(string, ...query.SessionMetadata) {} func (*emitter) Export(string, string) {} var _ = Describe("agent seam", func() { + It("refuses params the agent's advertised probe version cannot honour, before arming", func() { + probe := &versionedProbe{} + telltale.RegisterProbe(probe) + registry := testRegistry() + server := telltale.Loopback() + a, err := agentclient.Connect(context.Background(), server.Connect(), "") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _ = a.Close(); _ = server.Close() }) + Expect(registry.Add(a)).To(Succeed()) + var seen protocol.ProbeSpec + s := seam.AgentSeam[target.Target, string]{Probe: "versioned-arm", Registry: registry, Target: func(t target.Target) target.Target { return t }, Compatible: func(_ target.Target, spec protocol.ProbeSpec) error { + seen = spec + return errors.New("needs v10") + }} + _, _, err = s.Prepare(dbcontext.New(), target.Target{Seam: target.Agent, Agent: a.Hello.AgentID}, &emitter{}) + Expect(err).To(MatchError(ContainSubstring("needs v10"))) + Expect(seen.Version).To(Equal(9)) + Expect(probe.arms.Load()).To(BeZero()) + }) It("warns when one selector agent refuses to arm but another succeeds", func() { telltale.RegisterProbe(&partialProbe{}) registry := testRegistry() @@ -119,7 +148,9 @@ var _ = Describe("agent seam", func() { DeferCleanup(func() { _ = a.Close(); _ = b.Close(); _ = first.Close(); _ = second.Close() }) Expect(registry.Add(a)).To(Succeed()) Expect(registry.Add(b)).To(Succeed()) - s := seam.AgentSeam[target.Target, string]{Probe: "stacks", Registry: registry, Target: func(t target.Target) target.Target { return t }, DecodeBlob: func(_ agentclient.Blob, _ map[string]string) ([]string, error) { return []string{"stack"}, nil }} + s := seam.AgentSeam[target.Target, string]{Probe: "stacks", Registry: registry, Target: func(t target.Target) target.Target { return t }, DecodeBlob: func(_ agentclient.Blob, _ map[string]string) (seam.DecodedBlob[string], error) { + return seam.DecodedBlob[string]{Rows: []string{"stack"}}, nil + }} params := target.Target{Seam: target.Agent, Selector: map[string]string{"process.pid": strconv.Itoa(os.Getpid())}} e := &emitter{warnings: make(chan string, 2)} base, cancel := context.WithCancel(context.Background()) @@ -161,7 +192,9 @@ var _ = Describe("agent seam", func() { b.Hello.AgentID = "late-agent" DeferCleanup(func() { Expect(a.Close()).To(Succeed()); Expect(b.Close()).To(Succeed()) }) Expect(registry.Add(a)).To(Succeed()) - s := seam.AgentSeam[target.Target, string]{Probe: "stacks", Registry: registry, Target: func(t target.Target) target.Target { return t }, DecodeBlob: func(_ agentclient.Blob, _ map[string]string) ([]string, error) { return []string{"stack"}, nil }} + s := seam.AgentSeam[target.Target, string]{Probe: "stacks", Registry: registry, Target: func(t target.Target) target.Target { return t }, DecodeBlob: func(_ agentclient.Blob, _ map[string]string) (seam.DecodedBlob[string], error) { + return seam.DecodedBlob[string]{Rows: []string{"stack"}}, nil + }} params := target.Target{Seam: target.Agent, Selector: map[string]string{"process.pid": strconv.Itoa(os.Getpid())}} e := &emitter{} base, cancel := context.WithCancel(context.Background()) @@ -204,7 +237,9 @@ var _ = Describe("agent seam", func() { }) It("arms the self probe and decodes a checkpoint blob", func() { e := &emitter{} - s := seam.AgentSeam[target.Target, string]{Probe: "stacks", Target: func(t target.Target) target.Target { return t }, DecodeBlob: func(_ agentclient.Blob, _ map[string]string) ([]string, error) { return []string{"stack"}, nil }} + s := seam.AgentSeam[target.Target, string]{Probe: "stacks", Target: func(t target.Target) target.Target { return t }, DecodeBlob: func(_ agentclient.Blob, _ map[string]string) (seam.DecodedBlob[string], error) { + return seam.DecodedBlob[string]{Rows: []string{"stack"}}, nil + }} ctx, release, err := s.Prepare(dbcontext.New(), target.Target{Seam: target.Self}, e) Expect(err).NotTo(HaveOccurred()) DeferCleanup(func() { Expect(release()).To(Succeed()) }) diff --git a/pkg/tracing/source.go b/pkg/tracing/source.go index c5223e1..29dfcba 100644 --- a/pkg/tracing/source.go +++ b/pkg/tracing/source.go @@ -59,16 +59,18 @@ type source struct { // page budget for one sample. capacity never exceeds it. drainLimit int - mu sync.Mutex - space chan struct{} - rows []recordstore.Row - sizes []int - bytes int - base int64 - running bool - frozen bool - summary Summary - err error + mu sync.Mutex + space chan struct{} + rows []recordstore.Row + sizes []int + bytes int + base int64 + sampleEnd int64 + sampling bool + running bool + frozen bool + summary Summary + err error // warnings are the kind's, which the session ends with; handle and // metadata are what it described its session with. @@ -310,6 +312,10 @@ func (s *source) Read(_ context.Context, cursor probe.Cursor) (probe.Batch, erro return probe.Batch{}, fmt.Errorf("trace capture %s: cursor %d is before the committed %d", s.generation, cursor.Next, s.base) } write := s.base + int64(len(s.rows)) + if !s.sampling { + s.sampleEnd, s.sampling = write, true + } + write = s.sampleEnd next := min(write, cursor.Next+pageRows) return probe.Batch{ Generation: s.generation, Next: next, Write: write, More: next < write, Active: s.running, @@ -346,6 +352,9 @@ func (s *source) distinct(page []recordstore.Row) []recordstore.Row { func (s *source) Commit(_ context.Context, batch probe.Batch) error { s.mu.Lock() defer s.mu.Unlock() + if !batch.More && batch.Next == s.sampleEnd { + s.sampling = false + } if batch.Next <= s.base { return nil } @@ -368,7 +377,13 @@ func (s *source) Freeze(ctx context.Context) error { s.frozen = true s.signalSpaceLocked() s.mu.Unlock() - return s.wait(ctx) + if err := s.wait(ctx); err != nil { + return err + } + s.mu.Lock() + s.sampling = false + s.mu.Unlock() + return nil } func (s *source) Finalize(context.Context) (probe.Final, error) { diff --git a/pkg/tracing/source_paging_internal_test.go b/pkg/tracing/source_paging_internal_test.go new file mode 100644 index 0000000..087385a --- /dev/null +++ b/pkg/tracing/source_paging_internal_test.go @@ -0,0 +1,77 @@ +package tracing + +import ( + "context" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/recordstore/probe" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("source snapshot paging", func() { + It("samples new rows after committing an initially empty buffer", func() { + src := newSource("empty", 2, defaultBufferBytes, "", false) + empty, err := src.Read(context.Background(), probe.Cursor{}) + Expect(err).NotTo(HaveOccurred()) + Expect(src.Commit(context.Background(), empty)).To(Succeed()) + Expect(src.tryPush(map[string]any{"phase": "after"})).To(BeTrue()) + next, err := src.Read(context.Background(), probe.Cursor{}) + Expect(err).NotTo(HaveOccurred()) + Expect(next.Rows).To(ConsistOf(map[string]any{"phase": "after"})) + }) + + It("finishes a sample at its initial write cursor while a busy producer keeps emitting", func() { + src := newSource("busy", 3*pageRows, defaultBufferBytes, "", false) + for range pageRows + 1 { + Expect(src.tryPush(map[string]any{"phase": "before"})).To(BeTrue()) + } + first, err := src.Read(context.Background(), probe.Cursor{}) + Expect(err).NotTo(HaveOccurred()) + Expect(first.More).To(BeTrue()) + Expect(src.Commit(context.Background(), first)).To(Succeed()) + for range pageRows { + Expect(src.tryPush(map[string]any{"phase": "after"})).To(BeTrue()) + } + last, err := src.Read(context.Background(), probe.Cursor{Next: first.Next}) + Expect(err).NotTo(HaveOccurred()) + Expect(last.Rows).To(HaveLen(1)) + Expect(last.More).To(BeFalse()) + Expect(src.Commit(context.Background(), last)).To(Succeed()) + next, err := src.Read(context.Background(), probe.Cursor{Next: last.Next}) + Expect(err).NotTo(HaveOccurred()) + Expect(next.Rows).To(HaveLen(pageRows)) + }) + + It("returns the same pending page when emission continues before a retry", func() { + src := newSource("retry", 3, defaultBufferBytes, "", false) + Expect(src.tryPush(map[string]any{"phase": "before"})).To(BeTrue()) + first, err := src.Read(context.Background(), probe.Cursor{}) + Expect(err).NotTo(HaveOccurred()) + Expect(src.tryPush(map[string]any{"phase": "after"})).To(BeTrue()) + retry, err := src.Read(context.Background(), probe.Cursor{}) + Expect(err).NotTo(HaveOccurred()) + Expect(retry.Next).To(Equal(first.Next)) + Expect(retry.Rows).To(Equal(first.Rows)) + }) + + It("includes records emitted during shutdown even when an earlier sample was incomplete", func() { + src := newSource("final", 3*pageRows, defaultBufferBytes, "", false) + for range pageRows + 1 { + Expect(src.tryPush(map[string]any{"phase": "before"})).To(BeTrue()) + } + src.prepare(dbcontext.New(), func(ctx dbcontext.Context) error { + <-ctx.Done() + return src.push(context.Background(), map[string]any{"phase": "final"}) + }, nil) + src.start() + first, err := src.Read(context.Background(), probe.Cursor{}) + Expect(err).NotTo(HaveOccurred()) + Expect(src.Commit(context.Background(), first)).To(Succeed()) + Expect(src.Freeze(context.Background())).To(Succeed()) + last, err := src.Read(context.Background(), probe.Cursor{Next: first.Next}) + Expect(err).NotTo(HaveOccurred()) + Expect(last.Rows).To(ConsistOf(map[string]any{"phase": "before"}, map[string]any{"phase": "final"})) + Expect(last.Active).To(BeFalse()) + }) +}) diff --git a/pkg/tracing/spans/agent.go b/pkg/tracing/spans/agent.go new file mode 100644 index 0000000..f25d145 --- /dev/null +++ b/pkg/tracing/spans/agent.go @@ -0,0 +1,42 @@ +package spans + +import ( + "encoding/json" + "fmt" + "maps" + + "github.com/flanksource/telltale/pkg/agentclient" + otlpspans "github.com/flanksource/telltale/pkg/otlp/spans" + "github.com/flanksource/telltale/pkg/tracing/seam" + sdk "github.com/flanksource/telltale/sdk" +) + +func decodeBlob(blob agentclient.Blob, resource map[string]string) (seam.DecodedBlob[sdk.Span], error) { + if blob.Info.ContentType != "application/json" { + return seam.DecodedBlob[sdk.Span]{}, fmt.Errorf("span mirror requires application/json, got %q", blob.Info.ContentType) + } + var rows []struct { + Native string `json:"native"` + } + if err := json.Unmarshal(blob.Data, &rows); err != nil { + return seam.DecodedBlob[sdk.Span]{}, fmt.Errorf("decode span mirror: %w", err) + } + if rows == nil { + return seam.DecodedBlob[sdk.Span]{}, fmt.Errorf("span mirror must be an array") + } + decoded := seam.DecodedBlob[sdk.Span]{Rows: make([]sdk.Span, 0, len(rows))} + for _, row := range rows { + span, err := otlpspans.DecodeNative(row.Native) + if err != nil { + return seam.DecodedBlob[sdk.Span]{}, err + } + merged := make(map[string]any, len(resource)+len(span.Resource)) + for key, value := range resource { + merged[key] = value + } + maps.Copy(merged, span.Resource) + span.Resource = merged + decoded.Rows = append(decoded.Rows, span) + } + return decoded, nil +} diff --git a/pkg/tracing/spans/agent_internal_test.go b/pkg/tracing/spans/agent_internal_test.go new file mode 100644 index 0000000..32f6fa0 --- /dev/null +++ b/pkg/tracing/spans/agent_internal_test.go @@ -0,0 +1,56 @@ +package spans + +import ( + "encoding/base64" + "encoding/json" + "time" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/sdk/protocol" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/ptrace/ptraceotlp" +) + +var _ = Describe("Java span mirror blobs", func() { + It("rebuilds authoritative fields and exact attributes from redacted native OTLP", func() { + request := ptraceotlp.NewExportRequest() + resource := request.Traces().ResourceSpans().AppendEmpty() + resource.Resource().Attributes().PutStr("service.name", "original-service") + scope := resource.ScopeSpans().AppendEmpty() + scope.Scope().SetName("existing-sdk") + span := scope.Spans().AppendEmpty() + span.SetTraceID(pcommon.TraceID{1}) + span.SetSpanID(pcommon.SpanID{2}) + span.SetName("original-operation") + span.SetStartTimestamp(pcommon.NewTimestampFromTime(time.Date(2026, 11, 14, 10, 0, 0, 0, time.UTC))) + span.SetEndTimestamp(span.StartTimestamp() + 100) + span.Attributes().PutInt("large.integer", 9007199254740993) + span.Attributes().PutStr("password", "private-native-value") + span.Events().AppendEmpty().SetName("original-event") + native, err := request.MarshalProto() + Expect(err).NotTo(HaveOccurred()) + data, err := json.Marshal([]map[string]any{{"operation": "untrusted-json", "attributes": map[string]any{"large.integer": 1}, "native": base64.StdEncoding.EncodeToString(native)}}) + Expect(err).NotTo(HaveOccurred()) + decoded, err := decodeBlob(agentclient.Blob{Info: protocol.Blob{ContentType: "application/json"}, Data: data}, map[string]string{"process.pid": "42"}) + Expect(err).NotTo(HaveOccurred()) + Expect(decoded.Rows).To(HaveLen(1)) + Expect(decoded.Rows[0].Operation).To(Equal("original-operation")) + Expect(decoded.Rows[0].Resource).To(HaveKeyWithValue("process.pid", "42")) + Expect(decoded.Rows[0].Attributes).To(HaveKeyWithValue("large.integer", int64(9007199254740993))) + Expect(decoded.Rows[0].Digest).NotTo(BeEmpty()) + stored, err := base64.StdEncoding.DecodeString(decoded.Rows[0].Native) + Expect(err).NotTo(HaveOccurred()) + restored := ptraceotlp.NewExportRequest() + Expect(restored.UnmarshalProto(stored)).To(Succeed()) + encoded, err := restored.MarshalJSON() + Expect(err).NotTo(HaveOccurred()) + Expect(string(encoded)).NotTo(ContainSubstring("private-native-value")) + Expect(restored.Traces().ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Events().Len()).To(Equal(1)) + }) + It("rejects a mirror record without native data", func() { + _, err := decodeBlob(agentclient.Blob{Info: protocol.Blob{ContentType: "application/json"}, Data: []byte(`[{"operation":"missing-native"}]`)}, nil) + Expect(err).To(MatchError(ContainSubstring("native"))) + }) +}) diff --git a/pkg/tracing/spans/pretty.go b/pkg/tracing/spans/pretty.go new file mode 100644 index 0000000..fed50ea --- /dev/null +++ b/pkg/tracing/spans/pretty.go @@ -0,0 +1,27 @@ +package spans + +import ( + "time" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" + "github.com/flanksource/telltale/pkg/tracing" + telltale "github.com/flanksource/telltale/sdk" +) + +type spanPresentation struct{ telltale.Span } + +func (r spanPresentation) Pretty() api.Text { + return clicky.Text(r.Service, "text-blue-500 font-bold").Space().Append(r.Operation, "font-bold"). + Space().Append(r.Status, tracing.SeverityStyle(r.Status)).Space(). + Add(api.Human(time.Duration(r.DurationMs*float64(time.Millisecond)), "text-muted")).Space().Append(r.TraceID, "text-muted") +} + +func (r spanPresentation) PrettyFull() api.Textable { + return r.Pretty().NewLine().Add(clicky.Map(r.Attributes)).NewLine().Add(clicky.Map(r.Resource)) +} + +func (capture) Present(record telltale.Span) tracing.RecordPresentation { + r := spanPresentation{record} + return tracing.RecordPresentation{Summary: r.Pretty(), Detail: r.PrettyFull()} +} diff --git a/pkg/tracing/spans/spans.go b/pkg/tracing/spans/spans.go index e3ecc8a..df6711a 100644 --- a/pkg/tracing/spans/spans.go +++ b/pkg/tracing/spans/spans.go @@ -123,7 +123,7 @@ func (n normalizing) TryEmit(span telltale.Span) bool { } func NewKind(registry *agentclient.Registry) *tracing.Handler[Params, telltale.Span] { - agent := seam.AgentSeam[Params, telltale.Span]{Probe: Kind, Registry: registry, Target: func(p Params) target.Target { return p.Target }, ArmParams: func(p Params) (json.RawMessage, error) { return json.Marshal(p) }} + agent := seam.AgentSeam[Params, telltale.Span]{Probe: Kind, Registry: registry, Target: func(p Params) target.Target { return p.Target }, ArmParams: func(p Params) (json.RawMessage, error) { return json.Marshal(p) }, DecodeBlob: decodeBlob} return tracing.NewHandler[Params, telltale.Span](capture{agent}, tracing.Capabilities{Live: true, Historical: true, Follow: true}, target.Self, target.Agent, target.Host). WithSecretMasking("key") } diff --git a/pkg/tracing/spans/summary.go b/pkg/tracing/spans/summary.go new file mode 100644 index 0000000..6164915 --- /dev/null +++ b/pkg/tracing/spans/summary.go @@ -0,0 +1,18 @@ +package spans + +import ( + telltale "github.com/flanksource/telltale/sdk" + "iter" +) + +func (capture) Summarize(rows iter.Seq[telltale.Span]) (map[string]any, error) { + statuses := map[string]int{} + traces := map[string]bool{} + duration := 0.0 + for row := range rows { + statuses[row.Status]++ + traces[row.TraceID] = true + duration += row.DurationMs + } + return map[string]any{"traces": len(traces), "statuses": statuses, "durationMs": duration}, nil +} diff --git a/pkg/tracing/sql/agent.go b/pkg/tracing/sql/agent.go new file mode 100644 index 0000000..28fb8e8 --- /dev/null +++ b/pkg/tracing/sql/agent.go @@ -0,0 +1,86 @@ +package sql + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/flanksource/commons-db/connection" + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/tracing/seam" +) + +func decodeAgentStatement(raw json.RawMessage) (StatementRow, error) { + var record struct { + connection.Statement + Observation + DurationNS *int64 `json:"durationNs"` + Rows *int64 `json:"rows"` + } + if err := json.Unmarshal(raw, &record); err != nil { + return StatementRow{}, err + } + if record.API != "" { + if record.API != "jdbc" && record.API != "r2dbc" { + return StatementRow{}, fmt.Errorf("unsupported SQL API %q", record.API) + } + if record.OperationID == "" || record.StartedAt.IsZero() || record.DurationNS == nil || *record.DurationNS < 0 { + return StatementRow{}, fmt.Errorf("JVM SQL requires operationId, startedAt and non-negative durationNs") + } + if strings.TrimSpace(record.SQL) == "" && record.SQLUnavailableReason == "" { + return StatementRow{}, fmt.Errorf("JVM SQL requires SQL text or sqlUnavailableReason") + } + if record.SQLUnavailableReason != "" && record.SQL != "" { + return StatementRow{}, fmt.Errorf("unavailable JVM SQL must not contain invented SQL text") + } + record.Duration = time.Duration(*record.DurationNS) + record.Statement.Rows = -1 + } + if record.Rows != nil { + record.Statement.Rows = *record.Rows + } + row := FromStatement(record.Statement) + row.Observation = record.Observation + if row.SQLUnavailableReason != "" { + row.RepeatKey = "" + } + return row, nil +} + +func agentArmParams(p Params) (json.RawMessage, error) { + minimum, err := p.minimum() + if err != nil { + return nil, err + } + return json.Marshal(struct { + Params + MinDurationNS int64 `json:"minDurationNs"` + }{p, int64(minimum)}) +} + +func decodeAgentBlob(blob agentclient.Blob, resource map[string]string) (seam.DecodedBlob[StatementRow], error) { + if blob.Info.ContentType != "application/json" { + return seam.DecodedBlob[StatementRow]{}, fmt.Errorf("SQL observations require application/json, received %q", blob.Info.ContentType) + } + var raw []json.RawMessage + if err := json.Unmarshal(blob.Data, &raw); err != nil { + return seam.DecodedBlob[StatementRow]{}, fmt.Errorf("decode SQL observation batch: %w", err) + } + if raw == nil { + return seam.DecodedBlob[StatementRow]{}, fmt.Errorf("SQL observation batch must be an array") + } + rows := make([]StatementRow, 0, len(raw)) + for index, item := range raw { + row, err := decodeAgentStatement(item) + if err != nil { + return seam.DecodedBlob[StatementRow]{}, fmt.Errorf("decode SQL observation %d: %w", index+1, err) + } + if row.API == "" { + return seam.DecodedBlob[StatementRow]{}, fmt.Errorf("SQL observation %d requires a JVM SQL API", index+1) + } + row.Resource = resource + rows = append(rows, row) + } + return seam.DecodedBlob[StatementRow]{Rows: rows}, nil +} diff --git a/pkg/tracing/sql/agent_internal_test.go b/pkg/tracing/sql/agent_internal_test.go new file mode 100644 index 0000000..e88c761 --- /dev/null +++ b/pkg/tracing/sql/agent_internal_test.go @@ -0,0 +1,120 @@ +package sql + +import ( + "encoding/json" + "time" + + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/sdk/protocol" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gstruct" +) + +var _ = Describe("JVM SQL records", func() { + It("passes parsed duration limits to the JVM including an explicit zero", func() { + for _, duration := range []string{"0s", "1ms"} { + params, err := agentArmParams(Params{Options: Options{Connections: []string{"*"}, MinDuration: duration}}) + Expect(err).NotTo(HaveOccurred()) + var values map[string]any + Expect(json.Unmarshal(params, &values)).To(Succeed()) + parsed, err := time.ParseDuration(duration) + Expect(err).NotTo(HaveOccurred()) + Expect(values).To(HaveKeyWithValue("minDurationNs", float64(parsed))) + } + }) + It("preserves observed JDBC metadata without inventing row counts", func() { + row, err := decodeAgentStatement(json.RawMessage(`{"startedAt":"2026-11-14T10:00:00Z","durationNs":2500000,"sql":"SELECT id FROM orders WHERE id = ?","driver":"jdbc","connection":"orders","operationId":"operation-1","traceId":"trace-1","spanId":"span-1","api":"jdbc","batchSize":2,"database":"orders","engine":"h2"}`)) + Expect(err).NotTo(HaveOccurred()) + Expect(row.StartedAt).To(Equal(time.Date(2026, 11, 14, 10, 0, 0, 0, time.UTC))) + Expect(row.DurationMs).To(Equal(2.5)) + Expect(row.Rows).To(Equal(int64(-1))) + Expect(row.RepeatKey).NotTo(BeEmpty()) + data, err := json.Marshal(row) + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).To(ContainSubstring(`"operationId":"operation-1"`)) + Expect(string(data)).To(ContainSubstring(`"batchSize":2`)) + Expect(string(data)).To(ContainSubstring(`"database":"orders"`)) + Expect(string(data)).To(ContainSubstring(`"engine":"h2"`)) + }) + + It("keeps missing prepared SQL explicit and out of repeated-query groups", func() { + row, err := decodeAgentStatement(json.RawMessage(`{"startedAt":"2026-11-14T10:00:00Z","durationNs":1000,"operationId":"old-prepared","api":"jdbc","sqlUnavailableReason":"prepared-before-instrumentation"}`)) + Expect(err).NotTo(HaveOccurred()) + Expect(row.SQL).To(BeEmpty()) + Expect(row.RepeatKey).To(BeEmpty()) + Expect(row.Rows).To(Equal(int64(-1))) + Expect(Summarize([]StatementRow{row, row}).Repeated).To(BeZero()) + }) + + It("preserves a capture-ended reactive execution as explicitly incomplete", func() { + row, err := decodeAgentStatement(json.RawMessage(`{"startedAt":"2026-11-14T10:00:00Z","durationNs":1000,"operationId":"unconsumed","api":"r2dbc","sql":"SELECT id FROM orders","phase":"awaiting-consumption","incomplete":true,"completion":"capture-ended"}`)) + Expect(err).NotTo(HaveOccurred()) + data, err := json.Marshal(row) + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).To(ContainSubstring(`"phase":"awaiting-consumption"`)) + Expect(string(data)).To(ContainSubstring(`"incomplete":true`)) + Expect(string(data)).To(ContainSubstring(`"completion":"capture-ended"`)) + }) + + DescribeTable("rejects malformed JVM observations", func(raw string) { + _, err := decodeAgentStatement(json.RawMessage(raw)) + Expect(err).To(HaveOccurred()) + }, + Entry("missing execution time", `{"api":"jdbc","operationId":"one","sql":"SELECT 1"}`), + Entry("negative duration", `{"api":"jdbc","operationId":"one","startedAt":"2026-11-14T10:00:00Z","durationNs":-1,"sql":"SELECT 1"}`), + Entry("missing SQL without a reason", `{"api":"jdbc","operationId":"one","startedAt":"2026-11-14T10:00:00Z","durationNs":1}`), + ) +}) + +var _ = Describe("JVM SQL checkpoint blobs", func() { + It("wires checkpoint and stop decoding through the SQL capture seam", func() { + capture, ok := NewCapture(nil).(statements) + Expect(ok).To(BeTrue()) + Expect(capture.agent.DecodeBlob).NotTo(BeNil()) + decoded, err := capture.agent.DecodeBlob(agentclient.Blob{Info: protocol.Blob{ContentType: "application/json"}, Data: []byte(`[]`)}, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(decoded.Rows).To(BeEmpty()) + }) + + It("decodes JDBC and reactive completion metadata with the attached JVM resource", func() { + blob := agentclient.Blob{Info: protocol.Blob{ContentType: "application/json"}, Data: []byte(`[ + {"startedAt":"2026-11-14T10:00:00Z","durationNs":2500000,"sql":"SELECT id FROM orders","operationId":"jdbc-query","api":"jdbc","rows":2}, + {"startedAt":"2026-11-14T10:00:01Z","durationNs":5000000,"sql":"SELECT id FROM orders","operationId":"reactive-query","api":"r2dbc","phase":"awaiting-consumption","incomplete":true,"completion":"capture-ended"} + ]`)} + resource := map[string]string{"service.name": "orders", "process.pid": "17"} + decoded, err := decodeAgentBlob(blob, resource) + Expect(err).NotTo(HaveOccurred()) + Expect(decoded.Rows).To(HaveExactElements( + gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Observation": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"OperationID": Equal("jdbc-query"), "API": Equal("jdbc")}), + "Rows": Equal(int64(2)), "DurationMs": Equal(2.5), "Resource": Equal(resource), + }), + gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Observation": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"OperationID": Equal("reactive-query"), "API": Equal("r2dbc"), "Incomplete": BeTrue(), "Completion": Equal("capture-ended")}), + "Rows": Equal(int64(-1)), "DurationMs": Equal(float64(5)), "Resource": Equal(resource), + }), + )) + }) + + It("accepts an explicitly empty checkpoint array", func() { + decoded, err := decodeAgentBlob(agentclient.Blob{Info: protocol.Blob{ContentType: "application/json"}, Data: []byte(`[]`)}, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(decoded.Rows).To(BeEmpty()) + }) + + DescribeTable("rejects invalid checkpoint payloads without returning partial rows", func(contentType, payload string) { + decoded, err := decodeAgentBlob(agentclient.Blob{Info: protocol.Blob{ContentType: contentType}, Data: []byte(payload)}, nil) + Expect(err).To(HaveOccurred()) + Expect(decoded.Rows).To(BeEmpty()) + }, + Entry("unrelated content type", "application/x-jfr", `[]`), + Entry("malformed JSON", "application/json", `[`), + Entry("null instead of event array", "application/json", `null`), + Entry("object instead of event array", "application/json", `{}`), + Entry("null event", "application/json", `[null]`), + Entry("missing execution fields", "application/json", `[{"api":"jdbc","operationId":"query","sql":"SELECT 1"}]`), + Entry("untyped event", "application/json", `[{}]`), + Entry("invalid event following valid SQL", "application/json", `[{"startedAt":"2026-11-14T10:00:00Z","durationNs":1000,"sql":"SELECT 1","operationId":"query","api":"jdbc"},null]`), + ) +}) diff --git a/pkg/tracing/sql/pretty.go b/pkg/tracing/sql/pretty.go new file mode 100644 index 0000000..1c58ce3 --- /dev/null +++ b/pkg/tracing/sql/pretty.go @@ -0,0 +1,29 @@ +package sql + +import ( + "errors" + "time" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" + dbgorm "github.com/flanksource/commons-db/gorm" + "github.com/flanksource/commons/properties" +) + +func (r StatementRow) Pretty() api.Text { + duration := time.Duration(r.DurationMs * float64(time.Millisecond)) + threshold := properties.Duration(time.Second, "log.db.slowThreshold") + trace := dbgorm.SQLTrace{SQL: r.SQL, Duration: duration, Rows: r.Rows, Slow: threshold > 0 && duration > threshold} + if r.Error != "" { + trace.Err = errors.New(r.Error) + } + return trace.Pretty() +} + +func (r StatementRow) PrettyFull() api.Textable { + t := r.Pretty().NewLine().Add(clicky.CodeBlock("sql", r.SQL)) + if len(r.Args) > 0 { + t = t.NewLine().Add(clicky.Map(map[string]any{"Arguments": r.Args})) + } + return t +} diff --git a/pkg/tracing/sql/row.go b/pkg/tracing/sql/row.go index 79e0194..1d5fb73 100644 --- a/pkg/tracing/sql/row.go +++ b/pkg/tracing/sql/row.go @@ -21,6 +21,7 @@ const RowKind = "sql" // RepeatKey keys the statements that differ only by their values. type StatementRow struct { tracing.Truncation + Observation StartedAt time.Time `json:"startedAt" pretty:"label=Started" sort:"startedAt"` Connection string `json:"connection" filter:"terms"` Driver string `json:"driver" filter:"terms"` @@ -30,13 +31,26 @@ type StatementRow struct { RepeatKey string `json:"repeatKey" pretty:"hide" filter:"exact"` Args []any `json:"args,omitempty" pretty:"hide"` DurationMs float64 `json:"durationMs" pretty:"label=Duration,type=duration,unit=ms" sort:"durationMs"` - // Rows is how many rows the statement returned, or -1 when its driver - // could not tell. + // Rows is the observed affected or returned row count, or -1 when unknown. Rows int64 `json:"rows" sort:"rows"` Error string `json:"error,omitempty" filter:"text"` Resource map[string]string `json:"resource,omitempty"` } +type Observation struct { + OperationID string `json:"operationId,omitempty" filter:"exact"` + TraceID string `json:"traceId,omitempty" filter:"exact"` + SpanID string `json:"spanId,omitempty" filter:"exact"` + API string `json:"api,omitempty" filter:"terms"` + Database string `json:"database,omitempty" filter:"terms"` + Engine string `json:"engine,omitempty" filter:"terms"` + Phase string `json:"phase,omitempty" filter:"terms"` + Incomplete bool `json:"incomplete,omitempty" filter:"terms"` + Completion string `json:"completion,omitempty" filter:"terms"` + BatchSize *int64 `json:"batchSize,omitempty"` + SQLUnavailableReason string `json:"sqlUnavailableReason,omitempty" filter:"terms"` +} + // FromStatement is the row a capture stores for statement. A statement naming // no table has an empty list of them, never none. func FromStatement(statement connection.Statement) StatementRow { @@ -67,6 +81,17 @@ func (StatementRow) Columns() []api.ColumnDef { api.Column("args").Kind("json").Hidden().Build(), api.Column("repeatKey").Hidden().Build(), api.Column("truncated").Hidden().Build(), + api.Column("operationId").Hidden().Build(), + api.Column("traceId").Hidden().Build(), + api.Column("spanId").Hidden().Build(), + api.Column("api").DefaultHidden().Build(), + api.Column("batchSize").DefaultHidden().Build(), + api.Column("sqlUnavailableReason").DefaultHidden().Build(), + api.Column("phase").DefaultHidden().Build(), + api.Column("incomplete").DefaultHidden().Build(), + api.Column("completion").DefaultHidden().Build(), + api.Column("database").DefaultHidden().Build(), + api.Column("engine").DefaultHidden().Build(), } } @@ -85,12 +110,19 @@ func (r StatementRow) Row() map[string]any { "rows": api.TableCell{Value: rows, FilterValue: r.Rows}, "connection": r.Connection, "driver": r.Driver, "error": r.Error, "args": r.Args, "repeatKey": r.RepeatKey, "truncated": r.Truncated, + "operationId": r.OperationID, "traceId": r.TraceID, "spanId": r.SpanID, + "api": r.API, "batchSize": r.BatchSize, "sqlUnavailableReason": r.SQLUnavailableReason, + "phase": r.Phase, "incomplete": r.Incomplete, "completion": r.Completion, + "database": r.Database, "engine": r.Engine, } } // statementCell is the statement's SQL, and for a statement that failed, its // error in red beneath it. func (r StatementRow) statementCell() api.Textable { + if r.SQLUnavailableReason != "" { + return api.Text{}.AddText("SQL unavailable: "+r.SQLUnavailableReason, "text-muted") + } code := api.CodeBlock("text/x-sql", r.SQL) if r.Error == "" { return code diff --git a/pkg/tracing/sql/row_spec_test.go b/pkg/tracing/sql/row_spec_test.go index 503487f..2f05caa 100644 --- a/pkg/tracing/sql/row_spec_test.go +++ b/pkg/tracing/sql/row_spec_test.go @@ -55,12 +55,19 @@ var _ = Describe("a statement row", func() { Expect(order).To(Equal([]string{ "startedAt", "sql", "op", "tables", "durationMs", "rows", "connection", "driver", "error", "args", "repeatKey", "truncated", + "operationId", "traceId", "spanId", "api", "batchSize", "sqlUnavailableReason", + "phase", "incomplete", "completion", + "database", "engine", })) Expect(shapes).To(Equal(map[string]shape{ "startedAt": {Label: "Time"}, "sql": {Label: "Statement"}, "op": {Label: "Op"}, "tables": {Label: "Tables"}, "durationMs": {Label: "Duration"}, "rows": {Label: "Rows"}, "connection": {Label: "Connection", DefaultHidden: true}, "driver": {Label: "Driver", DefaultHidden: true}, "error": {Hidden: true}, "args": {Hidden: true}, "repeatKey": {Hidden: true}, "truncated": {Hidden: true}, + "operationId": {Hidden: true}, "traceId": {Hidden: true}, "spanId": {Hidden: true}, + "api": {DefaultHidden: true}, "batchSize": {DefaultHidden: true}, "sqlUnavailableReason": {DefaultHidden: true}, + "phase": {DefaultHidden: true}, "incomplete": {DefaultHidden: true}, "completion": {DefaultHidden: true}, + "database": {DefaultHidden: true}, "engine": {DefaultHidden: true}, })) }) diff --git a/pkg/tracing/sql/sql.go b/pkg/tracing/sql/sql.go index ffb8385..4b66353 100644 --- a/pkg/tracing/sql/sql.go +++ b/pkg/tracing/sql/sql.go @@ -5,7 +5,6 @@ package sql import ( "context" - "encoding/json" "errors" "fmt" "strings" @@ -17,7 +16,6 @@ import ( "github.com/flanksource/telltale/pkg/tracing/target" _ "github.com/flanksource/telltale/sdk/commonsdb" - "github.com/flanksource/commons-db/connection" dbcontext "github.com/flanksource/commons-db/context" "github.com/flanksource/commons-db/query" "github.com/flanksource/commons-db/recordstore/recordresults" @@ -74,7 +72,7 @@ var repeatsView = recordresults.ResultView{ Name: "repeats", Title: "Repeated SQL statements", Query: `SELECT repeatKey, min(sql) AS sql, count(*) AS runs, sum(durationMs) AS durationMs, max(durationMs) AS maxDurationMs - FROM stream_rows GROUP BY repeatKey HAVING count(*) > 1`, + FROM stream_rows WHERE repeatKey <> '' AND NOT coalesce(incomplete, false) GROUP BY repeatKey HAVING count(*) > 1`, Columns: []query.ColumnDef{ {Name: "repeatKey", Type: query.ColumnTypeString, Hidden: true}, {Name: "sql", Label: "Statement", Type: query.ColumnTypeString}, @@ -180,15 +178,10 @@ func NewCapture(registry *agentclient.Registry) interface { AgentAvailable(string, target.Target) (bool, error) } { agent := seam.AgentSeam[Params, StatementRow]{Probe: RowKind, Registry: registry, - Target: func(p Params) target.Target { return p.Target }, - ArmParams: func(p Params) (json.RawMessage, error) { return json.Marshal(p) }, - DecodeRecord: func(raw json.RawMessage) (StatementRow, error) { - var statement connection.Statement - if err := json.Unmarshal(raw, &statement); err != nil { - return StatementRow{}, err - } - return FromStatement(statement), nil - }, + Target: func(p Params) target.Target { return p.Target }, + ArmParams: agentArmParams, + DecodeRecord: decodeAgentStatement, + DecodeBlob: decodeAgentBlob, } return statements{agent: agent} } diff --git a/pkg/tracing/sql/summary.go b/pkg/tracing/sql/summary.go index 745a615..9c78aab 100644 --- a/pkg/tracing/sql/summary.go +++ b/pkg/tracing/sql/summary.go @@ -24,14 +24,14 @@ const maxRepeats = 20 type Summary struct { Statements int `json:"statements"` Errors int `json:"errors,omitempty"` - // DurationMs sums every statement's duration; statements on concurrent + Incomplete int `json:"incomplete,omitempty"` + // DurationMs sums completed statements' durations; statements on concurrent // connections overlap, so it can exceed WindowMs. DurationMs float64 `json:"durationMs"` AvgDurationMs float64 `json:"avgDurationMs"` P95DurationMs float64 `json:"p95DurationMs"` MaxDurationMs float64 `json:"maxDurationMs"` - // Rows sums the rows the statements returned, leaving out the statements - // whose driver could not count them. + // Rows sums observed row counts from completed statements. Rows int64 `json:"rows"` // WindowMs is wall clock from the first statement's start to the last // statement's end. @@ -76,6 +76,15 @@ func NewAccumulator() *Accumulator { // Add folds one statement. func (a *Accumulator) Add(row StatementRow) { a.sum.Statements++ + a.sum.ByOp[row.Op]++ + for _, table := range row.Tables { + a.tables[table] = struct{}{} + } + a.observeWindow(row) + if row.Incomplete { + a.sum.Incomplete++ + return + } if row.Error != "" { a.sum.Errors++ } @@ -85,9 +94,8 @@ func (a *Accumulator) Add(row StatementRow) { if row.Rows > 0 { a.sum.Rows += row.Rows } - a.sum.ByOp[row.Op]++ - for _, table := range row.Tables { - a.tables[table] = struct{}{} + if row.RepeatKey == "" { + return } repeat, ok := a.repeats[row.RepeatKey] if !ok { @@ -96,7 +104,6 @@ func (a *Accumulator) Add(row StatementRow) { } repeat.Count++ repeat.DurationMs += row.DurationMs - a.observeWindow(row) } func (a *Accumulator) observeWindow(row StatementRow) { @@ -169,17 +176,25 @@ func WindowVars(rows iter.Seq[StatementRow]) map[string]any { for row := range rows { accumulator.Add(row) if len(queries) < maxSummaryQueries { - queries = append(queries, map[string]any{ + query := map[string]any{ "sql": row.SQL, "op": row.Op, "tables": row.Tables, "rows": row.Rows, "duration_ms": row.DurationMs, "error": row.Error, - }) + } + if row.Incomplete { + query["incomplete"], query["phase"], query["completion"] = true, row.Phase, row.Completion + } + queries = append(queries, query) } } summary := accumulator.Result() - return map[string]any{ + result := map[string]any{ "statements": summary.Statements, "errors": summary.Errors, "duration_ms": summary.DurationMs, "by_op": summary.ByOp, "tables": summary.Tables, "queries": queries, } + if summary.Incomplete > 0 { + result["incomplete"] = summary.Incomplete + } + return result } // Pretty renders the summary as one line, its durations toned as an xevent @@ -191,6 +206,9 @@ func (s Summary) Pretty() api.Text { if s.Errors > 0 { t = t.AddText(" · ", "text-muted").AddText(fmt.Sprintf("%d failed", s.Errors), "text-red-500 font-bold") } + if s.Incomplete > 0 { + t = t.AddText(" · ", "text-muted").AddText(fmt.Sprintf("%d incomplete", s.Incomplete), "text-yellow-500") + } if s.Statements == 0 { return t } diff --git a/pkg/tracing/sql/summary_spec_test.go b/pkg/tracing/sql/summary_spec_test.go index 38f5c00..3ddc2f4 100644 --- a/pkg/tracing/sql/summary_spec_test.go +++ b/pkg/tracing/sql/summary_spec_test.go @@ -4,6 +4,7 @@ package sql_test import ( + "encoding/json" "time" . "github.com/onsi/ginkgo/v2" @@ -72,6 +73,22 @@ var _ = Describe("summarizing statements", func() { Expect(tracesql.Summarize(nil)).To(Equal(tracesql.Summary{Tables: []string{}, ByOp: map[string]int{}})) }) + It("counts incomplete attempts without treating partial elapsed time as completed latency or a repeat", func() { + complete := statement(0, time.Millisecond, labels, 1, "") + incomplete := statement(0, 100*time.Millisecond, labels, -1, "") + incomplete.Incomplete = true + summary := tracesql.Summarize([]tracesql.StatementRow{complete, incomplete}) + Expect(summary.Statements).To(Equal(2)) + Expect(summary.DurationMs).To(Equal(float64(1))) + Expect(summary.AvgDurationMs).To(Equal(float64(1))) + Expect(summary.P95DurationMs).To(Equal(float64(1))) + Expect(summary.MaxDurationMs).To(Equal(float64(1))) + Expect(summary.Repeated).To(BeZero()) + data, err := json.Marshal(summary) + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).To(ContainSubstring(`"incomplete":1`)) + }) + It("lists the statements that ran most often first, then the costliest", func() { var rows []tracesql.StatementRow for _, run := range []struct { diff --git a/pkg/tracing/sqlconn/open.go b/pkg/tracing/sqlconn/open.go new file mode 100644 index 0000000..fc77bad --- /dev/null +++ b/pkg/tracing/sqlconn/open.go @@ -0,0 +1,55 @@ +// Package sqlconn opens the SQL connection a connection-seam capture reads. +package sqlconn + +import ( + "database/sql" + "fmt" + "slices" + "strings" + + "github.com/flanksource/commons-db/connection" + "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/models" +) + +// labels name the connection types an OpenOptions accepts in its refusal. +var labels = map[string]string{models.ConnectionTypeSQLServer: "SQL Server", models.ConnectionTypePostgres: "Postgres"} + +// OpenOptions are the connection a capture leases, the types it reads, and +// the capture named in a refusal. +type OpenOptions struct { + Reference string + Types []string + Purpose string +} + +// Open leases the connection named by options.Reference and opens a client on +// it, returning the client, the connection's type and the release that closes +// the client and gives the lease back. A connection of a type outside +// options.Types is refused. +func Open(ctx context.Context, options OpenOptions) (*sql.DB, string, func(), error) { + conn := connection.SQLConnection{ConnectionName: options.Reference} + if err := conn.HydrateConnection(ctx); err != nil { + return nil, "", nil, err + } + if !slices.Contains(options.Types, conn.Type) { + names := make([]string, 0, len(options.Types)) + for _, connType := range options.Types { + names = append(names, labels[connType]) + } + return nil, "", nil, fmt.Errorf("%s requires a %s connection; %s is a %q connection", options.Purpose, strings.Join(names, " or "), options.Reference, conn.Type) + } + releaseLease, err := ctx.AcquireConnectionLease(options.Reference) + if err != nil { + return nil, "", nil, err + } + client, err := conn.Client(ctx) + if err != nil { + releaseLease() + return nil, "", nil, err + } + return client, conn.Type, func() { + _ = client.Close() + releaseLease() + }, nil +} diff --git a/pkg/tracing/sqlserver/deadlock/kind.go b/pkg/tracing/sqlserver/deadlock/kind.go index 1126339..e4f09b9 100644 --- a/pkg/tracing/sqlserver/deadlock/kind.go +++ b/pkg/tracing/sqlserver/deadlock/kind.go @@ -10,6 +10,7 @@ import ( "github.com/flanksource/clicky" dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/models" "github.com/flanksource/commons-db/recordstore/recordresults" "github.com/flanksource/commons/collections" "github.com/flanksource/telltale/pkg/tracing" @@ -91,6 +92,8 @@ type captureKey struct{} type handler struct{} func (handler) Params() Params { return Params{Options: Options{Poll: "30s"}} } + +func (handler) ConnectionTypes() []string { return []string{models.ConnectionTypeSQLServer} } func (handler) Schema() recordresults.ResultType[history.Row] { return recordresults.ResultType[history.Row]{Title: "SQL Server deadlocks", TimeColumn: "timestamp", Follow: true, SearchColumns: []string{"statements", "signature"}} } diff --git a/pkg/tracing/sqlserver/history/pretty.go b/pkg/tracing/sqlserver/history/pretty.go new file mode 100644 index 0000000..f9ac04e --- /dev/null +++ b/pkg/tracing/sqlserver/history/pretty.go @@ -0,0 +1,21 @@ +package history + +import ( + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" + "github.com/flanksource/telltale/pkg/tracing/sqlserver/deadlocks" +) + +func (r Row) Pretty() api.Text { + shape := deadlocks.Shape(r.Shape) + return clicky.Text("deadlock", "text-red-500 font-bold").Space().Append(r.Database, "text-blue-500"). + Space().Append(shape.Label(), shape.Style()).Space().Append(participantsLabel(r.Participants, r.Victims)+" participants", "text-muted"). + Space().Add(victimStatement(r)) +} + +func (r Row) PrettyFull() api.Textable { + if r.DecodeError != "" { + return r.Pretty().NewLine().Add(clicky.CodeBlock("xml", r.XML)) + } + return r.Graph().Pretty() +} diff --git a/pkg/tracing/sqlserver/plancache/capture.go b/pkg/tracing/sqlserver/plancache/capture.go index 3b0fc15..42fbe84 100644 --- a/pkg/tracing/sqlserver/plancache/capture.go +++ b/pkg/tracing/sqlserver/plancache/capture.go @@ -7,6 +7,7 @@ import ( "sync" "time" + "github.com/flanksource/commons-db/recordstore/sampleframe" "github.com/flanksource/telltale/pkg/tracing" ) @@ -59,11 +60,26 @@ func (c *capture) poll(ctx context.Context, out tracing.Emitter[Sample]) error { if err != nil { return err } - for _, row := range rows { - row.SampledAt, row.IntervalMs = at, float64(at.Sub(c.sampledAt))/float64(time.Millisecond) - if err := out.Emit(ctx, row); err != nil { - return err + observed := len(rows) + limit := c.params.Top + if limit == 0 { + limit = 200 + } + active := 0 + rows = slices.DeleteFunc(rows, func(row Sample) bool { + if row.Evicted { + return false } + active++ + return active > limit + }) + for i := range rows { + rows[i].SampledAt, rows[i].IntervalMs = at, float64(at.Sub(c.sampledAt))/float64(time.Millisecond) + } + if err := tracing.PublishFrame(ctx, out, sampleframe.Frame{Source: c.params.Connection, StartedAt: c.sampledAt, CompletedAt: at, Observed: observed, Summary: map[string]any{"activePlanLimit": limit}}, rows); err != nil { + return err + } + for _, row := range rows { c.summary.add(row) } } @@ -89,13 +105,6 @@ func (c *capture) samples(current map[string]CachedPlan) ([]Sample, error) { } } slices.SortFunc(rows, compareCPU) - top := c.params.Top - if top == 0 { - top = 200 - } - if len(rows) > top { - rows = rows[:top] - } for handle, plan := range c.previous { if _, exists := current[handle]; exists { continue diff --git a/pkg/tracing/sqlserver/plancache/handle_spec_test.go b/pkg/tracing/sqlserver/plancache/handle_spec_test.go index 8f8efa2..847ad67 100644 --- a/pkg/tracing/sqlserver/plancache/handle_spec_test.go +++ b/pkg/tracing/sqlserver/plancache/handle_spec_test.go @@ -7,17 +7,28 @@ import ( dbcontext "github.com/flanksource/commons-db/context" "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore/sampleframe" "github.com/flanksource/telltale/pkg/tracing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) type recordingEmitter struct { + frames []sampleframe.Frame rows []Sample reports []any err error } +func (e *recordingEmitter) EmitFrame(_ context.Context, frame sampleframe.Frame, rows []Sample) error { + if e.err != nil { + return e.err + } + e.frames = append(e.frames, frame) + e.rows = append(e.rows, rows...) + return nil +} + func (e *recordingEmitter) Emit(_ context.Context, s Sample) error { if e.err != nil { return e.err @@ -223,6 +234,8 @@ var _ = Describe("plan cache capture", func() { } Expect(out.rows).To(HaveLen(1)) Expect(out.rows[0].PlanHandle).To(Equal(second.PlanHandle)) + Expect(out.frames[0].Observed).To(Equal(2)) + Expect(out.frames[0].Summary["activePlanLimit"]).To(Equal(1)) }) It("surfaces reader, emission and counter errors", func() { failure := errors.New("read failed") diff --git a/pkg/tracing/sqlserver/plancache/kind.go b/pkg/tracing/sqlserver/plancache/kind.go index f07aa84..2112f10 100644 --- a/pkg/tracing/sqlserver/plancache/kind.go +++ b/pkg/tracing/sqlserver/plancache/kind.go @@ -132,5 +132,5 @@ func (handler) Checkpoint(ctx dbcontext.Context, _ Params, out tracing.Emitter[S } func Kind() *tracing.Handler[Params, Sample] { - return tracing.NewHandler[Params, Sample](handler{}, tracing.Capabilities{Follow: true}, target.Connection) + return tracing.NewHandler[Params, Sample](handler{}, tracing.Capabilities{Follow: true}, target.Connection).WithTop(topSpec()) } diff --git a/pkg/tracing/sqlserver/plancache/pretty.go b/pkg/tracing/sqlserver/plancache/pretty.go new file mode 100644 index 0000000..205b8a7 --- /dev/null +++ b/pkg/tracing/sqlserver/plancache/pretty.go @@ -0,0 +1,32 @@ +package plancache + +import ( + "fmt" + "time" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" + "github.com/flanksource/commons-db/query/sqltext" +) + +func (r Sample) Pretty() api.Text { + duration := time.Duration(r.AvgElapsedMs * float64(time.Millisecond)) + t := clicky.Text(r.Database, "text-blue-500").Space().Append("CPU ", "text-muted").Add(api.Human(time.Duration(r.CPUMs*float64(time.Millisecond)))).Space().Append(fmt.Sprintf("%d executions", r.Executions), "text-muted"). + Space().Append("avg ", "text-muted").Add(api.Human(duration, sqltext.DurationStyle(duration))).Space().Append(r.SQL, "font-mono max-w-[200ch]") + if r.Recompiled { + t = t.Space().Append("recompiled", "text-yellow-500") + } + if r.Evicted { + t = t.Space().Append("evicted", "text-yellow-500") + } + return t +} + +func (r Sample) PrettyFull() api.Textable { + return r.Pretty().NewLine().Add(clicky.CodeBlock("sql", r.SQL)).NewLine().Add(clicky.Map(map[string]any{ + "CPU": api.Human(time.Duration(r.CPUMs * float64(time.Millisecond))), + "Elapsed": api.Human(time.Duration(r.ElapsedMs * float64(time.Millisecond))), + "Reads": r.LogicalReads, "Physical reads": r.PhysicalReads, "Writes": r.Writes, "Rows": r.Rows, + "Plan": r.PlanHandle, "Query hash": r.QueryHash, "Size": api.HumanizeBytes(r.SizeBytes), + })) +} diff --git a/pkg/tracing/sqlserver/plancache/schema.go b/pkg/tracing/sqlserver/plancache/schema.go index 15db406..9f8fbee 100644 --- a/pkg/tracing/sqlserver/plancache/schema.go +++ b/pkg/tracing/sqlserver/plancache/schema.go @@ -1,11 +1,14 @@ package plancache import ( + "github.com/flanksource/commons-db/models" "github.com/flanksource/commons-db/query" "github.com/flanksource/commons-db/recordstore" "github.com/flanksource/commons-db/recordstore/recordresults" ) +func (handler) ConnectionTypes() []string { return []string{models.ConnectionTypeSQLServer} } + var byQueryView = recordresults.ResultView{ Name: "by_query", Title: "Plan costs by query", Query: `SELECT queryHash, SUM(executions) AS executions, SUM(cpuMs) AS cpuMs, diff --git a/pkg/tracing/sqlserver/plancache/table.go b/pkg/tracing/sqlserver/plancache/table.go new file mode 100644 index 0000000..fbfbecc --- /dev/null +++ b/pkg/tracing/sqlserver/plancache/table.go @@ -0,0 +1,17 @@ +package plancache + +import ( + "github.com/flanksource/clicky/api" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/telltale/pkg/tracing" +) + +func (Sample) Columns() []api.ColumnDef { + return query.TableColumnsFor[Sample]("cpuMs", "executions", "elapsedMs", "logicalReads", "sql", "database") +} +func (s Sample) Row() map[string]any { return query.TableRowFor(s) } +func (s Sample) RowDetail() api.Textable { return s.PrettyFull() } + +func topSpec() tracing.TopSpec { + return tracing.TopSpec{Identity: []string{"planHandle"}, Order: query.Order{{Column: "cpuMs", Desc: true}, {Column: "planHandle"}}} +} diff --git a/pkg/tracing/sqlserver/xeprovider/sqlxevent.go b/pkg/tracing/sqlserver/xeprovider/sqlxevent.go index 5f9ae33..d7659c2 100644 --- a/pkg/tracing/sqlserver/xeprovider/sqlxevent.go +++ b/pkg/tracing/sqlserver/xeprovider/sqlxevent.go @@ -11,10 +11,10 @@ import ( "github.com/google/uuid" - "github.com/flanksource/commons-db/connection" "github.com/flanksource/commons-db/context" "github.com/flanksource/commons-db/models" "github.com/flanksource/commons-db/query" + "github.com/flanksource/telltale/pkg/tracing/sqlconn" "github.com/flanksource/telltale/pkg/tracing/sqlserver/xetrace" ) @@ -259,26 +259,8 @@ func (sqlXEventProvider) Stream(ctx context.Context, req query.ProviderRequest, // client on it for an Extended Events capture; release closes the client and // gives the lease back. A connection of any other type is refused. func OpenSQLServer(ctx context.Context, reference string) (*sql.DB, func(), error) { - conn := connection.SQLConnection{ConnectionName: reference} - if err := conn.HydrateConnection(ctx); err != nil { - return nil, nil, err - } - if conn.Type != models.ConnectionTypeSQLServer { - return nil, nil, fmt.Errorf("SQL XEvents requires a SQL Server connection") - } - releaseLease, err := ctx.AcquireConnectionLease(reference) - if err != nil { - return nil, nil, err - } - client, err := conn.Client(ctx) - if err != nil { - releaseLease() - return nil, nil, err - } - return client, func() { - _ = client.Close() - releaseLease() - }, nil + client, _, release, err := sqlconn.Open(ctx, sqlconn.OpenOptions{Reference: reference, Types: []string{models.ConnectionTypeSQLServer}, Purpose: "SQL XEvents"}) + return client, release, err } // eventRow converts a captured event to a generic row through its own JSON diff --git a/pkg/tracing/sqlserver/xesession/pretty.go b/pkg/tracing/sqlserver/xesession/pretty.go new file mode 100644 index 0000000..287f965 --- /dev/null +++ b/pkg/tracing/sqlserver/xesession/pretty.go @@ -0,0 +1,25 @@ +package xesession + +import ( + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" + "github.com/flanksource/telltale/pkg/tracing/sqlserver/xetrace" +) + +func (r EventRow) Pretty() api.Text { + return xetrace.StreamLine(r.Event(), xetrace.StreamLineOptions{ShowDatabase: true}) +} + +func (r EventRow) PrettyFull() api.Textable { + t := xetrace.StreamLine(r.Event(), xetrace.StreamLineOptions{ShowDatabase: true, Full: true}) + if r.SQL != "" { + t = t.NewLine().Add(clicky.CodeBlock("sql", r.SQL)) + } + if r.Deadlock != nil { + t = t.NewLine().Add(r.Deadlock.Pretty()) + } + for _, child := range r.Children { + t = t.NewLine().Add(child.PrettyFull()) + } + return t +} diff --git a/pkg/tracing/sqlserver/xevent/xevent.go b/pkg/tracing/sqlserver/xevent/xevent.go index feeafdd..5a848a4 100644 --- a/pkg/tracing/sqlserver/xevent/xevent.go +++ b/pkg/tracing/sqlserver/xevent/xevent.go @@ -17,6 +17,7 @@ import ( "github.com/flanksource/telltale/pkg/tracing/target" dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/models" "github.com/flanksource/commons-db/query" "github.com/flanksource/commons-db/recordstore/recordresults" ) @@ -96,6 +97,8 @@ func (events[P]) Params() P { return defaults } +func (events[P]) ConnectionTypes() []string { return []string{models.ConnectionTypeSQLServer} } + // Schema follows a capture's stream while it runs, and searches the text a // reader remembers of a statement rather than a column. func (events[P]) Schema() recordresults.ResultType[xesession.EventRow] { diff --git a/pkg/tracing/sqltop/active.go b/pkg/tracing/sqltop/active.go new file mode 100644 index 0000000..0bfd230 --- /dev/null +++ b/pkg/tracing/sqltop/active.go @@ -0,0 +1,49 @@ +package sqltop + +import ( + "fmt" + "time" + + "github.com/flanksource/commons-db/recordstore/recordresults" +) + +type ActiveSample struct { + Source string `json:"source" filter:"terms"` + OperationID string `json:"operationId" filter:"exact"` + BlockedByOperationID string `json:"blockedByOperationId,omitempty" filter:"exact"` + SampledAt time.Time `json:"sampledAt" filter:"time"` + StartedAt time.Time `json:"startedAt" filter:"time"` + ElapsedMs float64 `json:"elapsedMs" filter:"duration"` + API string `json:"api" filter:"terms"` + Phase string `json:"phase,omitempty" filter:"terms"` + Database string `json:"database,omitempty" filter:"terms"` + Engine string `json:"engine,omitempty" filter:"terms"` + SQL string `json:"sql,omitempty" filter:"text"` + QueryFingerprint string `json:"queryFingerprint,omitempty" filter:"exact"` + SQLTruncated bool `json:"sqlTruncated,omitempty"` + SQLUnavailableReason string `json:"sqlUnavailableReason,omitempty" filter:"terms"` + Connection string `json:"connection,omitempty" filter:"terms"` + TraceID string `json:"traceId,omitempty" filter:"exact"` + SpanID string `json:"spanId,omitempty" filter:"exact"` + BatchSize *int64 `json:"batchSize,omitempty"` +} + +func ActiveResult() recordresults.ResultType[ActiveSample] { + return recordresults.ResultType[ActiveSample]{Kind: "sql_top_active", Title: "Active SQL operations", TimeColumn: "sampledAt", Follow: true, SearchColumns: []string{"sql", "sqlUnavailableReason", "database"}} +} + +func projectDatabase(source string, rows []Sample) []ActiveSample { + projected := make([]ActiveSample, len(rows)) + for i, row := range rows { + projected[i] = ActiveSample{Source: source, OperationID: fmt.Sprintf("%d/%d", row.SessionID, row.RequestID), SampledAt: row.SampledAt, StartedAt: row.StartedAt, ElapsedMs: row.ElapsedMs, API: "database", Database: row.Database, Engine: row.Engine, SQL: row.SQL, Connection: source} + if row.BlockedBy != 0 { + for _, blocker := range rows { + if blocker.SessionID == row.BlockedBy { + projected[i].BlockedByOperationID = fmt.Sprintf("%d/%d", blocker.SessionID, blocker.RequestID) + break + } + } + } + } + return projected +} diff --git a/pkg/tracing/sqltop/active_spec_test.go b/pkg/tracing/sqltop/active_spec_test.go new file mode 100644 index 0000000..ffd2e5f --- /dev/null +++ b/pkg/tracing/sqltop/active_spec_test.go @@ -0,0 +1,39 @@ +package sqltop + +import ( + "encoding/json" + "time" + + "github.com/flanksource/telltale/pkg/tracing/target" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("SQL active projection", func() { + It("preserves database identities and observed blockers in a common top row", func() { + rows := projectDatabase("connection://orders", []Sample{{SampledAt: serverTime, StartedAt: serverTime.Add(-time.Second), SessionID: 7, RequestID: 2, BlockedBy: 9, ElapsedMs: 1000, SQL: "UPDATE orders SET status = ?", Database: "orders", Engine: EnginePostgres}, {SessionID: 9, RequestID: 3}}) + Expect(rows).To(HaveLen(2)) + Expect(rows[0]).To(Equal(ActiveSample{Source: "connection://orders", OperationID: "7/2", BlockedByOperationID: "9/3", SampledAt: serverTime, StartedAt: serverTime.Add(-time.Second), ElapsedMs: 1000, API: "database", Database: "orders", Engine: EnginePostgres, SQL: "UPDATE orders SET status = ?", Connection: "connection://orders"})) + }) + + It("does not serialize unobserved database metrics into JVM operations", func() { + data, err := json.Marshal(ActiveSample{Source: "java/1", OperationID: "one", SampledAt: serverTime, StartedAt: serverTime.Add(-time.Second), ElapsedMs: 1000, API: "jdbc", SQLUnavailableReason: "prepared-before-instrumentation"}) + Expect(err).NotTo(HaveOccurred()) + var row map[string]any + Expect(json.Unmarshal(data, &row)).To(Succeed()) + Expect(row).NotTo(HaveKey("sessionId")) + Expect(row).NotTo(HaveKey("blocked")) + Expect(row).NotTo(HaveKey("cpuMs")) + Expect(row).NotTo(HaveKey("openTransactions")) + }) + + It("allows JVM agent targets without database credentials and rejects database-only settings", func() { + params := Params{Target: target.Target{Seam: target.Agent}} + Expect(params.Validate()).To(Succeed()) + params.Idle = true + Expect(params.Validate()).To(MatchError(ContainSubstring("idle"))) + params.Idle = false + params.Connection = "connection://orders" + Expect(params.Validate()).To(MatchError(ContainSubstring("connection"))) + }) +}) diff --git a/pkg/tracing/sqltop/active_store.go b/pkg/tracing/sqltop/active_store.go new file mode 100644 index 0000000..43fe254 --- /dev/null +++ b/pkg/tracing/sqltop/active_store.go @@ -0,0 +1,121 @@ +package sqltop + +import ( + "context" + "fmt" + "sync" + + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/flanksource/telltale/pkg/otlp/profilepublication" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/google/uuid" +) + +type activeStore struct { + mu sync.Mutex + backend recordstore.BatchAppender + stream string + sequence int64 + activeSeq int64 + schemas []recordstore.KindSchema +} + +func newActiveStore(backend recordstore.BatchAppender, stream string) (*activeStore, error) { + if backend == nil { + return nil, fmt.Errorf("SQL top requires an atomic capture backend") + } + if err := recordstore.ValidateStream(stream); err != nil { + return nil, err + } + nativeType := (handler{}).Schema() + nativeType.Kind = "sql_top" + nativeSchema, err := nativeType.KindSchema() + if err != nil { + return nil, err + } + activeSchema, err := ActiveResult().KindSchema() + if err != nil { + return nil, err + } + frameSchema, err := sampleframe.Schema("sql_top_frames") + if err != nil { + return nil, err + } + return &activeStore{backend: backend, stream: stream, schemas: []recordstore.KindSchema{nativeSchema, activeSchema, frameSchema}}, nil +} + +func (s *activeStore) append(ctx context.Context, frame sampleframe.Frame, active []ActiveSample, native []Sample) error { + s.mu.Lock() + defer s.mu.Unlock() + if frame.Source == "" || frame.StartedAt.IsZero() || frame.CompletedAt.Before(frame.StartedAt) || frame.Observed < len(active) || frame.Limit < 0 || (frame.Limit > 0 && len(active) > frame.Limit) { + return fmt.Errorf("SQL top snapshot requires source, ordered timestamps and observed count within the limit") + } + activeRows, err := encodeRows(active) + if err != nil { + return err + } + nativeRows, err := encodeRows(native) + if err != nil { + return err + } + if profilepublication.IsProfile(frame) { + return s.publishProfile(ctx, frame, activeRows, nativeRows) + } + frame.ID, frame.Result, frame.Tick = uuid.NewString(), "active", s.sequence+1 + frame.AfterSeq, frame.ToSeq, frame.Retained = s.activeSeq, s.activeSeq+int64(len(active)), len(active) + encodedFrame, err := recordstore.EncodeRow(frame) + if err != nil { + return err + } + result, err := s.backend.AppendBatch(ctx, recordstore.Batch{ID: frame.ID, Atomic: true, Producer: recordstore.Producer{Instance: s.stream, Seq: s.sequence + 1}, Schemas: s.schemas, Entries: []recordstore.BatchEntry{ + {Op: recordstore.BatchAppend, Kind: "sql_top", Stream: s.stream, Rows: nativeRows}, + {Op: recordstore.BatchAppend, Kind: "sql_top_active", Stream: s.stream + "-active", Rows: activeRows}, + {Op: recordstore.BatchAppend, Kind: "sql_top_frames", Stream: s.stream + "-frames", Rows: []recordstore.Row{encodedFrame}}, + }}) + if err != nil { + return err + } + if len(result.Entries) != 3 { + return fmt.Errorf("SQL top atomic batch returned %d entries, expected 3", len(result.Entries)) + } + for _, entry := range result.Entries { + if entry.Error != nil { + return entry.Error + } + } + s.activeSeq, s.sequence = frame.ToSeq, s.sequence+1 + return nil +} + +func (s *activeStore) publishProfile(ctx context.Context, frame sampleframe.Frame, active, native []recordstore.Row) error { + backend, ok := s.backend.(profilepublication.Backend) + if !ok { + return fmt.Errorf("SQL profile capture requires durable identity reads and atomic publication") + } + frame.Result = "active" + _, err := profilepublication.Append(ctx, backend, profilepublication.Publication{Kind: "sql_top", Stream: s.stream + "-active", FrameStream: s.stream + "-frames", Schema: s.schemas[1], FrameSchema: s.schemas[2], Frame: frame, Rows: active, + AdditionalSchemas: []recordstore.KindSchema{s.schemas[0]}, AdditionalEntries: []recordstore.BatchEntry{{Op: recordstore.BatchAppend, Kind: "sql_top", Stream: s.stream, Rows: native}}}) + return err +} + +func encodeRows[T any](values []T) ([]recordstore.Row, error) { + rows := make([]recordstore.Row, len(values)) + for i, value := range values { + row, err := recordstore.EncodeRow(value) + if err != nil { + return nil, err + } + rows[i] = row + } + return rows, nil +} + +type activeEmitter struct { + tracing.Emitter[Sample] + store *activeStore +} + +func (e activeEmitter) EmitFrame(ctx context.Context, frame sampleframe.Frame, rows []Sample) error { + return e.store.append(ctx, frame, projectDatabase(frame.Source, rows), rows) +} diff --git a/pkg/tracing/sqltop/active_store_spec_test.go b/pkg/tracing/sqltop/active_store_spec_test.go new file mode 100644 index 0000000..8273bc3 --- /dev/null +++ b/pkg/tracing/sqltop/active_store_spec_test.go @@ -0,0 +1,113 @@ +package sqltop + +import ( + "context" + "fmt" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/commons-db/recordstore/recordresults/recordresultstest" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/tracingtest" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("SQL top active storage", func() { + It("stores database details and the common top projection with an empty follow-up frame", func() { + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{"sql_top": Kind()}) + store, err := newActiveStore(env.Results.Backend, "sql-active-test") + Expect(err).NotTo(HaveOccurred()) + native := []Sample{{SampledAt: serverTime, SessionID: 7, RequestID: 2, SQL: "SELECT id FROM orders", ElapsedMs: 50}} + frame := sampleframe.Frame{Source: "connection://orders", StartedAt: serverTime, CompletedAt: serverTime, Observed: 1} + Expect(store.append(context.Background(), frame, projectDatabase(frame.Source, native), native)).To(Succeed()) + frame.Observed = 0 + Expect(store.append(context.Background(), frame, nil, nil)).To(Succeed()) + Expect(env.Read("sql_top", "sql-active-test", query.PageRequest{Limit: 20}, nil)).To(HaveLen(1)) + Expect(env.Read("sql_top_active", "sql-active-test-active", query.PageRequest{Limit: 20}, nil)).To(HaveLen(1)) + frames := env.Read("sql_top_frames", "sql-active-test-frames", query.PageRequest{Limit: 20}, nil) + Expect(frames).To(HaveLen(2)) + Expect(frames[0]).To(HaveKeyWithValue("result", "active")) + Expect(frames[1]).To(HaveKeyWithValue("retained", BeNumerically("==", 0))) + Expect(frames[1]).To(HaveKeyWithValue("afterSeq", BeNumerically("==", 1))) + Expect(frames[1]).To(HaveKeyWithValue("toSeq", BeNumerically("==", 1))) + }) + + It("deduplicates retried profile frames and rejects conflicting sequences", func() { + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{"sql_top": Kind()}) + store, err := newActiveStore(env.Results.Backend, "sql-profile-retry") + Expect(err).NotTo(HaveOccurred()) + frame := profileStoreFrame(1, "java") + Expect(store.append(context.Background(), frame, nil, nil)).To(Succeed()) + Expect(store.append(context.Background(), frame, nil, nil)).To(Succeed()) + Expect(env.Read("sql_top_frames", "sql-profile-retry-frames", query.PageRequest{Limit: 20}, nil)).To(HaveLen(1)) + frame.Summary["digest"] = fmt.Sprintf("%064x", 2) + Expect(store.append(context.Background(), frame, nil, nil)).To(MatchError(ContainSubstring("conflicting"))) + }) + + It("preserves interleaved producer ticks and completeness while retaining reordered history and old retries", func() { + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{"sql_top": Kind()}) + store, err := newActiveStore(env.Results.Backend, "sql-profile-order") + Expect(err).NotTo(HaveOccurred()) + for _, frame := range []sampleframe.Frame{profileStoreFrame(7, "java-a"), profileStoreFrame(2, "java-b"), profileStoreFrame(9, "java-a"), profileStoreFrame(8, "java-a")} { + Expect(store.append(context.Background(), frame, nil, nil)).To(Succeed()) + } + restarted, err := newActiveStore(env.Results.Backend, "sql-profile-order") + Expect(err).NotTo(HaveOccurred()) + retry := profileStoreFrame(7, "java-a") + retry.Summary["receivedAt"] = "retry" + Expect(restarted.append(context.Background(), retry, nil, nil)).To(Succeed()) + frames := env.Read("sql_top_frames", "sql-profile-order-frames", query.PageRequest{Limit: 20}, nil) + Expect(frames).To(HaveLen(4)) + for index, tick := range []int64{7, 2, 9, 8} { + Expect(frames[index]).To(HaveKeyWithValue("tick", BeNumerically("==", tick))) + Expect(frames[index]["summary"]).To(HaveKeyWithValue("complete", true)) + } + Expect(frames[0]["summary"]).To(HaveKeyWithValue("receivedAt", "first")) + }) + + It("rejects a profile UUID reused across producer sources and sequences", func() { + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{"sql_top": Kind()}) + store, err := newActiveStore(env.Results.Backend, "sql-profile-uuid") + Expect(err).NotTo(HaveOccurred()) + first := profileStoreFrame(3, "java-a") + Expect(store.append(context.Background(), first, nil, nil)).To(Succeed()) + conflict := profileStoreFrame(4, "java-b") + conflict.Summary["profileId"] = first.Summary["profileId"] + Expect(store.append(context.Background(), conflict, nil, nil)).To(MatchError(ContainSubstring("conflicting"))) + Expect(env.Read("sql_top_frames", "sql-profile-uuid-frames", query.PageRequest{Limit: 20}, nil)).To(HaveLen(1)) + }) + + It("deduplicates an old retry after closing and reopening the registered results store", func() { + catalog := tracing.NewKinds() + Expect(catalog.RegisterKind("sql_top", Kind())).To(Succeed()) + settings := recordresultstest.LocalSettings(recordstore.BackendSQLite) + open := func() *recordresults.Results { + results, err := recordresults.Open(recordresults.OpenOptions{Prefix: "profiles", ConnectionName: "profiles", Settings: settings, Register: catalog.RegisterResultTypes}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(results.Close) + return results + } + first := open() + store, err := newActiveStore(first.Backend, "sql-profile-reopen") + Expect(err).NotTo(HaveOccurred()) + Expect(store.append(context.Background(), profileStoreFrame(3, "java"), nil, nil)).To(Succeed()) + Expect(store.append(context.Background(), profileStoreFrame(5, "java"), nil, nil)).To(Succeed()) + Expect(first.Close()).To(Succeed()) + second := open() + store, err = newActiveStore(second.Backend, "sql-profile-reopen") + Expect(err).NotTo(HaveOccurred()) + Expect(store.append(context.Background(), profileStoreFrame(3, "java"), nil, nil)).To(Succeed()) + var frames []recordstore.Row + Expect(second.Backend.Scan(context.Background(), "sql-profile-reopen-frames", 0, func(_ int64, row recordstore.Row) error { frames = append(frames, row); return nil })).To(Succeed()) + Expect(frames).To(HaveLen(2)) + Expect(frames[0]).To(HaveKeyWithValue("tick", BeNumerically("==", 3))) + }) +}) + +func profileStoreFrame(sequence int64, source string) sampleframe.Frame { + return sampleframe.Frame{Source: source + "/capture", Tick: sequence, StartedAt: serverTime, CompletedAt: serverTime, + Summary: map[string]any{"profileId": fmt.Sprintf("00000000-0000-4000-8000-%012d", sequence), "captureId": "capture", "digest": fmt.Sprintf("%064x", sequence), "complete": true, "receivedAt": "first"}} +} diff --git a/pkg/tracing/sqltop/agent.go b/pkg/tracing/sqltop/agent.go new file mode 100644 index 0000000..39f8020 --- /dev/null +++ b/pkg/tracing/sqltop/agent.go @@ -0,0 +1,106 @@ +package sqltop + +import ( + "cmp" + "context" + "encoding/json" + "fmt" + "slices" + "time" + + "github.com/flanksource/commons/collections" + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/otlp/profiles" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/seam" + "github.com/flanksource/telltale/pkg/tracing/target" +) + +type activeStoreKey struct{} + +func agentArmParams(p Params) (json.RawMessage, error) { + interval, err := p.pollInterval() + if err != nil { + return nil, err + } + minimum, err := p.minElapsed() + if err != nil { + return nil, err + } + return json.Marshal(struct { + PollIntervalMS int64 `json:"pollIntervalMs"` + Limit int `json:"limit"` + Databases []string `json:"databases,omitempty"` + Text []string `json:"text,omitempty"` + MinElapsed string `json:"minElapsed,omitempty"` + MinElapsedNS int64 `json:"minElapsedNs,omitempty"` + }{interval.Milliseconds(), p.top(), p.Databases, p.Text, p.MinElapsed, int64(minimum)}) +} + +func (handler) AgentAvailable(probe string, selection target.Target) (bool, error) { + return agentCapture(Params{Target: selection}).Available(probe, selection) +} + +func agentCapture(p Params) seam.AgentSeam[Params, Sample] { + return seam.AgentSeam[Params, Sample]{Probe: "sql_top", Target: func(p Params) target.Target { return p.Target }, ArmParams: agentArmParams, + HandleBlob: func(ctx context.Context, blob agentclient.Blob, resource map[string]string, _ tracing.Emitter[Sample]) error { + store, ok := ctx.Value(activeStoreKey{}).(*activeStore) + if !ok { + return fmt.Errorf("SQL top active store was not prepared") + } + if blob.Info.ContentType != profiles.ContentType { + return fmt.Errorf("SQL top requires OTLP Profiles, got %q", blob.Info.ContentType) + } + source := resource["telltale.source.instance"] + if source == "" { + return fmt.Errorf("SQL top agent Hello is missing telltale.source.instance") + } + if blob.Info.Session == "" { + return fmt.Errorf("SQL top agent blob is missing the authenticated capture session") + } + snapshots, err := profiles.Decode(blob.Data, profiles.Options{ExpectedKind: "sql_top", ExpectedCaptureID: blob.Info.Session, ExpectedSource: source}) + if err != nil { + return err + } + for _, snapshot := range snapshots { + rows, err := FromSnapshot(snapshot) + if err != nil { + return err + } + rows, err = filterActive(p.Options, rows) + if err != nil { + return err + } + frame := snapshot.Frame() + if err := store.append(ctx, frame, rows, nil); err != nil { + return err + } + } + return nil + }, + } +} + +func filterActive(options Options, rows []ActiveSample) ([]ActiveSample, error) { + minimum, err := options.minElapsed() + if err != nil { + return nil, err + } + kept := make([]ActiveSample, 0, len(rows)) + for _, row := range rows { + if row.ElapsedMs < float64(minimum)/float64(time.Millisecond) { + continue + } + if patterns := normalizePatterns(options.Databases); len(patterns) > 0 && !collections.MatchItems(row.Database, patterns...) { + continue + } + if patterns := substringPatterns(options.Text); len(patterns) > 0 && !collections.MatchItems(row.SQL, patterns...) { + continue + } + kept = append(kept, row) + } + slices.SortStableFunc(kept, func(a, b ActiveSample) int { + return cmp.Or(cmp.Compare(b.ElapsedMs, a.ElapsedMs), cmp.Compare(a.OperationID, b.OperationID)) + }) + return kept[:min(len(kept), options.top())], nil +} diff --git a/pkg/tracing/sqltop/capture.go b/pkg/tracing/sqltop/capture.go new file mode 100644 index 0000000..21917c7 --- /dev/null +++ b/pkg/tracing/sqltop/capture.go @@ -0,0 +1,137 @@ +package sqltop + +import ( + "cmp" + "context" + "fmt" + "slices" + "strings" + "sync" + "time" + + "github.com/flanksource/commons-db/query/sqlactivity" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/flanksource/commons/collections" + "github.com/flanksource/telltale/pkg/tracing" +) + +// snapshot is one read of an engine: the server's time and its requests. +type snapshot = sqlactivity.Snapshot + +type capture struct { + mu sync.Mutex + options Options + engine string + read func(context.Context, bool) (snapshot, error) + databases []string + text []string + minElapsed time.Duration + polls int +} + +func newCapture(options Options, engine string, read func(context.Context, bool) (snapshot, error)) (*capture, error) { + minElapsed, err := options.minElapsed() + if err != nil { + return nil, err + } + return &capture{options: options, engine: engine, read: read, minElapsed: minElapsed, + databases: normalizePatterns(options.Databases), text: substringPatterns(options.Text)}, nil +} + +// poll reads the engine once and emits the busiest requests it keeps, by +// elapsed time, at most options.Top of them. +func (c *capture) poll(ctx context.Context, out tracing.Emitter[Sample]) error { + c.mu.Lock() + defer c.mu.Unlock() + result, err := c.read(ctx, c.options.Idle) + if err != nil { + return fmt.Errorf("sample %s requests: %w", c.engine, err) + } + rows, err := c.keep(result) + if err != nil { + return err + } + active, blocked := len(rows), 0 + for _, row := range rows { + if row.Blocked { + blocked++ + } + } + slices.SortStableFunc(rows, func(a, b Sample) int { + return cmp.Or(cmp.Compare(b.ElapsedMs, a.ElapsedMs), cmp.Compare(a.SessionID, b.SessionID), cmp.Compare(a.RequestID, b.RequestID)) + }) + if len(rows) > c.options.top() { + rows = rows[:c.options.top()] + } + if err := tracing.PublishFrame(ctx, out, sampleframe.Frame{Source: c.options.Connection, StartedAt: result.ServerTime, CompletedAt: result.ServerTime, Observed: active, Limit: c.options.top()}, rows); err != nil { + return err + } + c.polls++ + out.Report(map[string]any{"engine": c.engine, "polls": c.polls, "active": active, "blocked": blocked, "recorded": len(rows)}) + return nil +} + +// keep stamps every request with the server time and drops those the options +// filter out, refusing a snapshot without a server time or with a request +// twice. +func (c *capture) keep(result snapshot) ([]Sample, error) { + if result.ServerTime.IsZero() { + return nil, fmt.Errorf("%s snapshot has no server timestamp", c.engine) + } + seen := map[[2]int64]bool{} + var rows []Sample + for _, row := range result.Rows { + key := [2]int64{row.SessionID, row.RequestID} + if seen[key] { + return nil, fmt.Errorf("%s snapshot has session %d request %d twice", c.engine, row.SessionID, row.RequestID) + } + seen[key] = true + row.Engine = c.engine + row.Derive(result.ServerTime) + if len(c.databases) > 0 && !collections.MatchItems(row.Database, c.databases...) { + continue + } + if len(c.text) > 0 && !collections.MatchItems(row.SQL, c.text...) { + continue + } + if row.ElapsedMs < float64(c.minElapsed)/float64(time.Millisecond) { + continue + } + rows = append(rows, row) + } + return rows, nil +} + +func normalizePatterns(values []string) []string { + var out []string + for _, value := range values { + for _, part := range strings.Split(value, ",") { + if part = strings.TrimSpace(part); part != "" { + out = append(out, part) + } + } + } + return out +} + +func validatePatterns(name string, values []string) error { + for _, pattern := range normalizePatterns(values) { + if pattern == "!" { + return fmt.Errorf("%s pattern %q excludes no value", name, pattern) + } + } + return nil +} + +// substringPatterns makes each bare text pattern match anywhere in the +// statement, as a pattern with its own wildcards matches as written. +func substringPatterns(values []string) []string { + patterns := normalizePatterns(values) + for i, pattern := range patterns { + value := strings.TrimPrefix(pattern, "!") + if !strings.Contains(value, "*") { + patterns[i] = strings.Replace(pattern, value, "*"+value+"*", 1) + } + } + return patterns +} diff --git a/pkg/tracing/sqltop/capture_spec_test.go b/pkg/tracing/sqltop/capture_spec_test.go new file mode 100644 index 0000000..81ed6c1 --- /dev/null +++ b/pkg/tracing/sqltop/capture_spec_test.go @@ -0,0 +1,161 @@ +package sqltop + +import ( + "context" + "errors" + "time" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore/sampleframe" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var serverTime = time.Date(2026, 11, 9, 10, 0, 0, 0, time.UTC) + +// recorder is an Emitter that keeps what a capture emits and reports. +type recorder struct { + frames []sampleframe.Frame + rows []Sample + report any +} + +func (r *recorder) EmitFrame(_ context.Context, frame sampleframe.Frame, rows []Sample) error { + r.frames = append(r.frames, frame) + r.rows = append(r.rows, rows...) + return nil +} + +func (r *recorder) Emit(_ context.Context, row Sample) error { + r.rows = append(r.rows, row) + return nil +} +func (r *recorder) TryEmit(row Sample) bool { r.rows = append(r.rows, row); return true } +func (r *recorder) Report(summary any) { r.report = summary } +func (r *recorder) Warn(string) {} +func (r *recorder) Describe(string, ...query.SessionMetadata) {} +func (r *recorder) Export(string, string) {} + +func started(ago time.Duration) time.Time { return serverTime.Add(-ago) } + +func pollOnce(options Options, result snapshot) (*recorder, error) { + var idleAsked bool + c, err := newCapture(options, EnginePostgres, func(_ context.Context, idle bool) (snapshot, error) { + idleAsked = idle + return result, nil + }) + Expect(err).NotTo(HaveOccurred()) + out := &recorder{} + err = c.poll(context.Background(), out) + Expect(idleAsked).To(Equal(options.Idle)) + return out, err +} + +func sessions(rows []Sample) []int64 { + ids := make([]int64, 0, len(rows)) + for _, row := range rows { + ids = append(ids, row.SessionID) + } + return ids +} + +var _ = Describe("sql top options", func() { + DescribeTable("refuses invalid options before connecting", func(options Options, message string) { + Expect(options.Validate()).To(MatchError(ContainSubstring(message))) + }, + Entry("no connection", Options{}, "connection is required"), + Entry("unparsable poll", Options{Connection: "connection://db", Poll: "soon"}, `poll "soon"`), + Entry("zero poll", Options{Connection: "connection://db", Poll: "0s"}, "positive duration"), + Entry("unparsable minimum elapsed", Options{Connection: "connection://db", MinElapsed: "long"}, `minElapsed "long"`), + Entry("negative top", Options{Connection: "connection://db", Top: -1}, "top must be non-negative"), + Entry("bare database exclusion", Options{Connection: "connection://db", Databases: []string{"orders,!"}}, "databases pattern"), + Entry("bare text exclusion", Options{Connection: "connection://db", Text: []string{"!"}}, "text pattern"), + ) + It("defaults to a five second poll and the fifty longest requests", func() { + options := (handler{}).Params().Options + Expect(options.Validate()).To(MatchError(ContainSubstring("connection is required"))) + options.Connection = "connection://db" + Expect(options.Validate()).To(Succeed()) + Expect(options.pollInterval()).To(Equal(5 * time.Second)) + Expect(Options{}.top()).To(Equal(50)) + }) +}) + +var _ = Describe("sql top capture", func() { + It("stamps every request with the server time and derives elapsed, transaction age and blocked", func() { + out, err := pollOnce(Options{}, snapshot{ServerTime: serverTime, Rows: []Sample{ + {SessionID: 7, StartedAt: started(1500 * time.Millisecond), TransactionStartedAt: started(4 * time.Second), BlockedBy: 9}, + {SessionID: 9, ElapsedMs: 250, StartedAt: started(time.Hour)}, + }}) + Expect(err).NotTo(HaveOccurred()) + Expect(out.rows).To(Equal([]Sample{ + {SampledAt: serverTime, Engine: EnginePostgres, SessionID: 7, BlockedBy: 9, Blocked: true, + StartedAt: started(1500 * time.Millisecond), TransactionStartedAt: started(4 * time.Second), ElapsedMs: 1500, TransactionMs: 4000}, + {SampledAt: serverTime, Engine: EnginePostgres, SessionID: 9, ElapsedMs: 250, StartedAt: started(time.Hour)}, + })) + Expect(out.report).To(Equal(map[string]any{"engine": EnginePostgres, "polls": 1, "active": 2, "blocked": 1, "recorded": 2})) + }) + It("ranks by elapsed time, longest first, and keeps the top requests", func() { + out, err := pollOnce(Options{Top: 2}, snapshot{ServerTime: serverTime, Rows: []Sample{ + {SessionID: 1, ElapsedMs: 10}, {SessionID: 2, ElapsedMs: 300}, {SessionID: 3, ElapsedMs: 20}, + }}) + Expect(err).NotTo(HaveOccurred()) + Expect(sessions(out.rows)).To(Equal([]int64{2, 3})) + Expect(out.report).To(HaveKeyWithValue("active", 3)) + Expect(out.report).To(HaveKeyWithValue("recorded", 2)) + }) + It("drops requests shorter than the minimum elapsed time", func() { + out, err := pollOnce(Options{MinElapsed: "100ms"}, snapshot{ServerTime: serverTime, Rows: []Sample{ + {SessionID: 1, ElapsedMs: 99}, {SessionID: 2, ElapsedMs: 100}, + }}) + Expect(err).NotTo(HaveOccurred()) + Expect(sessions(out.rows)).To(Equal([]int64{2})) + }) + DescribeTable("filters by database and statement text patterns", func(options Options, want []int64) { + out, err := pollOnce(options, snapshot{ServerTime: serverTime, Rows: []Sample{ + {SessionID: 1, ElapsedMs: 3, Database: "orders", SQL: "SELECT * FROM ledger"}, + {SessionID: 2, ElapsedMs: 2, Database: "orders_archive", SQL: "UPDATE secrets SET value = 1"}, + {SessionID: 3, ElapsedMs: 1, Database: "billing", SQL: "select pg_sleep(5)"}, + }}) + Expect(err).NotTo(HaveOccurred()) + Expect(sessions(out.rows)).To(Equal(want)) + }, + Entry("every database when unset", Options{}, []int64{1, 2, 3}), + Entry("a wildcard database", Options{Databases: []string{"orders*"}}, []int64{1, 2}), + Entry("an excluded database", Options{Databases: []string{"!billing"}}, []int64{1, 2}), + Entry("a bare text value matches a substring, ignoring case", Options{Text: []string{"PG_SLEEP"}}, []int64{3}), + Entry("an excluded text value", Options{Text: []string{"!secrets"}}, []int64{1, 3}), + Entry("database and text together", Options{Databases: []string{"orders*"}, Text: []string{"ledger"}}, []int64{1}), + ) + It("asks the engine for idle sessions only when the options do", func() { + _, err := pollOnce(Options{Idle: true}, snapshot{ServerTime: serverTime}) + Expect(err).NotTo(HaveOccurred()) + }) + It("records an idle server as a poll with no rows", func() { + out, err := pollOnce(Options{}, snapshot{ServerTime: serverTime}) + Expect(err).NotTo(HaveOccurred()) + Expect(out.rows).To(BeEmpty()) + Expect(out.report).To(HaveKeyWithValue("polls", 1)) + Expect(out.frames).To(ConsistOf(HaveField("Observed", 0))) + }) + DescribeTable("refuses a snapshot it cannot trust", func(result snapshot, message string) { + out, err := pollOnce(Options{}, result) + Expect(err).To(MatchError(ContainSubstring(message))) + Expect(out.rows).To(BeEmpty()) + }, + Entry("no server time", snapshot{Rows: []Sample{{SessionID: 1}}}, "no server timestamp"), + Entry("one request twice", snapshot{ServerTime: serverTime, Rows: []Sample{{SessionID: 4, RequestID: 1}, {SessionID: 4, RequestID: 1}}}, "session 4 request 1 twice"), + ) + It("keeps two requests of one session apart", func() { + out, err := pollOnce(Options{}, snapshot{ServerTime: serverTime, Rows: []Sample{{SessionID: 4, RequestID: 0}, {SessionID: 4, RequestID: 1}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(out.rows).To(HaveLen(2)) + }) + It("surfaces a failed read with the engine it read", func() { + c, err := newCapture(Options{}, EngineSQLServer, func(context.Context, bool) (snapshot, error) { + return snapshot{}, errors.New("login failed") + }) + Expect(err).NotTo(HaveOccurred()) + Expect(c.poll(context.Background(), &recorder{})).To(MatchError("sample sqlserver requests: login failed")) + }) +}) diff --git a/pkg/tracing/sqltop/kind.go b/pkg/tracing/sqltop/kind.go new file mode 100644 index 0000000..24180a9 --- /dev/null +++ b/pkg/tracing/sqltop/kind.go @@ -0,0 +1,230 @@ +package sqltop + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/flanksource/clicky" + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/models" + "github.com/flanksource/commons-db/query/sqlactivity" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/sqlconn" + "github.com/flanksource/telltale/pkg/tracing/target" +) + +const ( + defaultPoll = 5 * time.Second + defaultTop = 50 +) + +type Params struct { + Target target.Target `json:"target,omitzero"` + Options +} + +func (p Params) Validate() error { + if p.Target.Seam != target.Agent { + return p.Options.Validate() + } + if p.Connection != "" || p.Idle { + return fmt.Errorf("connection and idle configure the database seam, not the agent seam") + } + if _, err := p.pollInterval(); err != nil { + return err + } + if _, err := p.minElapsed(); err != nil { + return err + } + if p.Top < 0 { + return fmt.Errorf("top must be non-negative") + } + if err := validatePatterns("databases", p.Databases); err != nil { + return err + } + return validatePatterns("text", p.Text) +} + +// Options are the connection whose running requests a capture samples, which +// of them it keeps, and how often and how many it records. +type Options struct { + Connection string `json:"connection,omitempty" clicky:"title=Connection,desc=Postgres or SQL Server connection reference for the database seam"` + Databases clicky.MultiFilter `json:"databases,omitempty" clicky:"title=Databases,desc=Database patterns with * wildcards and ! exclusions; empty keeps every database"` + Text clicky.MultiFilter `json:"text,omitempty" clicky:"title=Statement text,desc=SQL text patterns with * wildcards and ! exclusions; bare values match substrings"` + Poll string `json:"poll,omitempty" clicky:"title=Poll,desc=Poll interval such as 5s"` + Top int `json:"top,omitempty" clicky:"title=Top,desc=Maximum requests per poll ranked by elapsed time; defaults to 50"` + MinElapsed string `json:"minElapsed,omitempty" clicky:"title=Minimum elapsed,desc=Only record requests running at least this long such as 500ms"` + Idle bool `json:"idle,omitempty" clicky:"title=Idle sessions,desc=Also record idle sessions; sessions holding an open transaction are always recorded"` +} + +func (o Options) pollInterval() (time.Duration, error) { + return positiveDuration("poll", o.Poll, defaultPoll) +} + +func (o Options) minElapsed() (time.Duration, error) { + if o.MinElapsed == "" { + return 0, nil + } + return positiveDuration("minElapsed", o.MinElapsed, 0) +} + +func (o Options) top() int { + if o.Top == 0 { + return defaultTop + } + return o.Top +} + +func positiveDuration(name, value string, empty time.Duration) (time.Duration, error) { + if value == "" { + return empty, nil + } + d, err := time.ParseDuration(value) + if err != nil || d <= 0 { + return 0, fmt.Errorf("%s %q must be a positive duration", name, value) + } + return d, nil +} + +// Validate is the static check of the options: it opens no connection. +func (o Options) Validate() error { + if strings.TrimSpace(o.Connection) == "" { + return fmt.Errorf("connection is required") + } + if _, err := o.pollInterval(); err != nil { + return err + } + if _, err := o.minElapsed(); err != nil { + return err + } + if o.Top < 0 { + return fmt.Errorf("top must be non-negative") + } + if err := validatePatterns("databases", o.Databases); err != nil { + return err + } + return validatePatterns("text", o.Text) +} + +type handler struct{} +type captureKey struct{} + +func (handler) Params() Params { + return Params{Options: Options{Poll: defaultPoll.String(), Top: defaultTop}} +} + +// readers are the engine readers by connection type. +var engines = map[string]string{ + models.ConnectionTypePostgres: EnginePostgres, + models.ConnectionTypeSQLServer: EngineSQLServer, +} + +func (handler) Prepare(ctx dbcontext.Context, p Params, out tracing.Emitter[Sample]) (dbcontext.Context, func() error, error) { + if err := p.Validate(); err != nil { + return ctx, nil, err + } + backend, err := tracing.CaptureBackend(ctx) + if err != nil { + return ctx, nil, err + } + store, err := newActiveStore(backend, tracing.StreamOf(ctx)) + if err != nil { + return ctx, nil, err + } + ctx = ctx.WithValue(activeStoreKey{}, store) + if p.Target.Seam == target.Agent { + return agentCapture(p).Prepare(ctx, p, out) + } + out = activeEmitter{Emitter: out, store: store} + pool, connType, release, err := sqlconn.Open(ctx, sqlconn.OpenOptions{Reference: p.Connection, + Types: []string{models.ConnectionTypePostgres, models.ConnectionTypeSQLServer}, Purpose: "SQL top"}) + if err != nil { + return ctx, nil, err + } + engine := engines[connType] + c, err := newCapture(p.Options, engine, func(ctx context.Context, idle bool) (snapshot, error) { + return sqlactivity.Read(ctx, pool, engine, sqlactivity.Filter{Idle: idle}) + }) + if err != nil { + release() + return ctx, nil, err + } + if err := c.poll(ctx, out); err != nil { + release() + return ctx, nil, err + } + out.Describe(fmt.Sprintf("%s running requests on %s", engine, p.Connection)) + return ctx.WithValue(captureKey{}, c), func() error { release(); return nil }, nil +} + +func (handler) Handle(ctx dbcontext.Context, p Params, out tracing.Emitter[Sample], _ tracing.Records[Sample]) error { + if p.Target.Seam == target.Agent { + return agentCapture(p).Handle(ctx, p, out) + } + store, ok := ctx.Value(activeStoreKey{}).(*activeStore) + if !ok { + return fmt.Errorf("SQL top active store was not prepared") + } + out = activeEmitter{Emitter: out, store: store} + c, err := preparedCapture(ctx) + if err != nil { + return err + } + interval, err := p.pollInterval() + if err != nil { + return err + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + final, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + return c.poll(final, out) + case <-ticker.C: + if err := c.poll(ctx, out); err != nil { + if ctx.Err() != nil { + continue + } + return err + } + } + } +} + +func (handler) Checkpoint(ctx dbcontext.Context, p Params, out tracing.Emitter[Sample]) error { + var params struct { + Reason string `json:"reason,omitempty"` + } + if _, err := tracing.DecodeCheckpointParams(ctx, ¶ms); err != nil { + return err + } + if p.Target.Seam == target.Agent { + return agentCapture(p).Checkpoint(ctx, nil, out) + } + c, err := preparedCapture(ctx) + if err != nil { + return err + } + store, ok := ctx.Value(activeStoreKey{}).(*activeStore) + if !ok { + return fmt.Errorf("SQL top active store was not prepared") + } + return c.poll(ctx, activeEmitter{Emitter: out, store: store}) +} + +func preparedCapture(ctx dbcontext.Context) (*capture, error) { + c, ok := ctx.Value(captureKey{}).(*capture) + if !ok { + return nil, fmt.Errorf("sql top capture was not prepared") + } + return c, nil +} + +func Kind() *tracing.Handler[Params, Sample] { + return tracing.NewHandler[Params, Sample](handler{}, tracing.Capabilities{Follow: true}, target.Connection, target.Agent). + WithResults(tracing.NewResultTable("active", ActiveResult())).WithTop(topSpec()) +} diff --git a/pkg/tracing/sqltop/live_test.go b/pkg/tracing/sqltop/live_test.go new file mode 100644 index 0000000..d0aa09a --- /dev/null +++ b/pkg/tracing/sqltop/live_test.go @@ -0,0 +1,55 @@ +package sqltop + +import ( + "context" + "encoding/json" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/tracingtest" + . "github.com/onsi/gomega" +) + +func prepareLiveCapture(ctx dbcontext.Context, params Params, out *recorder) (dbcontext.Context, func() error, error) { + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{"sql_top": Kind()}) + raw, err := json.Marshal(params) + if err != nil { + return ctx, nil, err + } + managed, err := env.Runtime.Start(ctx, tracing.StartRequest{Kind: "sql_top", Params: raw}) + if err != nil { + return ctx, nil, err + } + info := managed.Session().Snapshot() + err = env.Results.Backend.Scan(ctx, info.Events.Stream, 0, func(_ int64, row recordstore.Row) error { + data, err := json.Marshal(row) + if err != nil { + return err + } + var sample Sample + if err := json.Unmarshal(data, &sample); err != nil { + return err + } + out.rows = append(out.rows, sample) + return nil + }) + if err != nil { + return ctx, nil, err + } + Expect(env.Read("sql_top_active", info.Events.Stream+"-active", query.PageRequest{Limit: 100}, nil)).To(HaveLen(len(out.rows))) + stop, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if _, err := managed.Stop(stop); err != nil { + return ctx, nil, err + } + out.report = tracingtest.Summary(managed.Session().Snapshot()).Report + return ctx, func() error { + stop, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err := managed.Stop(stop) + return err + }, nil +} diff --git a/pkg/tracing/sqltop/postgres_live_spec_test.go b/pkg/tracing/sqltop/postgres_live_spec_test.go new file mode 100644 index 0000000..9618e7c --- /dev/null +++ b/pkg/tracing/sqltop/postgres_live_spec_test.go @@ -0,0 +1,96 @@ +package sqltop + +import ( + "context" + "database/sql" + "fmt" + "time" + + "github.com/flanksource/clicky" + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/dbtest" + "github.com/flanksource/commons-db/models" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// backend is one connection of the live spec, with the pid Postgres gave it. +type backend struct { + conn *sql.Conn + pid int64 +} + +func openBackend(ctx context.Context, pool *sql.DB) backend { + conn, err := pool.Conn(ctx) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(conn.Close) + var pid int64 + Expect(conn.QueryRowContext(ctx, "SELECT pg_backend_pid()").Scan(&pid)).To(Succeed()) + return backend{conn: conn, pid: pid} +} + +func sampleOf(rows []Sample, pid int64) Sample { + for _, row := range rows { + if row.SessionID == pid { + return row + } + } + Fail(fmt.Sprintf("no sample of backend %d in %+v", pid, rows)) + return Sample{} +} + +var _ = Describe("sql top on a live Postgres", func() { + It("records a sleeping query, a lock waiter with its blocker, and the blocker idle in its transaction", func(spec SpecContext) { + db := dbtest.ForGinkgo(dbtest.Options{Name: "sqltop"}) + pool := db.SQL() + var database string + Expect(pool.QueryRowContext(spec, "SELECT current_database()").Scan(&database)).To(Succeed()) + _, err := pool.ExecContext(spec, "CREATE TABLE ledger (id int PRIMARY KEY, value int); INSERT INTO ledger VALUES (1, 0)") + Expect(err).NotTo(HaveOccurred()) + + holder, waiter, sleeper := openBackend(spec, pool), openBackend(spec, pool), openBackend(spec, pool) + _, err = holder.conn.ExecContext(spec, "BEGIN; UPDATE ledger SET value = value + 1 WHERE id = 1") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _, _ = holder.conn.ExecContext(context.Background(), "ROLLBACK") }) + background, cancel := context.WithCancel(context.Background()) + DeferCleanup(cancel) + go func() { _, _ = waiter.conn.ExecContext(background, "UPDATE ledger SET value = value + 1 WHERE id = 1") }() + go func() { _, _ = sleeper.conn.ExecContext(background, "SELECT pg_sleep(30)") }() + Eventually(func() int { + var waiting int + Expect(pool.QueryRowContext(spec, "SELECT count(*) FROM pg_stat_activity WHERE pid IN ($1, $2) AND wait_event IN ('transactionid', 'PgSleep')", waiter.pid, sleeper.pid).Scan(&waiting)).To(Succeed()) + return waiting + }).WithTimeout(10 * time.Second).Should(Equal(2)) + + ctx := dbcontext.New().WithConnectionResolver(func(dbcontext.Context, string) (*models.Connection, error) { + return &models.Connection{Name: "live", Type: models.ConnectionTypePostgres, URL: db.DSN()}, nil + }) + out := &recorder{} + _, release, err := prepareLiveCapture(ctx, Params{Options: Options{Connection: "connection://live", Databases: clicky.MultiFilter{database}}}, out) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(release) + + Expect(out.rows).To(HaveLen(3)) + for _, row := range out.rows { + Expect(row.SQL).NotTo(ContainSubstring("commons-db:sqlactivity"), "the poller recorded its own backend") + Expect(row.Engine).To(Equal(EnginePostgres)) + Expect(row.Database).To(Equal(database)) + } + sleeping := sampleOf(out.rows, sleeper.pid) + Expect(sleeping.State).To(Equal("active")) + Expect(sleeping.WaitType).To(Equal("Timeout:PgSleep")) + Expect(sleeping.Command).To(Equal("SELECT")) + Expect(sleeping.SQL).To(Equal("SELECT pg_sleep(30)")) + Expect(sleeping.ElapsedMs).To(BeNumerically(">", 0)) + blocked := sampleOf(out.rows, waiter.pid) + Expect(blocked.Blocked).To(BeTrue()) + Expect(blocked.BlockedBy).To(Equal(holder.pid)) + Expect(blocked.WaitType).To(Equal("Lock:transactionid")) + Expect(blocked.OpenTransactions).To(Equal(int64(1))) + blocker := sampleOf(out.rows, holder.pid) + Expect(blocker.State).To(Equal("idle in transaction")) + Expect(blocker.Blocked).To(BeFalse()) + Expect(blocker.TransactionMs).To(BeNumerically(">", 0)) + Expect(out.report).To(HaveKeyWithValue("blocked", BeNumerically("==", 1))) + }) +}) diff --git a/pkg/tracing/sqltop/profile.go b/pkg/tracing/sqltop/profile.go new file mode 100644 index 0000000..9244d64 --- /dev/null +++ b/pkg/tracing/sqltop/profile.go @@ -0,0 +1,55 @@ +package sqltop + +import ( + "encoding/json" + "fmt" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/telltale/pkg/otlp/profiles" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/target" +) + +func FromSnapshot(snapshot profiles.Snapshot) ([]ActiveSample, error) { + if snapshot.Kind != "sql_top" { + return nil, fmt.Errorf("SQL top cannot project profile kind %q", snapshot.Kind) + } + rows := make([]ActiveSample, len(snapshot.Rows)) + for i, observation := range snapshot.Rows { + data, err := json.Marshal(observation.Attributes) + if err != nil { + return nil, err + } + var row ActiveSample + if err := json.Unmarshal(data, &row); err != nil { + return nil, fmt.Errorf("decode SQL profile attributes: %w", err) + } + if row.StartedAt.IsZero() || (row.API != "jdbc" && row.API != "r2dbc") || (row.SQL == "" && row.SQLUnavailableReason == "") { + return nil, fmt.Errorf("SQL profile requires startedAt, jdbc/r2dbc api and SQL text or unavailable reason") + } + if row.SQL != "" && row.SQLUnavailableReason != "" { + return nil, fmt.Errorf("unavailable SQL profile must not contain invented SQL text") + } + row.Source, row.OperationID, row.SampledAt = snapshot.Frame().Source, observation.OperationID, observation.SampledAt + row.ElapsedMs = float64(observation.ElapsedNS) / float64(time.Millisecond) + row.TraceID, row.SpanID = observation.TraceID, observation.SpanID + rows[i] = row + } + return rows, nil +} + +type activeReader struct{} + +func (activeReader) Params() Params { return Params{Target: target.Target{Seam: target.Agent}} } +func (activeReader) Schema() recordresults.ResultType[ActiveSample] { return ActiveResult() } +func (activeReader) Handle(dbcontext.Context, Params, tracing.Emitter[ActiveSample], tracing.Records[ActiveSample]) error { + return fmt.Errorf("received SQL active results cannot start captures") +} + +func NewActiveKind() *tracing.Handler[Params, ActiveSample] { + spec := topSpec() + spec.Result = "" + return tracing.NewHandler[Params, ActiveSample](activeReader{}, tracing.Capabilities{Follow: true}, target.Agent).WithTop(spec) +} diff --git a/pkg/tracing/sqltop/profile_spec_test.go b/pkg/tracing/sqltop/profile_spec_test.go new file mode 100644 index 0000000..ec9a722 --- /dev/null +++ b/pkg/tracing/sqltop/profile_spec_test.go @@ -0,0 +1,85 @@ +package sqltop + +import ( + "context" + "encoding/json" + "time" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/otlp/profiles" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/tracingtest" + "github.com/flanksource/telltale/sdk/protocol" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("SQL top JVM profile projection", func() { + DescribeTable("requires authenticated agent identities before decoding profiles", func(capture, source, reason string) { + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{"sql_top": Kind()}) + store, err := newActiveStore(env.Results.Backend, "sql-profile-source") + Expect(err).NotTo(HaveOccurred()) + ctx := context.WithValue(context.Background(), activeStoreKey{}, store) + data, err := profiles.Encode([]profiles.Snapshot{{Kind: "sql_top", CaptureID: "capture", Source: "java-instance", ProfileID: "0123456789abcdef0123456789abcdef", Sequence: 1, StartedAt: serverTime, CompletedAt: serverTime.Add(time.Second), Limit: 50, Complete: true}}) + Expect(err).NotTo(HaveOccurred()) + err = agentCapture(Params{}).HandleBlob(ctx, agentclient.Blob{Info: protocol.Blob{ContentType: profiles.ContentType, Session: capture}, Data: data}, map[string]string{"telltale.source.instance": source}, nil) + Expect(err).To(MatchError(ContainSubstring(reason))) + }, Entry("missing source", "capture", "", "agent Hello is missing telltale.source.instance"), Entry("missing capture", "", "java-instance", "missing the authenticated capture session")) + + It("preserves the producer limit instead of replacing it with the local default", func() { + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{"sql_top": Kind()}) + store, err := newActiveStore(env.Results.Backend, "sql-profile-controls") + Expect(err).NotTo(HaveOccurred()) + ctx := context.WithValue(context.Background(), activeStoreKey{}, store) + const producerLimit = 7 + data, err := profiles.Encode([]profiles.Snapshot{{Kind: "sql_top", CaptureID: "capture", Source: "java-instance", ProfileID: "0123456789abcdef0123456789abcdef", Sequence: 1, StartedAt: serverTime, CompletedAt: serverTime.Add(time.Second), Limit: producerLimit, Complete: true}}) + Expect(err).NotTo(HaveOccurred()) + Expect(agentCapture(Params{}).HandleBlob(ctx, agentclient.Blob{Info: protocol.Blob{ContentType: profiles.ContentType, Session: "capture"}, Data: data}, map[string]string{"telltale.source.instance": "java-instance"}, nil)).To(Succeed()) + frames := env.Read("sql_top_frames", "sql-profile-controls-frames", query.PageRequest{Limit: 10}, nil) + Expect(frames).To(HaveLen(1)) + Expect(frames[0]).To(HaveKeyWithValue("limit", BeNumerically("==", producerLimit))) + }) + + It("keeps producer times, identity, correlation and measured API lifetime", func() { + snapshot := profiles.Snapshot{Kind: "sql_top", CaptureID: "capture", Source: "java-instance", Rows: []profiles.Row{{OperationID: "execution", ElapsedNS: int64(2 * time.Millisecond), SampledAt: serverTime, TraceID: "trace", SpanID: "span", Attributes: map[string]any{"startedAt": serverTime.Add(-2 * time.Millisecond).Format(time.RFC3339Nano), "api": "r2dbc", "sql": "SELECT id FROM orders"}}}} + rows, err := FromSnapshot(snapshot) + Expect(err).NotTo(HaveOccurred()) + Expect(rows).To(Equal([]ActiveSample{{Source: "java-instance/capture", OperationID: "execution", StartedAt: serverTime.Add(-2 * time.Millisecond), SampledAt: serverTime, ElapsedMs: 2, API: "r2dbc", SQL: "SELECT id FROM orders", TraceID: "trace", SpanID: "span"}})) + }) + + It("retains missing SQL as a diagnostic and rejects silently absent metadata", func() { + snapshot := profiles.Snapshot{Kind: "sql_top", Rows: []profiles.Row{{Attributes: map[string]any{"startedAt": serverTime.Format(time.RFC3339), "api": "jdbc"}}}} + _, err := FromSnapshot(snapshot) + Expect(err).To(MatchError(ContainSubstring("SQL text or unavailable reason"))) + snapshot.Rows[0].Attributes["sqlUnavailableReason"] = "prepared-before-instrumentation" + rows, err := FromSnapshot(snapshot) + Expect(err).NotTo(HaveOccurred()) + Expect(rows[0].SQLUnavailableReason).To(Equal("prepared-before-instrumentation")) + }) + + It("preserves the active phase and bounded-query metadata", func() { + snapshot := profiles.Snapshot{Kind: "sql_top", Rows: []profiles.Row{{Attributes: map[string]any{"startedAt": serverTime.Format(time.RFC3339), "api": "r2dbc", "sql": "SELECT id FROM orders", "queryFingerprint": "normalized-query-hash", "sqlTruncated": true, "phase": "awaiting-consumption"}}}} + rows, err := FromSnapshot(snapshot) + Expect(err).NotTo(HaveOccurred()) + Expect(rows[0].Phase).To(Equal("awaiting-consumption")) + Expect(rows[0].QueryFingerprint).To(Equal("normalized-query-hash")) + Expect(rows[0].SQLTruncated).To(BeTrue()) + }) + + It("translates normal polling options to runtime capture milliseconds and limits", func() { + data, err := agentArmParams(Params{Options: Options{Poll: "250ms", Top: 12}}) + Expect(err).NotTo(HaveOccurred()) + var params map[string]any + Expect(json.Unmarshal(data, ¶ms)).To(Succeed()) + Expect(params).To(Equal(map[string]any{"pollIntervalMs": float64(250), "limit": float64(12)})) + }) + + It("passes the parsed minimum elapsed duration to the Java boundary", func() { + data, err := agentArmParams(Params{Options: Options{MinElapsed: "100ms"}}) + Expect(err).NotTo(HaveOccurred()) + var params map[string]any + Expect(json.Unmarshal(data, ¶ms)).To(Succeed()) + Expect(params).To(HaveKeyWithValue("minElapsedNs", float64(100*time.Millisecond))) + }) +}) diff --git a/pkg/tracing/sqltop/sample.go b/pkg/tracing/sqltop/sample.go new file mode 100644 index 0000000..5341416 --- /dev/null +++ b/pkg/tracing/sqltop/sample.go @@ -0,0 +1,10 @@ +package sqltop + +import "github.com/flanksource/commons-db/query/sqlactivity" + +type Sample = sqlactivity.Sample + +const ( + EnginePostgres = sqlactivity.EnginePostgres + EngineSQLServer = sqlactivity.EngineSQLServer +) diff --git a/pkg/tracing/sqltop/schema.go b/pkg/tracing/sqltop/schema.go new file mode 100644 index 0000000..43f75ef --- /dev/null +++ b/pkg/tracing/sqltop/schema.go @@ -0,0 +1,16 @@ +package sqltop + +import ( + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/commons-db/recordstore/recordresults" + "maps" + "slices" +) + +func (handler) ConnectionTypes() []string { return slices.Sorted(maps.Keys(engines)) } + +func (handler) Schema() recordresults.ResultType[Sample] { + return recordresults.ResultType[Sample]{Title: "SQL running requests", TimeColumn: "sampledAt", Follow: true, + Indexes: []recordstore.IndexDef{{Columns: []string{"sessionId", "sampledAt"}}}, + SearchColumns: []string{"sql", "user", "application"}} +} diff --git a/pkg/tracing/sqltop/schema_spec_test.go b/pkg/tracing/sqltop/schema_spec_test.go new file mode 100644 index 0000000..2a3c333 --- /dev/null +++ b/pkg/tracing/sqltop/schema_spec_test.go @@ -0,0 +1,77 @@ +package sqltop + +import ( + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/query" + _ "github.com/flanksource/commons-db/query/providers/sqlite" + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/target" + "github.com/flanksource/telltale/pkg/tracing/tracingtest" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type sampleFixture struct { + handler + rows []Sample +} + +func (sampleFixture) Prepare(ctx dbcontext.Context, _ Params, _ tracing.Emitter[Sample]) (dbcontext.Context, func() error, error) { + return ctx, func() error { return nil }, nil +} + +func (f sampleFixture) Handle(ctx dbcontext.Context, _ Params, out tracing.Emitter[Sample], _ tracing.Records[Sample]) error { + for _, row := range f.rows { + if err := out.Emit(ctx, row); err != nil { + return err + } + } + return nil +} + +// filterKinds are the filter every column of a sample offers. +var filterKinds = map[string]query.ColumnFilterKind{ + "sampledAt": "time", "engine": "terms", "database": "terms", "sessionId": "exact", "requestId": "exact", + "blockedBy": "exact", "blocked": "boolean", "user": "terms", "application": "terms", "clientHost": "terms", + "state": "terms", "command": "terms", "waitType": "terms", "waitResource": "exact", "waitMs": "duration", + "startedAt": "time", "transactionStartedAt": "time", "elapsedMs": "duration", "transactionMs": "duration", + "cpuMs": "duration", "logicalReads": "range", "reads": "range", "writes": "range", "memoryBytes": "range", + "openTransactions": "range", "queryId": "exact", "sql": "text", +} + +var _ = Describe("sql top result schema", func() { + It("offers a filter on every column", func() { + catalog := tracing.NewKinds() + Expect(catalog.RegisterKind("sql_top", Kind())).To(Succeed()) + infos, err := catalog.List() + Expect(err).NotTo(HaveOccurred()) + Expect(infos).To(HaveLen(1)) + got := map[string]query.ColumnFilterKind{} + for _, column := range infos[0].Columns { + Expect(column.Filter).NotTo(BeNil(), "column %s offers no filter", column.Name) + got[column.Name] = column.Filter.Kind + } + Expect(got).To(Equal(filterKinds)) + }) + It("appends a session's repeated samples and filters them by blocked and elapsed", func() { + schema := (handler{}).Schema() + Expect(schema.TimeColumn).To(Equal("sampledAt")) + Expect(schema.Follow).To(BeTrue()) + Expect(schema.Indexes).To(ContainElement(recordstore.IndexDef{Columns: []string{"sessionId", "sampledAt"}})) + Expect(schema.SearchColumns).To(ConsistOf("sql", "user", "application")) + rows := []Sample{ + {SampledAt: serverTime, Engine: EnginePostgres, SessionID: 7, ElapsedMs: 900, BlockedBy: 9, Blocked: true, SQL: "UPDATE orders"}, + {SampledAt: serverTime.Add(5e9), Engine: EnginePostgres, SessionID: 7, ElapsedMs: 5900, BlockedBy: 9, Blocked: true, SQL: "UPDATE orders"}, + {SampledAt: serverTime, Engine: EnginePostgres, SessionID: 9, ElapsedMs: 50, SQL: "SELECT 1"}, + } + plugin := tracing.NewHandler[Params, Sample](sampleFixture{rows: rows}, tracing.Capabilities{Follow: true}, target.Connection) + env := tracingtest.NewEnv(map[string]tracing.TracePlugin{"sql_top": plugin}) + info := tracingtest.Ended(env.Start("sql_top", `{"connection":"connection://example"}`)) + Expect(info.Error).To(BeEmpty()) + Expect(env.Rows(info)).To(HaveLen(3)) + blocked := env.Read("sql_top", info.Events.Stream, query.PageRequest{Limit: 20}, map[string]any{"filter.blocked": "true", "filter.elapsedMs": ">1s"}) + Expect(blocked).To(HaveLen(1)) + Expect(blocked[0]).To(HaveKeyWithValue("elapsedMs", BeNumerically("==", 5900))) + }) +}) diff --git a/pkg/tracing/sqltop/sqlserver_live_spec_test.go b/pkg/tracing/sqltop/sqlserver_live_spec_test.go new file mode 100644 index 0000000..cdc54ed --- /dev/null +++ b/pkg/tracing/sqltop/sqlserver_live_spec_test.go @@ -0,0 +1,85 @@ +package sqltop + +import ( + "context" + "database/sql" + "fmt" + "os" + "strings" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/models" + "github.com/flanksource/telltale/pkg/tracing/sqlconn" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func openSession(ctx context.Context, pool *sql.DB) backend { + conn, err := pool.Conn(ctx) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(conn.Close) + var spid int64 + Expect(conn.QueryRowContext(ctx, "SELECT CAST(@@SPID AS bigint)").Scan(&spid)).To(Succeed()) + return backend{conn: conn, pid: spid} +} + +var _ = Describe("sql top on a live SQL Server", func() { + It("records a waiting batch, a lock waiter with its blocker, and the sleeping blocker holding its transaction", func(spec SpecContext) { + address := strings.TrimSpace(os.Getenv("COMMONS_DB_SQLSERVER_URL")) + if address == "" { + Skip("COMMONS_DB_SQLSERVER_URL is not set; this fixture creates and drops one table") + } + ctx := dbcontext.New().WithConnectionResolver(func(dbcontext.Context, string) (*models.Connection, error) { + return &models.Connection{Name: "live", Type: models.ConnectionTypeSQLServer, URL: address}, nil + }) + pool, _, release, err := sqlconn.Open(ctx, sqlconn.OpenOptions{Reference: "connection://live", Types: []string{models.ConnectionTypeSQLServer}, Purpose: "the live spec"}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(release) + table := fmt.Sprintf("dbo.telltale_sqltop_%d", time.Now().UnixNano()) + _, err = pool.ExecContext(spec, "CREATE TABLE "+table+" (id int PRIMARY KEY, value int); INSERT INTO "+table+" VALUES (1, 0)") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _, dropErr := pool.Exec("DROP TABLE " + table); Expect(dropErr).NotTo(HaveOccurred()) }) + + holder, waiter, sleeper := openSession(spec, pool), openSession(spec, pool), openSession(spec, pool) + _, err = holder.conn.ExecContext(spec, "BEGIN TRAN; UPDATE "+table+" SET value = value + 1 WHERE id = 1") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _, _ = holder.conn.ExecContext(context.Background(), "IF @@TRANCOUNT > 0 ROLLBACK") }) + background, cancel := context.WithCancel(context.Background()) + DeferCleanup(cancel) + go func() { + _, _ = waiter.conn.ExecContext(background, "UPDATE "+table+" SET value = value + 1 WHERE id = 1") + }() + go func() { _, _ = sleeper.conn.ExecContext(background, "WAITFOR DELAY '00:00:30'") }() + Eventually(func() int { + var waiting int + Expect(pool.QueryRowContext(spec, "SELECT COUNT(*) FROM sys.dm_exec_requests WHERE session_id IN (@p1, @p2) AND wait_type IN ('LCK_M_U', 'LCK_M_X', 'WAITFOR')", + waiter.pid, sleeper.pid).Scan(&waiting)).To(Succeed()) + return waiting + }).WithTimeout(10 * time.Second).Should(Equal(2)) + + out := &recorder{} + _, stop, err := prepareLiveCapture(ctx, Params{Options: Options{Connection: "connection://live"}}, out) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(stop) + + for _, row := range out.rows { + Expect(row.SQL).NotTo(ContainSubstring("commons-db:sqlactivity"), "the poller recorded its own session") + Expect(row.Engine).To(Equal(EngineSQLServer)) + } + sleeping := sampleOf(out.rows, sleeper.pid) + Expect(sleeping.WaitType).To(Equal("WAITFOR")) + Expect(sleeping.SQL).To(ContainSubstring("WAITFOR DELAY")) + Expect(sleeping.ElapsedMs).To(BeNumerically(">", 0)) + blocked := sampleOf(out.rows, waiter.pid) + Expect(blocked.Blocked).To(BeTrue()) + Expect(blocked.BlockedBy).To(Equal(holder.pid)) + Expect(blocked.WaitType).To(HavePrefix("LCK_M_")) + Expect(blocked.WaitResource).NotTo(BeEmpty()) + Expect(blocked.QueryID).To(HavePrefix("0x")) + blocker := sampleOf(out.rows, holder.pid) + Expect(blocker.State).To(Equal("sleeping")) + Expect(blocker.OpenTransactions).To(BeNumerically(">=", 1)) + Expect(blocker.TransactionMs).To(BeNumerically(">", 0)) + }) +}) diff --git a/pkg/tracing/sqltop/sqltop_suite_test.go b/pkg/tracing/sqltop/sqltop_suite_test.go new file mode 100644 index 0000000..9dea151 --- /dev/null +++ b/pkg/tracing/sqltop/sqltop_suite_test.go @@ -0,0 +1,13 @@ +package sqltop + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestSQLTop(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "SQL top") +} diff --git a/pkg/tracing/sqltop/top.go b/pkg/tracing/sqltop/top.go new file mode 100644 index 0000000..8031291 --- /dev/null +++ b/pkg/tracing/sqltop/top.go @@ -0,0 +1,10 @@ +package sqltop + +import ( + "github.com/flanksource/commons-db/query" + "github.com/flanksource/telltale/pkg/tracing" +) + +func topSpec() tracing.TopSpec { + return tracing.TopSpec{Result: "active", PerSource: true, Relation: &tracing.TopRelation{Column: "blockedByOperationId", Target: "operationId", Label: "Blocker"}, Identity: []string{"source", "operationId"}, Order: query.Order{{Column: "elapsedMs", Desc: true}, {Column: "operationId"}}} +} diff --git a/pkg/tracing/stacks/checkpoint.go b/pkg/tracing/stacks/checkpoint.go index 6777e93..829b290 100644 --- a/pkg/tracing/stacks/checkpoint.go +++ b/pkg/tracing/stacks/checkpoint.go @@ -13,6 +13,7 @@ import ( "github.com/flanksource/telltale/pkg/tracing/target" dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/recordstore/sampleframe" ) // Checkpoint dumps the stacks its params ask for (CheckpointParams), the @@ -40,12 +41,14 @@ func (s stacks) Checkpoint(ctx dbcontext.Context, params Params, records tracing if err != nil { return err } - for _, record := range dump { - if err := records.Emit(ctx, record); err != nil { - return err + at := time.Now().UTC() + frame := sampleframe.Frame{Source: fmt.Sprintf("%s/%d", head.Key, head.PID), StartedAt: at, CompletedAt: at, Observed: len(dump)} + for _, row := range dump { + if row.Error != "" { + frame.Error = row.Error } } - return nil + return tracing.PublishFrame(ctx, records, frame, dump) } // take is the records of the dump request asks for, each a copy of head diff --git a/pkg/tracing/stacks/pretty.go b/pkg/tracing/stacks/pretty.go new file mode 100644 index 0000000..809db29 --- /dev/null +++ b/pkg/tracing/stacks/pretty.go @@ -0,0 +1,45 @@ +package stacks + +import ( + "fmt" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" +) + +func (r Stack) Pretty() api.Text { + t := clicky.Text(fmt.Sprintf("pid %d", r.PID), "text-muted") + if r.GoroutineID > 0 { + t = t.Space().Append(fmt.Sprintf("goroutine %d", r.GoroutineID), "text-blue-500") + } + t = t.Space().Append(r.State, "text-yellow-500") + if len(r.Frames) > 0 { + t = t.Space().Append(r.Frames[0].Func, "font-mono") + } + if r.Error != "" { + t = t.Space().Append(r.Error, "text-red-500") + } + return t +} + +func (r Stack) PrettyFull() api.Textable { + t := r.Pretty() + for _, frame := range r.Frames { + t = t.NewLine().Add(frame.Pretty()) + } + if r.Raw != "" { + t = t.NewLine().Add(clicky.CodeBlock("text", r.Raw)) + } + return t +} + +func (f Frame) Pretty() api.Text { + t := clicky.Text(f.Func, "font-mono") + if f.File != "" { + t = t.Space().Append(f.File, "text-muted") + if f.Line > 0 { + t = t.Append(fmt.Sprintf(":%d", f.Line), "text-muted") + } + } + return t +} diff --git a/pkg/tracing/stacks/stacks.go b/pkg/tracing/stacks/stacks.go index f44e019..b20af3a 100644 --- a/pkg/tracing/stacks/stacks.go +++ b/pkg/tracing/stacks/stacks.go @@ -17,6 +17,7 @@ import ( dbcontext "github.com/flanksource/commons-db/context" "github.com/flanksource/commons-db/recordstore" "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/commons-db/recordstore/sampleframe" "github.com/flanksource/telltale/pkg/agentclient" "github.com/flanksource/telltale/pkg/tracing" "github.com/flanksource/telltale/pkg/tracing/seam" @@ -166,24 +167,26 @@ func NewKind(registry *agentclient.Registry, dumpers *Dumpers) *tracing.Handler[ if dumpers == nil { panic("stacks: NewKind needs the host's Dumpers") } - agent := seam.AgentSeam[Params, Stack]{Probe: Kind, Registry: registry, Target: func(p Params) target.Target { return p.Target }, DecodeBlob: func(blob agentclient.Blob, resource map[string]string) ([]Stack, error) { + agent := seam.AgentSeam[Params, Stack]{Probe: Kind, Registry: registry, Target: func(p Params) target.Target { return p.Target }, DecodeBlob: func(blob agentclient.Blob, resource map[string]string) (seam.DecodedBlob[Stack], error) { + started := time.Now().UTC() if blob.Info.Name != "stacks" || blob.Info.ContentType != "text/plain" { - return nil, fmt.Errorf("unexpected stacks blob %s (%s)", blob.Info.Name, blob.Info.ContentType) + return seam.DecodedBlob[Stack]{}, fmt.Errorf("unexpected stacks blob %s (%s)", blob.Info.Name, blob.Info.ContentType) } pid, err := strconv.Atoi(resource["process.pid"]) if err != nil { - return nil, fmt.Errorf("agent process pid: %w", err) + return seam.DecodedBlob[Stack]{}, fmt.Errorf("agent process pid: %w", err) } var request CheckpointParams if len(blob.CheckpointParams) > 0 { if err := json.Unmarshal(blob.CheckpointParams, &request); err != nil { - return nil, fmt.Errorf("stacks checkpoint params: %w", err) + return seam.DecodedBlob[Stack]{}, fmt.Errorf("stacks checkpoint params: %w", err) } } - return Records(Stack{CheckpointID: newCheckpointID(), Reason: request.Reason, PID: pid, Transport: "agent"}, Dump{Format: FormatGo, Text: string(blob.Data), CollectedAt: time.Now().UTC()}), nil + rows := Records(Stack{CheckpointID: newCheckpointID(), Reason: request.Reason, PID: pid, Transport: "agent"}, Dump{Format: FormatGo, Text: string(blob.Data), CollectedAt: time.Now().UTC()}) + return seam.DecodedBlob[Stack]{Rows: rows, Frame: &sampleframe.Frame{Source: resource["telltale.agent.id"] + "/" + resource["process.pid"] + "/" + blob.Info.Name, StartedAt: started, CompletedAt: time.Now().UTC(), Observed: len(rows)}}, nil }} return tracing.NewHandler[Params, Stack](stacks{dumpers: dumpers, agent: agent}, tracing.Capabilities{Live: true}, target.Process, target.Agent, target.Self). - WithTruncation(maxRawBytes) + WithTruncation(maxRawBytes).WithTop(topSpec()) } func newCheckpointID() string { return uuid.NewString() } diff --git a/pkg/tracing/stacks/table.go b/pkg/tracing/stacks/table.go new file mode 100644 index 0000000..a809b66 --- /dev/null +++ b/pkg/tracing/stacks/table.go @@ -0,0 +1,17 @@ +package stacks + +import ( + "github.com/flanksource/clicky/api" + "github.com/flanksource/commons-db/query" + "github.com/flanksource/telltale/pkg/tracing" +) + +func (Stack) Columns() []api.ColumnDef { + return query.TableColumnsFor[Stack]("pid", "goroutineId", "waitMinutes", "state", "frames", "error") +} +func (s Stack) Row() map[string]any { return query.TableRowFor(s) } +func (s Stack) RowDetail() api.Textable { return s.PrettyFull() } + +func topSpec() tracing.TopSpec { + return tracing.TopSpec{Identity: []string{"key", "goroutineId"}, GroupBy: []string{"key"}, Order: query.Order{{Column: "waitMinutes", Desc: true}, {Column: "goroutineId"}}} +} diff --git a/pkg/tracing/task/pretty.go b/pkg/tracing/task/pretty.go new file mode 100644 index 0000000..da29424 --- /dev/null +++ b/pkg/tracing/task/pretty.go @@ -0,0 +1,32 @@ +package task + +import ( + "fmt" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" +) + +var statusStyles = map[string]string{"success": "text-green-500", "failed": "text-red-500", "canceled": "text-yellow-500", "running": "text-blue-500"} + +func (r Snapshot) Pretty() api.Text { + t := clicky.Text(r.Phase, "text-muted").Space().Append(r.Name, "font-bold").Space().Append(r.Status, statusStyles[r.Status]+" font-bold") + if r.Group != "" { + t = t.Space().Append(r.Group, "text-muted") + } + if r.MaxValue > 0 { + t = t.Space().Append(fmt.Sprintf("%d/%d", r.Progress, r.MaxValue), "text-blue-500") + } + if r.Error != "" { + t = t.Space().Append(r.Error, "text-red-500") + } + return t +} + +func (r Snapshot) PrettyFull() api.Textable { + t := r.Pretty() + if r.Message != "" { + t = t.NewLine().Append(r.Message, "text-muted") + } + return t +} diff --git a/pkg/tracing/task/task.go b/pkg/tracing/task/task.go new file mode 100644 index 0000000..15c5c52 --- /dev/null +++ b/pkg/tracing/task/task.go @@ -0,0 +1,95 @@ +// Package task records the clicky tasks an agent runs, armed on the task +// probe github.com/flanksource/clicky/telltale registers. +package task + +import ( + "encoding/json" + "errors" + "fmt" + "iter" + "path/filepath" + + clickytask "github.com/flanksource/clicky/task" + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/telltale/pkg/agentclient" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/seam" + "github.com/flanksource/telltale/pkg/tracing/target" +) + +const Kind = "task" + +// Options choose the tasks a capture records and how often it keeps progress. +type Options struct { + Name string `json:"name,omitempty" clicky:"desc=Task name glob"` + Group string `json:"group,omitempty" clicky:"desc=Parent group name glob"` + ProgressIntervalMs int `json:"progressIntervalMs,omitempty" clicky:"desc=Minimum milliseconds between progress records of a task; 0 takes the agent's default"` +} + +// Params are a capture's Options in the agent its target names. +type Params struct { + Target target.Target `json:"target,omitzero"` + Options +} + +// Validate is the static check of the globs and the progress interval. +func (p Options) Validate() error { + for _, glob := range []string{p.Name, p.Group} { + if _, err := filepath.Match(glob, ""); err != nil { + return fmt.Errorf("glob %q: %w", glob, err) + } + } + if p.ProgressIntervalMs < 0 { + return errors.New("progressIntervalMs must not be negative") + } + return nil +} + +// Snapshot mirrors clicky/telltale's TaskRecord: a task at a status +// transition, a throttled progress update, or (phase baseline) a task already +// present when the capture armed. It carries no logs or output. +type Snapshot struct { + tracing.Truncation + Phase string `json:"phase"` + clickytask.TaskSnapshot + Resource map[string]string `json:"resource,omitempty"` +} + +type capture struct { + agent seam.AgentSeam[Params, Snapshot] +} + +func (c capture) AgentAvailable(probe string, t target.Target) (bool, error) { + return c.agent.Available(probe, t) +} +func (capture) Params() Params { return Params{} } +func (capture) Schema() recordresults.ResultType[Snapshot] { + return recordresults.ResultType[Snapshot]{Title: "Tasks", Follow: true} +} +func (c capture) Prepare(ctx dbcontext.Context, p Params, out tracing.Emitter[Snapshot]) (dbcontext.Context, func() error, error) { + return c.agent.Prepare(ctx, p, out) +} +func (c capture) Handle(ctx dbcontext.Context, p Params, out tracing.Emitter[Snapshot], _ tracing.Records[Snapshot]) error { + return c.agent.Handle(ctx, p, out) +} + +// Summarize counts the tasks seen and each one's last status. +func (capture) Summarize(rows iter.Seq[Snapshot]) (map[string]any, error) { + last := map[string]string{} + for row := range rows { + last[row.ID] = row.Status + } + statuses := map[string]int{} + for _, status := range last { + statuses[status]++ + } + return map[string]any{"tasks": len(last), "statuses": statuses}, nil +} + +// NewKind is the task kind, armed only through an agent: telltale itself +// registers no task probe. +func NewKind(registry *agentclient.Registry) *tracing.Handler[Params, Snapshot] { + agent := seam.AgentSeam[Params, Snapshot]{Probe: Kind, Registry: registry, Target: func(p Params) target.Target { return p.Target }, ArmParams: func(p Params) (json.RawMessage, error) { return json.Marshal(p) }} + return tracing.NewHandler[Params, Snapshot](capture{agent}, tracing.Capabilities{Live: true}, target.Agent).WithSecretMasking() +} diff --git a/pkg/tracing/terminal/pretty.go b/pkg/tracing/terminal/pretty.go new file mode 100644 index 0000000..e994c1b --- /dev/null +++ b/pkg/tracing/terminal/pretty.go @@ -0,0 +1,39 @@ +package terminal + +import ( + "fmt" + "strings" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" +) + +func (r Event) Pretty() api.Text { + t := clicky.Text(fmt.Sprintf("+%.3fs", r.Offset), "text-muted") + switch r.Kind { + case KindResize: + t = t.Space().Append("resize", "text-blue-500").Space().Append(fmt.Sprintf("%d × %d", r.Width, r.Height)) + case KindExit: + t = t.Space().Append(r.Command, "font-mono").Space().Append("exit", "text-muted") + if r.ExitCode != nil { + style := "text-green-500" + if *r.ExitCode != 0 { + style = "text-red-500" + } + t = t.Space().Append(*r.ExitCode, style) + } + case KindOutput, KindSnapshot: + t = t.Space().Append(r.Kind, "text-blue-500").Space().Append(strings.Join(strings.Fields(r.Data), " "), "max-w-[120ch]") + } + if r.Truncated { + t = t.Space().Append("(truncated)", "text-yellow-500") + } + return t +} + +func (r Event) PrettyFull() api.Textable { + if r.Data != "" { + return r.Pretty().NewLine().Append(r.Data) + } + return r.Pretty() +} diff --git a/pkg/tracing/top.go b/pkg/tracing/top.go new file mode 100644 index 0000000..5aacc3b --- /dev/null +++ b/pkg/tracing/top.go @@ -0,0 +1,170 @@ +package tracing + +import ( + "context" + "fmt" + "slices" + + "github.com/flanksource/commons-db/query" + "github.com/flanksource/commons-db/recordstore" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/commons-db/recordstore/sampleframe" + "github.com/flanksource/telltale/pkg/otlp/profilepublication" +) + +type TopMetric struct { + Key string `json:"key"` + Label string `json:"label"` + Unit string `json:"unit,omitempty"` + View string `json:"view"` +} + +type TopSpec struct { + Relation *TopRelation `json:"relation,omitempty"` + Hierarchy func([]query.Row, string) (*sampleframe.Hierarchy, error) `json:"-"` + Profile func(context.Context, *Runtime, TraceRef, sampleframe.Frame) (query.Profile, error) `json:"-"` + ProjectionColumn string `json:"projectionColumn,omitempty"` + PerSource bool `json:"perSource,omitempty"` + Result string `json:"result,omitempty"` + Stack *TopStack `json:"stack,omitempty"` + Identity []string `json:"identity"` + Order query.Order `json:"order"` + Metrics []TopMetric `json:"metrics,omitempty"` + GroupBy []string `json:"groupBy,omitempty"` +} + +type TopRelation struct { + Column string `json:"column"` + Target string `json:"target"` + Label string `json:"label"` +} + +type TopStack struct { + Column string `json:"column"` + MetricColumn string `json:"metricColumn"` + FrameKey string `json:"frameKey,omitempty"` + Separator string `json:"separator,omitempty"` + LeafFirst bool `json:"leafFirst,omitempty"` +} + +func (h *Handler[P, R]) WithTop(spec TopSpec) *Handler[P, R] { + h.top = &spec + return h.WithResults(NewResultTable("frames", recordresults.ResultType[sampleframe.Frame]{Title: "Completed samples", TimeColumn: "completedAt", Follow: true})) +} + +func (h *Handler[P, R]) Top() *TopSpec { return h.top } + +func TopOf(plugin TracePlugin) *TopSpec { + if provider, ok := plugin.(interface{ Top() *TopSpec }); ok { + return provider.Top() + } + return nil +} + +type frameEmitter[R any] interface { + EmitFrame(context.Context, sampleframe.Frame, []R) error +} + +// PublishFrame refuses emitters without atomic sample support. +func PublishFrame[R any](ctx context.Context, out Emitter[R], frame sampleframe.Frame, rows []R) error { + writer, ok := out.(frameEmitter[R]) + if !ok { + return fmt.Errorf("sampled capture requires an atomic frame emitter") + } + return writer.EmitFrame(ctx, frame, rows) +} + +func (e *emitter[R]) EmitFrame(ctx context.Context, frame sampleframe.Frame, records []R) error { + if e.frames == nil { + return fmt.Errorf("kind does not declare a Top capability") + } + rows := make([]recordstore.Row, len(records)) + for i, record := range records { + row, err := e.encode(record) + if err != nil { + return err + } + rows[i] = row + } + if e.top.Stack != nil { + metrics := []TopMetric{} + for _, row := range rows { + key, _ := row[e.top.Stack.MetricColumn].(string) + index := slices.IndexFunc(e.top.Metrics, func(metric TopMetric) bool { return metric.Key == key }) + if index < 0 { + return fmt.Errorf("unregistered sample metric %q", key) + } + metric := e.top.Metrics[index] + if unit := row["unit"]; unit != metric.Unit { + return fmt.Errorf("sample metric %s unit %v disagrees with %s", key, unit, metric.Unit) + } + if !slices.ContainsFunc(metrics, func(m TopMetric) bool { return m.Key == key }) { + metrics = append(metrics, metric) + } + } + if frame.Summary == nil { + frame.Summary = map[string]any{} + } + slices.SortFunc(metrics, func(a, b TopMetric) int { + return slices.IndexFunc(e.top.Metrics, func(m TopMetric) bool { return m.Key == a.Key }) - slices.IndexFunc(e.top.Metrics, func(m TopMetric) bool { return m.Key == b.Key }) + }) + frame.Summary["metrics"] = metrics + } + if profilepublication.IsProfile(frame) { + published, err := e.publishProfile(ctx, frame, rows) + if err != nil || !published { + return err + } + } else { + if _, err := e.frames.Append(ctx, frame, rows); err != nil { + return err + } + } + e.source.mu.Lock() + e.source.summary.Emitted += int64(len(rows)) + e.source.mu.Unlock() + return nil +} + +func (e *emitter[R]) publishProfile(ctx context.Context, frame sampleframe.Frame, rows []recordstore.Row) (bool, error) { + backend, ok := e.frames.Backend.(profilepublication.Backend) + if !ok { + return false, fmt.Errorf("profile capture requires durable identity reads and atomic publication") + } + frames, err := sampleframe.Schema(e.frames.Schema.Kind + "_frames") + if err != nil { + return false, err + } + return profilepublication.Append(ctx, backend, profilepublication.Publication{Kind: e.frames.Schema.Kind, Stream: e.frames.Stream, FrameStream: e.frames.Stream + "-frames", Schema: e.frames.Schema, FrameSchema: frames, Frame: frame, Rows: rows}) +} + +func (s TopSpec) validate(schema recordstore.KindSchema) error { + keys := append(slices.Clone(s.Identity), s.GroupBy...) + if s.ProjectionColumn != "" { + keys = append(keys, s.ProjectionColumn) + } + if s.Relation != nil { + keys = append(keys, s.Relation.Column, s.Relation.Target) + } + if s.Stack != nil { + keys = append(keys, s.Stack.Column, s.Stack.MetricColumn) + } + for _, key := range keys { + if !slices.ContainsFunc(schema.Columns, func(c query.ColumnDef) bool { return c.Name == key }) { + return fmt.Errorf("unknown Top identity column %q", key) + } + } + metrics := map[string]bool{} + for _, metric := range s.Metrics { + if metric.Key == "" || metric.Label == "" || metrics[metric.Key] || !slices.Contains([]string{"table", "flamegraph", "treemap"}, metric.View) { + return fmt.Errorf("invalid Top metric %q", metric.Key) + } + metrics[metric.Key] = true + } + for _, order := range s.Order { + if !slices.ContainsFunc(schema.Columns, func(c query.ColumnDef) bool { return c.Name == order.Column }) { + return fmt.Errorf("unknown Top order column %q", order.Column) + } + } + return nil +} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 1878247..182d5f9 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,4 +1,6 @@ packages: - webapp minimumReleaseAgeExclude: - - '@flanksource/clicky-ui@0.3.52 || 0.3.56' + - '@flanksource/clicky-ui@0.3.52 || 0.3.56 || 0.3.58' +overrides: + '@flanksource/clicky-ui': link:../clicky-ui/packages/ui diff --git a/result_pretty.go b/result_pretty.go new file mode 100644 index 0000000..72f4ae8 --- /dev/null +++ b/result_pretty.go @@ -0,0 +1,83 @@ +package telltale + +import ( + "fmt" + "maps" + "slices" + "strings" + "time" + + "github.com/flanksource/clicky/api" + "github.com/flanksource/telltale/pkg/tracing/target" +) + +// Pretty is the run on one line, then a line per capture: its target, the +// scalar entries of its summary and the command that reads its records back, +// then a line per reason the run failed. --format json keeps every field. +func (r Result) Pretty() api.Text { + text := api.Text{}.Append("Run ", "text-muted").Append(r.RunID, "font-bold").Append(" · "+r.EndedAt.Sub(r.StartedAt).Round(r.precision()).String(), "text-muted") + switch { + case len(r.Command) > 0 && r.ExitCode != 0: + text = text.Append(fmt.Sprintf(" · %s exited %d", strings.Join(r.Command, " "), r.ExitCode), "text-red-600") + case len(r.Command) > 0: + text = text.Append(fmt.Sprintf(" · %s exited 0", strings.Join(r.Command, " ")), "text-muted") + case r.PID != 0: + text = text.Append(fmt.Sprintf(" · pid %d", r.PID), "text-muted") + } + for _, capture := range r.Captures { + text = text.Append("\n ").Append(capture.Kind, "font-bold") + for _, part := range append(targetParts(capture.Target), summaryParts(capture.Summary)...) { + text = text.Append(" · " + part) + } + text = text.Append(fmt.Sprintf(" · telltale records %s --run %s", capture.Kind, r.RunID), "text-muted") + } + for _, failure := range append(slices.Clone(r.CheckpointErrors), r.Error) { + if failure != "" { + text = text.Append("\n ✗ "+failure, "text-red-600") + } + } + return text +} + +// precision rounds a sub-second run to milliseconds and a longer one to a tenth of a second. +func (r Result) precision() time.Duration { + if r.EndedAt.Sub(r.StartedAt) < time.Second { + return time.Millisecond + } + return 100 * time.Millisecond +} + +func targetParts(t target.Target) []string { + switch { + case t.Agent != "": + return []string{"agent " + t.Agent} + case t.PID != 0: + return []string{fmt.Sprintf("pid %d", t.PID)} + case len(t.Selector) > 0: + pairs := []string{} + for _, key := range slices.Sorted(maps.Keys(t.Selector)) { + pairs = append(pairs, key+"="+t.Selector[key]) + } + return []string{"agents " + strings.Join(pairs, ",")} + case t.Seam != "": + return []string{string(t.Seam)} + } + return nil +} + +// summaryParts are a summary's numbers, strings and booleans by name; its +// lists and maps are left to --format json. A count is the records recorded. +func summaryParts(summary map[string]any) []string { + parts := []string{} + for _, key := range slices.Sorted(maps.Keys(summary)) { + switch value := summary[key].(type) { + case string, bool, int, int32, int64, float32, float64, uint, uint32, uint64: + if key == "count" { + parts = append(parts, fmt.Sprintf("%v records", value)) + } else { + parts = append(parts, fmt.Sprintf("%s %v", key, value)) + } + } + } + return parts +} diff --git a/result_pretty_test.go b/result_pretty_test.go new file mode 100644 index 0000000..f86ac65 --- /dev/null +++ b/result_pretty_test.go @@ -0,0 +1,45 @@ +package telltale_test + +import ( + "time" + + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/target" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("a run's summary", func() { + started := time.Date(2026, 11, 16, 0, 1, 48, 0, time.UTC) + const runID = "host20261116T000148Z" + + It("prints the run on one line, then one line per capture with its target, scalar summary and the command that reads it back", func() { + result := telltale.Result{RunID: runID, StartedAt: started, EndedAt: started.Add(3600 * time.Millisecond), Captures: []telltale.CaptureResult{ + {Kind: "jvm_trace", Ref: tracing.TraceRef{Kind: "jvm_trace", Stream: "S1"}, Summary: map[string]any{"count": 45}, Target: target.Target{Seam: target.Agent, Agent: "80618"}}, + {Kind: "http", Ref: tracing.TraceRef{Kind: "http", Stream: "S2"}, Summary: map[string]any{"entries": 12, "errors": 1, "hosts": []string{"example.com"}, "methods": map[string]int{"GET": 12}}, Target: target.Target{Seam: target.Proxy}}, + }} + Expect(result.Pretty().String()).To(Equal( + "Run " + runID + " · 3.6s\n" + + " jvm_trace · agent 80618 · 45 records · telltale records jvm_trace --run " + runID + "\n" + + " http · proxy · entries 12 · errors 1 · telltale records http --run " + runID)) + }) + + It("names the command and its exit code, and lists why the run failed", func() { + result := telltale.Result{RunID: runID, Command: []string{"make", "test"}, ExitCode: 2, StartedAt: started, EndedAt: started.Add(90 * time.Second), + CheckpointErrors: []string{"heap: process exited"}, Error: "summarize sql: closed"} + Expect(result.Pretty().String()).To(Equal( + "Run " + runID + " · 1m30s · make test exited 2\n" + + " ✗ heap: process exited\n" + + " ✗ summarize sql: closed")) + }) + + It("names a traced pid, and a capture with nothing recorded yet", func() { + result := telltale.Result{RunID: runID, PID: 4242, StartedAt: started, EndedAt: started.Add(250 * time.Millisecond), Captures: []telltale.CaptureResult{ + {Kind: "process", Ref: tracing.TraceRef{Kind: "process", Stream: "S3"}, Target: target.Target{PID: 4242}}, + }} + Expect(result.Pretty().String()).To(Equal( + "Run " + runID + " · 250ms · pid 4242\n" + + " process · pid 4242 · telltale records process --run " + runID)) + }) +}) diff --git a/run.go b/run.go index a89967b..fa9e23d 100644 --- a/run.go +++ b/run.go @@ -2,6 +2,7 @@ package telltale import ( "context" + "encoding/json" "errors" "fmt" "maps" @@ -15,7 +16,10 @@ import ( "github.com/flanksource/telltale/pkg/traces" "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/process" + "github.com/flanksource/telltale/pkg/tracing/processcapture" "github.com/flanksource/telltale/pkg/tracing/stacks" + "github.com/flanksource/telltale/pkg/tracing/target" ) // ErrNotLive is a Stop of a run this process is not running: finished, @@ -36,7 +40,7 @@ type Run struct { captures []*traces.Capture checkpoints checkpoints cmd *exec.Cmd - output *os.File + output []*os.File stopOnce sync.Once stopErr error // stopping is closed by Stop of an attached run, which owns no process @@ -45,10 +49,12 @@ type Run struct { exited chan struct{} // checkpointed is closed once no interval checkpoint can start or is in // flight, so the captures can be stopped. - checkpointed chan struct{} - done chan struct{} - result *Result - err error + checkpointed chan struct{} + checkpointCtx context.Context + cancelCheckpoints context.CancelFunc + done chan struct{} + result *Result + err error } // Start arms options.Traces, starts options.Command or attaches to @@ -65,6 +71,8 @@ func Start(ctx context.Context, options StartOptions) (*Run, error) { startedAt := time.Now() host, err := traces.Open(traces.OpenOptions{ DataDir: options.DataDir, RunID: traces.RunID(startedAt), Connections: options.Connections, RunConnections: options.RunConnections, + // Reserve the tree, samples and stacks captures alongside the requested ones. + MaxCaptures: len(specs) + 3, }) if err != nil { return nil, err @@ -73,6 +81,7 @@ func Start(ctx context.Context, options StartOptions) (*Run, error) { host: host, options: options, specs: specs, startedAt: startedAt, stopping: make(chan struct{}), exited: make(chan struct{}), checkpointed: make(chan struct{}), done: make(chan struct{}), } + run.checkpointCtx, run.cancelCheckpoints = context.WithCancel(ctx) if run.key, err = registryKey(host.Dir()); err == nil { err = run.start() } @@ -84,7 +93,7 @@ func Start(ctx context.Context, options StartOptions) (*Run, error) { } } if err != nil { - return nil, errors.Join(err, run.close()) + return nil, errors.Join(err, run.close(context.WithoutCancel(ctx))) } live.add(run) go run.wait() @@ -143,11 +152,11 @@ func (r *Run) Info() RunInfo { } captures := make([]CaptureResult, 0, len(r.captures)) for _, capture := range r.captures { - captures = append(captures, CaptureResult{Kind: capture.Kind(), Ref: capture.Ref(0, 0)}) + captures = append(captures, CaptureResult{Kind: capture.Kind(), Ref: capture.Ref(0, 0), Target: capture.Target()}) } info := RunInfo{ RunID: r.ID(), Dir: r.Dir(), Status: StatusRunning, Command: r.options.Command, PID: r.options.PID, StartedAt: r.startedAt, - Captures: captures, Process: r.processRefs(), CheckpointErrors: r.checkpoints.failed(), Definition: r.options.Definition, + Captures: captures, Process: r.processRefs(), Output: r.outputRefs(), CheckpointErrors: r.checkpoints.failed(), Definition: r.options.Definition, } if r.process != nil { info.Live = &LiveRun{Sessions: r.process.Sessions()} @@ -171,6 +180,7 @@ func (r *Run) processRefs() *traces.ProcessRefs { // The run still summarizes its captures and writes its manifest. func (r *Run) Stop() error { r.stopOnce.Do(func() { + r.cancelCheckpoints() if r.cmd == nil { close(r.stopping) return @@ -238,6 +248,28 @@ func (r *Run) start() error { func (r *Run) arm() (map[string]string, error) { env := map[string]string{} for _, spec := range r.specs { + if spec.Kind == processcapture.Kind && spec.target == target.Process { + var params processcapture.Params + if err := json.Unmarshal(spec.Params, ¶ms); err != nil { + return nil, err + } + if params.Environment == nil { + params.Environment = r.options.Tree.Environment + } + if r.options.Tree.Facets { + if params.Facets == nil { + params.Facets = []string{processcapture.Files, processcapture.Sockets, "connections"} + } + if params.Environment == nil { + params.Environment = &process.EnvironmentParams{} + } + } + var err error + spec.Params, err = json.Marshal(params) + if err != nil { + return nil, err + } + } capture, err := r.host.Start(traces.StartOptions{Kind: spec.Kind, Params: spec.Params, Labels: spec.Labels}) if err != nil { return nil, err @@ -258,12 +290,14 @@ func (r *Run) command(env map[string]string) (*exec.Cmd, error) { cmd.Dir = r.options.Dir cmd.Stdin, cmd.Stdout, cmd.Stderr = r.options.Stdin, r.options.Stdout, r.options.Stderr if r.options.Stdout == nil && r.options.Stderr == nil { - output, err := os.Create(filepath.Join(r.Dir(), OutputFile)) - if err != nil { - return nil, fmt.Errorf("create run output: %w", err) + for _, name := range []string{StdoutFile, StderrFile} { + output, err := os.Create(filepath.Join(r.Dir(), name)) + if err != nil { + return nil, fmt.Errorf("create run %s: %w", name, err) + } + r.output = append(r.output, output) } - r.output = output - cmd.Stdout, cmd.Stderr = output, output + cmd.Stdout, cmd.Stderr = r.output[0], r.output[1] } if r.options.TraceRoot == TraceRootCommand { cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} @@ -283,8 +317,26 @@ func (r *Run) startCommand() error { } func (r *Run) startProcessTrace(rootPID int) error { + options := traces.ProcessTraceOptions{RootPID: rootPID, Tree: r.options.Tree} + for i, spec := range r.specs { + if spec.Kind != processcapture.Kind || spec.target != target.Process { + continue + } + var params processcapture.Params + if err := json.Unmarshal(spec.Params, ¶ms); err != nil { + return err + } + pid := params.Target.PID + if pid == 0 { + pid = os.Getpid() + } + if pid == rootPID { + options.Capture = r.captures[i] + break + } + } var err error - if r.process, err = r.host.StartProcessTrace(traces.ProcessTraceOptions{RootPID: rootPID, Tree: r.options.Tree}); err != nil { + if r.process, err = r.host.StartProcessTrace(options); err != nil { return fmt.Errorf("process trace: %w", err) } return nil @@ -301,19 +353,26 @@ func (r *Run) wait() { r.waitAttached() } close(r.exited) - <-r.checkpointed + r.cancelCheckpoints() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + select { + case <-r.checkpointed: + case <-ctx.Done(): + err = errors.Join(err, fmt.Errorf("settle interval checkpoints: %w", ctx.Err())) + } if r.cmd == nil && (r.options.host() || alive(r.options.PID)) { - r.checkpoint(context.Background(), "stop") + r.checkpoint(ctx, "stop") } result := &Result{ RunID: r.ID(), Dir: r.Dir(), Command: r.options.Command, PID: r.options.PID, StartedAt: r.startedAt, EndedAt: time.Now(), - ExitCode: exitCode, Process: r.processRefs(), Definition: r.options.Definition, + ExitCode: exitCode, Process: r.processRefs(), Output: r.outputRefs(), Definition: r.options.Definition, } if err == nil { - result.Captures, err = r.summarize(context.Background()) + result.Captures, err = r.summarize(ctx) result.CheckpointErrors = r.checkpoints.failed() } - if err = errors.Join(err, r.close()); err != nil { + if err = errors.Join(err, ctx.Err(), r.close(ctx)); err != nil { result.Error = err.Error() } // A run that could not end cleanly still records that it ended, and why, @@ -366,28 +425,36 @@ func commandExitCode(err error, program string) (int, error) { // summarize stops every capture, committing its stream, and summarizes it. func (r *Run) summarize(ctx context.Context) ([]CaptureResult, error) { - results := make([]CaptureResult, 0, len(r.captures)) - for _, capture := range r.captures { + results := make([]CaptureResult, len(r.captures)) + for index, capture := range r.captures { + results[index] = CaptureResult{Kind: capture.Kind(), Ref: capture.Ref(0, 0), Target: capture.Target()} + } + var errs []error + for index, capture := range r.captures { seq, err := capture.Stop(ctx) if err != nil { - return nil, err + errs = append(errs, err) + continue } ref := capture.Ref(0, seq) + results[index].Ref = ref summary, err := r.host.Summarize(ctx, ref) if err != nil { - return nil, fmt.Errorf("summarize %s trace: %w", capture.Kind(), err) + errs = append(errs, fmt.Errorf("summarize %s trace: %w", capture.Kind(), err)) + continue } - results = append(results, CaptureResult{Kind: capture.Kind(), Ref: ref, Summary: summary}) + results[index].Summary = summary } - return results, nil + return results, errors.Join(errs...) } // close takes a last reading of the process capture — under TraceRootSelf // telltale's final goroutines into its stacks — re-observes the tree, stops // the run's captures and closes its store, which stays on disk to be served; // the oldest runs' stores beyond traces.KeepRuns go. -func (r *Run) close() error { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) +func (r *Run) close(ctx context.Context) error { + r.cancelCheckpoints() + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() var errs []error if r.process != nil { @@ -396,8 +463,8 @@ func (r *Run) close() error { } errs = append(errs, r.process.CheckpointTree(ctx), r.process.Stop(ctx)) } - if r.output != nil { - errs = append(errs, r.output.Close()) + for _, output := range r.output { + errs = append(errs, output.Close()) } errs = append(errs, r.host.Close(ctx), traces.Prune(r.options.DataDir, traces.KeepRuns)) return errors.Join(errs...) diff --git a/run_shutdown_test.go b/run_shutdown_test.go new file mode 100644 index 0000000..23db31d --- /dev/null +++ b/run_shutdown_test.go @@ -0,0 +1,77 @@ +package telltale_test + +import ( + "context" + "fmt" + "sync/atomic" + "time" + + dbcontext "github.com/flanksource/commons-db/context" + "github.com/flanksource/commons-db/recordstore/recordresults" + "github.com/flanksource/telltale" + "github.com/flanksource/telltale/pkg/traces" + "github.com/flanksource/telltale/pkg/tracing" + "github.com/flanksource/telltale/pkg/tracing/target" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type shutdownReading struct { + Reason string `json:"reason"` +} +type shutdownCapture struct{ started, release chan struct{} } + +var shutdownKindID atomic.Int64 + +func (shutdownCapture) Params() struct{} { return struct{}{} } +func (shutdownCapture) Schema() recordresults.ResultType[shutdownReading] { + return recordresults.ResultType[shutdownReading]{Title: "Shutdown checkpoints"} +} +func (shutdownCapture) Handle(ctx dbcontext.Context, _ struct{}, _ tracing.Emitter[shutdownReading], _ tracing.Records[shutdownReading]) error { + <-ctx.Done() + return nil +} +func (c shutdownCapture) Checkpoint(ctx dbcontext.Context, _ struct{}, out tracing.Emitter[shutdownReading]) error { + var request shutdownReading + if _, err := tracing.DecodeCheckpointParams(ctx, &request); err != nil { + return err + } + if request.Reason == "interval" { + close(c.started) + select { + case <-ctx.Done(): + return ctx.Err() + case <-c.release: + } + } + return out.Emit(ctx, request) +} + +var _ = Describe("Run shutdown checkpoints", func() { + It("cancels an interval reading and preserves the final stop reading and manifest", func() { + capture := shutdownCapture{started: make(chan struct{}), release: make(chan struct{})} + DeferCleanup(func() { close(capture.release) }) + kind := fmt.Sprintf("shutdown_checkpoint_%d", shutdownKindID.Add(1)) + Expect(traces.Register(traces.Registration{ + Name: kind, Title: "Shutdown checkpoints", Summary: "Records shutdown checkpoint lifecycle", Seams: []target.Seam{target.Host}, DefaultSeam: target.Host, + Checkpoint: shutdownReading{}, New: func(traces.KindDeps) tracing.TracePlugin { + return tracing.NewHandler[struct{}, shutdownReading](capture, tracing.Capabilities{Live: true}, target.Host) + }, + })).To(Succeed()) + ctx, cancel := context.WithCancel(context.Background()) + DeferCleanup(cancel) + dataDir := GinkgoT().TempDir() + run, err := telltale.Start(ctx, telltale.StartOptions{DataDir: dataDir, Every: 10 * time.Millisecond, Traces: []telltale.TraceSpec{{Kind: kind}}}) + Expect(err).NotTo(HaveOccurred()) + Eventually(capture.started).Should(BeClosed()) + cancel() + Eventually(run.Done(), time.Second).Should(BeClosed()) + result, err := run.Wait() + Expect(err).NotTo(HaveOccurred()) + Expect(result.CheckpointErrors).To(BeEmpty()) + Expect(result.Captures).To(ConsistOf(HaveField("Summary", HaveKeyWithValue("count", BeNumerically(">", 0))))) + info, err := telltale.Get(telltale.GetOptions{DataDir: dataDir, RunID: run.ID()}) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Status).To(Equal(telltale.StatusSucceeded)) + }) +}) diff --git a/run_specs.go b/run_specs.go index 7abb92c..86def1a 100644 --- a/run_specs.go +++ b/run_specs.go @@ -174,6 +174,9 @@ func (r *Run) checkpoint(ctx context.Context, reason string) { if err == nil { err = capture.Checkpoint(ctx, params) } + if reason == "interval" && ctx.Err() != nil && errors.Is(err, ctx.Err()) { + return + } // The command leaves its tree only by exiting: a read that found it gone // raced the exit that ends the run, and is no failure once the command exits. if ofCommand && errors.Is(err, tracing.ErrCheckpointParams) && r.exitsWithin(commandExitGrace) { @@ -212,10 +215,12 @@ func (r *Run) checkpointEvery() { defer ticker.Stop() for { select { + case <-r.checkpointCtx.Done(): + return case <-r.exited: return case <-ticker.C: - r.checkpoint(context.Background(), "interval") + r.checkpoint(r.checkpointCtx, "interval") } } } diff --git a/run_specs_internal_test.go b/run_specs_internal_test.go index 9fcf0cd..cd7ca4a 100644 --- a/run_specs_internal_test.go +++ b/run_specs_internal_test.go @@ -36,12 +36,10 @@ var _ = Describe("run target resolution", func() { Expect(string(got)).To(MatchJSON(`{"target":{"pid":123}}`)) }) It("uses the run PID only for seams that accept it", func() { - for _, kind := range []string{"process_groups"} { - result, err := aimedSpecs(StartOptions{PID: 123, Traces: []TraceSpec{{Kind: kind}}}) - Expect(err).NotTo(HaveOccurred()) - Expect(string(result[0].Params)).NotTo(ContainSubstring("pid")) - } - result, err := aimedSpecs(StartOptions{PID: 123, Traces: []TraceSpec{{Kind: "process_samples"}}}) + host, err := aimedSpecs(StartOptions{PID: 123, Traces: []TraceSpec{{Kind: "process", Params: json.RawMessage(`{"target":{"seam":"host"}}`)}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(string(host[0].Params)).NotTo(ContainSubstring("pid")) + result, err := aimedSpecs(StartOptions{PID: 123, Traces: []TraceSpec{{Kind: "process"}}}) Expect(err).NotTo(HaveOccurred()) Expect(string(result[0].Params)).To(MatchJSON(`{"target":{"pid":123}}`)) _, err = aimedSpecs(StartOptions{PID: 123, Traces: []TraceSpec{{Kind: "http"}}}) diff --git a/run_test.go b/run_test.go index a0c2783..6e62c38 100644 --- a/run_test.go +++ b/run_test.go @@ -15,10 +15,10 @@ import ( "github.com/flanksource/telltale" "github.com/flanksource/telltale/pkg/traces" - "github.com/flanksource/telltale/pkg/tracing/groups" "github.com/flanksource/telltale/pkg/tracing/httpproxy" - "github.com/flanksource/telltale/pkg/tracing/process" + "github.com/flanksource/telltale/pkg/tracing/processcapture" "github.com/flanksource/telltale/pkg/tracing/stacks" + "github.com/flanksource/telltale/pkg/tracing/target" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -60,7 +60,7 @@ var _ = Describe("running a command under trace", func() { Expect(result.Dir).To(BeADirectory()) Expect(result.StartedAt).To(BeTemporally(">=", before)) Expect(result.EndedAt).To(BeTemporally(">=", result.StartedAt)) - Expect(result.Process.Tree.Kind).To(Equal(process.TreeKind)) + Expect(result.Process.Tree.Kind).To(Equal(processcapture.Kind)) Expect(result.Captures).To(BeEmpty()) }) @@ -75,6 +75,10 @@ var _ = Describe("running a command under trace", func() { Expect(output).To(Equal("traced")) Expect(result.Captures).To(HaveLen(1)) Expect(result.Captures[0].Kind).To(Equal(httpproxy.Kind)) + Expect(result.Captures[0].Target).To(Equal(target.Target{Seam: target.Proxy})) + stored, err := telltale.ReadManifest(result.Dir) + Expect(err).NotTo(HaveOccurred()) + Expect(stored.Captures[0].Target).To(Equal(result.Captures[0].Target)) Expect(result.Captures[0].Summary).To(HaveKeyWithValue("count", BeNumerically("==", 1))) }) @@ -136,7 +140,7 @@ var _ = Describe("a run attached to a running process", func() { target, dataDir := sleeper("30"), GinkgoT().TempDir() run, err := telltale.Start(context.Background(), telltale.StartOptions{ DataDir: dataDir, PID: target.Process.Pid, Duration: time.Second, - Traces: []telltale.TraceSpec{{Kind: process.SamplesKind}}, + Traces: []telltale.TraceSpec{{Kind: processcapture.Kind}}, }) Expect(err).ToNot(HaveOccurred()) result, err := run.Wait() @@ -144,7 +148,8 @@ var _ = Describe("a run attached to a running process", func() { Expect(result.PID).To(Equal(target.Process.Pid)) Expect(result.Command).To(BeNil()) Expect(result.EndedAt.Sub(result.StartedAt)).To(BeNumerically(">=", time.Second)) - Expect(result.Captures).To(ConsistOf(HaveField("Kind", process.SamplesKind))) + Expect(result.Captures).To(ConsistOf(HaveField("Kind", processcapture.Kind))) + Expect(result.Process.Tree.Stream).To(Equal(result.Captures[0].Ref.Stream), "the automatic process view reuses the configured capture") Expect(target.Process.Signal(syscall.Signal(0))).To(Succeed(), "an attached run leaves its process running") info, err := telltale.Get(telltale.GetOptions{DataDir: dataDir, RunID: result.RunID}) @@ -154,9 +159,10 @@ var _ = Describe("a run attached to a running process", func() { }) It("ends when the process exits", func() { - target := sleeper("1") + target := sleeper("30") run, err := telltale.Start(context.Background(), telltale.StartOptions{DataDir: GinkgoT().TempDir(), PID: target.Process.Pid}) Expect(err).ToNot(HaveOccurred()) + Expect(target.Process.Kill()).To(Succeed()) Eventually(run.Done(), 10*time.Second).Should(BeClosed()) }) @@ -205,8 +211,8 @@ var _ = Describe("a run attached to a running process", func() { }, "not both"), Entry("neither", func(int) telltale.StartOptions { return telltale.StartOptions{} }, "a command, a pid to attach to, or a host capture is required"), Entry("neither, for a kind that observes a process", func(int) telltale.StartOptions { - return telltale.StartOptions{Traces: []telltale.TraceSpec{{Kind: groups.Kind}, {Kind: process.SamplesKind}}} - }, "process_samples observes a command or a process: give a command or a pid"), + return telltale.StartOptions{Traces: []telltale.TraceSpec{{Kind: processcapture.Kind}}} + }, "process observes a command or a process: give a command or a pid"), Entry("a process that is not running", func(int) telltale.StartOptions { return telltale.StartOptions{PID: 999999} }, "pid 999999 is not running"), @@ -222,12 +228,12 @@ var _ = Describe("a host run", func() { dataDir := GinkgoT().TempDir() run, err := telltale.Start(context.Background(), telltale.StartOptions{ DataDir: dataDir, Duration: 2500 * time.Millisecond, - Traces: []telltale.TraceSpec{{Kind: groups.Kind, Params: json.RawMessage(`{"interval": "1s", "all": true}`)}}, + Traces: []telltale.TraceSpec{{Kind: processcapture.Kind, Params: json.RawMessage(`{"interval":"1s","target":{"seam":"host"}}`)}}, }) Expect(err).ToNot(HaveOccurred()) info := run.Info() Expect(info).To(And(HaveField("Status", telltale.StatusRunning), HaveField("Process", BeNil()), HaveField("Live", BeNil()))) - follower, err := groups.NewFollower(run.Runtime(), info.Captures[0].Ref) + follower, err := processcapture.NewFollower(run.Runtime(), info.Captures[0].Ref) Expect(err).ToNot(HaveOccurred()) Eventually(func(g Gomega) { frame, ok, err := follower.Frame(context.Background()) @@ -241,7 +247,7 @@ var _ = Describe("a host run", func() { Expect(err).ToNot(HaveOccurred()) Expect(result.EndedAt.Sub(result.StartedAt)).To(BeNumerically(">=", 2500*time.Millisecond)) Expect(result.Process).To(BeNil()) - Expect(result.Captures).To(ConsistOf(And(HaveField("Kind", groups.Kind), HaveField("Summary", HaveKeyWithValue("count", BeNumerically(">", 0)))))) + Expect(result.Captures).To(ConsistOf(And(HaveField("Kind", processcapture.Kind), HaveField("Summary", HaveKeyWithValue("count", BeNumerically(">", 0)))))) stored, err := telltale.Get(telltale.GetOptions{DataDir: dataDir, RunID: result.RunID}) Expect(err).ToNot(HaveOccurred()) @@ -309,7 +315,7 @@ var _ = Describe("a run traced from its command", func() { Expect(info.Live.Sessions.Tree).ToNot(BeEmpty()) Eventually(func() (string, error) { - data, err := os.ReadFile(filepath.Join(run.Dir(), telltale.OutputFile)) + data, err := os.ReadFile(filepath.Join(run.Dir(), telltale.StdoutFile)) return string(data), err }).Should(Equal("started\n")) Expect(telltale.Stop(telltale.StopOptions{DataDir: dataDir, RunID: run.ID()})).To(Succeed()) diff --git a/schema.go b/schema.go index 4c7b499..193ecab 100644 --- a/schema.go +++ b/schema.go @@ -49,7 +49,7 @@ func StartSchema() (*rpc.OpenAPISchema, map[string]any, error) { if selection := params.Properties["target"]; selection != nil { delete(selection.Properties, "pid") } - captures.Properties[kind] = captureSchema(registration.Title, order, params) + captures.Properties[kind] = captureSchema(registration, order, params) defaults := map[string]any{enabledKey: kind == httpproxy.Kind} for name, property := range params.Properties { if property.Default != nil { @@ -91,20 +91,22 @@ func treeSchema() *rpc.OpenAPISchema { return schema } -// captureSchema is one armable kind's object: its enabled switch and, while -// it is on, the kind's params. -func captureSchema(title string, order int, params *rpc.OpenAPISchema) *rpc.OpenAPISchema { +// captureSchema is one armable kind's object: its summary, seams, enabled +// switch and, while it is on, the kind's params. +func captureSchema(registration traces.Registration, order int, params *rpc.OpenAPISchema) *rpc.OpenAPISchema { then := map[string]any{"properties": params.Properties} if len(params.Required) > 0 { then["required"] = params.Required } return &rpc.OpenAPISchema{ - Type: "object", Title: title, + Type: "object", Title: registration.Title, Description: registration.Summary, Properties: map[string]*rpc.OpenAPISchema{ enabledKey: {Type: "boolean", Title: "Enabled", Extensions: map[string]any{"x-clicky-order": -1}}, }, Extensions: map[string]any{ - "x-clicky-order": order, + "x-clicky-order": order, + "x-telltale-seams": slices.Clone(registration.Seams), + "x-telltale-default-seam": registration.DefaultSeam, "allOf": []any{map[string]any{ "if": map[string]any{"properties": map[string]any{enabledKey: map[string]any{"const": true}}}, "then": then, diff --git a/schema_test.go b/schema_test.go index 025e540..dccaefa 100644 --- a/schema_test.go +++ b/schema_test.go @@ -12,6 +12,7 @@ import ( "github.com/flanksource/telltale/pkg/tracing/target" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/onsi/gomega/types" ) var _ = Describe("the start form", func() { @@ -49,6 +50,40 @@ var _ = Describe("the start form", func() { Expect(captureDefaults[httpproxy.Kind]).NotTo(HaveKey("features")) Expect(captureDefaults[pgproxy.Kind]).To(And(HaveKeyWithValue("enabled", false), HaveKeyWithValue("target", map[string]any{"seam": target.Proxy}))) }) + + DescribeTable("publishes each armable kind's summary, seams and default seam on its capture object", + func(kind string, seams types.GomegaMatcher) { + registration, err := traces.Registered(kind) + Expect(err).ToNot(HaveOccurred()) + schema, _, err := telltale.StartSchema() + Expect(err).ToNot(HaveOccurred()) + data, err := json.Marshal(schema.Properties["traces"].Properties[kind]) + Expect(err).ToNot(HaveOccurred()) + var capture map[string]any + Expect(json.Unmarshal(data, &capture)).To(Succeed()) + Expect(capture).To(And( + HaveKeyWithValue("description", registration.Summary), + HaveKeyWithValue("x-telltale-seams", seams), + HaveKeyWithValue("x-telltale-default-seam", string(registration.DefaultSeam)), + )) + }, + Entry("sql_top, aimed only at a connection", "sql_top", Equal([]any{string(target.Connection)})), + Entry("http, through a proxy or an agent", httpproxy.Kind, ContainElements(string(target.Proxy), string(target.Agent))), + ) + + DescribeTable("looks a SQL Server kind's connection up among sql_server connections", + func(kind string) { + kinds, err := traces.Kinds() + Expect(err).ToNot(HaveOccurred()) + plugin, err := traces.Lookup(kinds, kind) + Expect(err).ToNot(HaveOccurred()) + params, err := plugin.Params() + Expect(err).ToNot(HaveOccurred()) + Expect(params.Properties["connection"].Extensions).To(HaveKeyWithValue("x-clicky-lookup", HaveKeyWithValue("url", "/api/v1/connection?types=sql_server"))) + }, + Entry("sql_deadlock", "sql_deadlock"), + Entry("sql_xevent", "sql_xevent"), + ) }) var _ = Describe("a start request", func() { @@ -94,6 +129,31 @@ var _ = Describe("a start request", func() { Expect(options.Traces).To(HaveExactElements(HaveField("Labels", HaveKeyWithValue(definition.LabelEntry, "adhoc-collect-prometheus-scrape")))) }) + It("arms each element of a kind's array as its own entry of the inline definition", func() { + endpoints := []string{"http://127.0.0.1:9100/metrics", "http://127.0.0.1:9200/metrics"} + options, err := telltale.StartRequest{Traces: map[string]json.RawMessage{ + "prometheus": json.RawMessage(`[{"enabled":true,"endpoints":["` + endpoints[0] + `"]},{"enabled":true,"endpoints":["` + endpoints[1] + `"]}]`), + }}.Options(base, definitions) + Expect(err).ToNot(HaveOccurred()) + Expect(options.Traces).To(HaveExactElements( + And(HaveField("Params", ContainSubstring(endpoints[0])), HaveField("Labels", HaveKeyWithValue(definition.LabelEntry, "adhoc-collect-prometheus-scrape-1"))), + And(HaveField("Params", ContainSubstring(endpoints[1])), HaveField("Labels", HaveKeyWithValue(definition.LabelEntry, "adhoc-collect-prometheus-scrape-2"))), + )) + }) + + It("names a refused element of a kind's array by its index", func() { + _, err := telltale.StartRequest{Traces: map[string]json.RawMessage{ + "sql_deadlock": json.RawMessage(`[{"enabled":false},{"enabled":true}]`), + "prometheus": json.RawMessage(`[{"enabled":true,"endpoints":["http://127.0.0.1:9100/metrics"]},3]`), + }}.Options(base, definitions) + var invalid *telltale.ValidationError + Expect(err).To(BeAssignableToTypeOf(invalid)) + Expect(err.(*telltale.ValidationError).Fields).To(ConsistOf( + HaveField("InstancePath", "/traces/sql_deadlock/1/connection"), + And(HaveField("InstancePath", "/traces/prometheus/1"), HaveField("Message", ContainSubstring("want an object"))), + )) + }) + It("asks an ad-hoc capture that names no connection for one in its own params", func() { _, err := telltale.StartRequest{Traces: map[string]json.RawMessage{ "sql_deadlock": json.RawMessage(`{"enabled":true}`), @@ -133,7 +193,7 @@ var _ = Describe("a start request", func() { telltale.FieldError{InstancePath: "/command", Message: "a command is required"}, And(HaveField("InstancePath", HavePrefix("/traces/"+httpproxy.Kind)), HaveField("Message", ContainSubstring(`mode "sideways"`))), And(HaveField("InstancePath", "/traces/"+pgproxy.Kind), HaveField("Message", ContainSubstring("want an object"))), - And(HaveField("InstancePath", "/traces/process_tree"), HaveField("Message", ContainSubstring("no trace definition list arms process_tree"))), + And(HaveField("InstancePath", "/traces/process_tree"), HaveField("Message", ContainSubstring(`unknown trace kind "process_tree"`))), )) }) }) diff --git a/sdk/auto/go.mod b/sdk/auto/go.mod index 8953acd..ff10e25 100644 --- a/sdk/auto/go.mod +++ b/sdk/auto/go.mod @@ -3,7 +3,7 @@ module github.com/flanksource/telltale/sdk/auto go 1.27.1 require ( - github.com/flanksource/telltale/sdk v0.0.0 + github.com/flanksource/telltale/sdk v0.0.2 github.com/onsi/ginkgo/v2 v2.32.1 github.com/onsi/gomega v1.42.1 ) @@ -11,7 +11,7 @@ require ( require ( github.com/Masterminds/semver/v3 v3.5.0 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect - github.com/flanksource/commons v1.63.0 // indirect + github.com/flanksource/commons v1.64.1 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/go-cmp v0.7.0 // indirect diff --git a/sdk/auto/go.sum b/sdk/auto/go.sum index 2856c98..08d011a 100644 --- a/sdk/auto/go.sum +++ b/sdk/auto/go.sum @@ -32,8 +32,8 @@ github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeO github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/flanksource/clicky v1.21.82 h1:YwCVk2WAZoY/tLMLmK8Hm5CA7rXZgsVS5514LbEXlNI= github.com/flanksource/clicky v1.21.82/go.mod h1:rLCgem+/vIDDC4r74hHq8WDDmzv8sGF8BIrqraDSzOY= -github.com/flanksource/commons v1.63.0 h1:gI5DecATKbay5Q1o1hYRxQUSnbC+yWP7BmdHQjrpvTY= -github.com/flanksource/commons v1.63.0/go.mod h1:RlVSQ+Jq96jKaJWf3WnwVO4DmDTdzkm3sremzJuYuVw= +github.com/flanksource/commons v1.64.1 h1:qm6ZnsoTw9JxzzH3R3WJs9kKig0+nfMBXJomWgCi/VU= +github.com/flanksource/commons v1.64.1/go.mod h1:XauWx50brQc4kK6t9OwK9JVMXy/Aq99mX6w6txASBag= github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= @@ -106,8 +106,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= -github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= diff --git a/sdk/captain/go.mod b/sdk/captain/go.mod index 69ae5e9..91f082d 100644 --- a/sdk/captain/go.mod +++ b/sdk/captain/go.mod @@ -3,15 +3,14 @@ module github.com/flanksource/telltale/sdk/captain go 1.27.1 require ( - github.com/flanksource/captain v0.0.73 - github.com/flanksource/telltale/sdk v0.0.0 + github.com/flanksource/captain v0.0.83 + github.com/flanksource/telltale/sdk v0.0.2 github.com/onsi/ginkgo/v2 v2.32.1 github.com/onsi/gomega v1.42.1 ) require ( - ariga.io/atlas v0.38.0 // indirect - cel.dev/expr v0.25.1 // indirect + cel.dev/expr v0.25.3 // indirect cloud.google.com/go v0.123.0 // indirect cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect @@ -23,15 +22,15 @@ require ( cloud.google.com/go/storage v1.62.3 // indirect dario.cat/mergo v1.0.2 // indirect filippo.io/edwards25519 v1.2.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 // indirect github.com/ClickHouse/ch-go v0.71.0 // indirect github.com/ClickHouse/clickhouse-go/v2 v2.46.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect @@ -46,7 +45,6 @@ require ( github.com/antchfx/xmlquery v1.5.1 // indirect github.com/antchfx/xpath v1.3.6 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect - github.com/atotto/clipboard v0.1.4 // indirect github.com/aws/aws-sdk-go-v2 v1.42.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.25 // indirect @@ -66,24 +64,18 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.31.3 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.43.3 // indirect - github.com/aws/smithy-go v1.27.1 // indirect + github.com/aws/smithy-go v1.27.3 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect - github.com/catppuccin/go v0.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect - github.com/cert-manager/cert-manager v1.20.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 // indirect - github.com/charmbracelet/bubbletea v1.3.10 // indirect github.com/charmbracelet/colorprofile v0.4.3 // indirect - github.com/charmbracelet/huh v1.0.0 // indirect github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 // indirect github.com/charmbracelet/x/ansi v0.11.6 // indirect github.com/charmbracelet/x/cellbuf v0.0.15 // indirect - github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect github.com/charmbracelet/x/term v0.2.2 // indirect github.com/clipperhouse/displaywidth v0.11.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect @@ -99,18 +91,17 @@ require ( github.com/eko/gocache/store/go_cache/v4 v4.2.5 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect - github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.39.1-0.20260819172001-e6e3fd93e4be // indirect github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect - github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/felixge/httpsnoop v1.1.0 // indirect - github.com/flanksource/clicky v1.21.88-0.20261027204456-34e65b99d69d // indirect - github.com/flanksource/commons v1.63.0 // indirect - github.com/flanksource/commons-db v0.1.61-0.20261027210213-a7b138ebccde // indirect + github.com/flanksource/clicky v1.21.95 // indirect + github.com/flanksource/commons v1.64.1 // indirect + github.com/flanksource/commons-db v0.1.69 // indirect github.com/flanksource/gomplate/v3 v3.24.90 // indirect - github.com/flanksource/is-healthy v1.0.92 // indirect + github.com/flanksource/is-healthy v1.0.93 // indirect github.com/flanksource/kubectl-neat v1.0.4 // indirect - github.com/flanksource/sqlglot-go v1.0.4-0.20261022123344-3264e440e925 // indirect - github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/flanksource/sqlglot-go v1.1.1 // indirect + github.com/fxamacker/cbor/v2 v2.9.1 // indirect github.com/geoffgarside/ber v1.2.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/glebarez/go-sqlite v1.23.0 // indirect @@ -119,24 +110,24 @@ require ( github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.9.0 // indirect github.com/go-git/go-git/v5 v5.19.2 // indirect - github.com/go-jose/go-jose/v4 v4.1.4 // indirect - github.com/go-logr/logr v1.4.3 // indirect + github.com/go-jose/go-jose/v4 v4.1.9 // indirect + github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect - github.com/go-openapi/jsonpointer v0.22.5 // indirect - github.com/go-openapi/jsonreference v0.21.4 // indirect - github.com/go-openapi/swag v0.25.4 // indirect - github.com/go-openapi/swag/cmdutils v0.25.4 // indirect - github.com/go-openapi/swag/conv v0.25.4 // indirect - github.com/go-openapi/swag/fileutils v0.25.4 // indirect - github.com/go-openapi/swag/jsonname v0.25.5 // indirect - github.com/go-openapi/swag/jsonutils v0.25.4 // indirect - github.com/go-openapi/swag/loading v0.25.4 // indirect - github.com/go-openapi/swag/mangling v0.25.4 // indirect - github.com/go-openapi/swag/netutils v0.25.4 // indirect - github.com/go-openapi/swag/stringutils v0.25.4 // indirect - github.com/go-openapi/swag/typeutils v0.25.4 // indirect - github.com/go-openapi/swag/yamlutils v0.25.4 // indirect + github.com/go-openapi/jsonpointer v1.0.0 // indirect + github.com/go-openapi/jsonreference v1.0.0 // indirect + github.com/go-openapi/swag v0.28.0 // indirect + github.com/go-openapi/swag/cmdutils v0.28.0 // indirect + github.com/go-openapi/swag/conv v0.28.0 // indirect + github.com/go-openapi/swag/fileutils v0.28.0 // indirect + github.com/go-openapi/swag/jsonutils v0.28.0 // indirect + github.com/go-openapi/swag/loading v0.28.0 // indirect + github.com/go-openapi/swag/mangling v0.28.0 // indirect + github.com/go-openapi/swag/netutils v0.28.0 // indirect + github.com/go-openapi/swag/pools v0.28.0 // indirect + github.com/go-openapi/swag/stringutils v0.28.0 // indirect + github.com/go-openapi/swag/typeutils v0.28.0 // indirect + github.com/go-openapi/swag/yamlutils v0.28.0 // indirect github.com/go-sql-driver/mysql v1.10.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/go-xmlfmt/xmlfmt v1.1.3 // indirect @@ -149,22 +140,20 @@ require ( github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/google/btree v1.1.3 // indirect github.com/google/cel-go v0.31.0 // indirect - github.com/google/dotprompt/go v0.0.0-20260708220100-73beb993ac95 // indirect github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect + github.com/google/pprof v0.0.0-20260507013755-92041b743c96 // indirect github.com/google/s2a-go v0.1.9 // indirect - github.com/google/uuid v1.6.0 // indirect + github.com/google/uuid v1.6.1-0.20241114170450-2d3c2a9cc518 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.17 // indirect github.com/googleapis/gax-go/v2 v2.22.0 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/gosimple/slug v1.15.0 // indirect github.com/gosimple/unidecode v1.0.1 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/hairyhenderson/toml v0.4.2-0.20210923231440-40456b8e66cf // indirect github.com/hairyhenderson/yaml v0.0.0-20220618171115-2d35fca545ce // indirect github.com/henvic/httpretty v0.1.4 // indirect - github.com/hexops/gotextdiff v1.0.3 // indirect github.com/hirochachacha/go-smb2 v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/invopop/jsonschema v0.14.0 // indirect @@ -187,7 +176,7 @@ require ( github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/kr/fs v0.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/labstack/echo/v4 v4.15.2 // indirect + github.com/labstack/echo/v4 v4.16.4 // indirect github.com/labstack/gommon v0.5.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect @@ -195,19 +184,14 @@ require ( github.com/lib/pq v1.12.3 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect - github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-colorable v0.1.15 // indirect github.com/mattn/go-isatty v0.0.22 // indirect - github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.21 // indirect - github.com/mbleigh/raymond v0.0.0-20250414171441-6b3a58ab9e0a // indirect github.com/microsoft/go-mssqldb v1.10.0 // indirect - github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/spdystream v0.5.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect - github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect - github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect @@ -215,7 +199,7 @@ require ( github.com/oklog/ulid/v2 v2.1.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect - github.com/patrickmn/go-cache v2.1.0+incompatible // indirect + github.com/patrickmn/go-cache v2.1.1-0.20191004192108-46f407853014+incompatible // indirect github.com/paulmach/orb v0.12.0 // indirect github.com/pb33f/ordered-map/v2 v2.3.1 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect @@ -224,7 +208,7 @@ require ( github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pkg/sftp v1.13.10 // indirect - github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/prometheus/client_golang v1.23.2 // indirect @@ -245,7 +229,7 @@ require ( github.com/skeema/knownhosts v1.3.2 // indirect github.com/spf13/cobra v1.10.2 // indirect github.com/spf13/pflag v1.0.10 // indirect - github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect + github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.2.0 // indirect github.com/tidwall/pretty v1.2.1 // indirect @@ -268,40 +252,41 @@ require ( github.com/yuin/gopher-lua v1.1.1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.43.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect - go.opentelemetry.io/otel v1.44.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 // indirect - go.opentelemetry.io/otel/metric v1.44.0 // indirect - go.opentelemetry.io/otel/sdk v1.44.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect - go.opentelemetry.io/otel/trace v1.44.0 // indirect - go.opentelemetry.io/proto/otlp v1.10.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.45.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect + go.opentelemetry.io/otel v1.47.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 // indirect + go.opentelemetry.io/otel/log v1.47.0 // indirect + go.opentelemetry.io/otel/metric v1.47.0 // indirect + go.opentelemetry.io/otel/sdk v1.47.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.47.0 // indirect + go.opentelemetry.io/otel/trace v1.47.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect go.yaml.in/yaml/v4 v4.0.0-rc.4 // indirect gocloud.dev v0.46.0 // indirect - golang.org/x/crypto v0.53.0 // indirect + golang.org/x/crypto v0.58.0 // indirect golang.org/x/exp v0.0.0-20260527015227-08cc5374adb3 // indirect - golang.org/x/mod v0.37.0 // indirect - golang.org/x/net v0.56.0 // indirect + golang.org/x/mod v0.41.0 // indirect + golang.org/x/net v0.62.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.22.0 // indirect - golang.org/x/sys v0.46.0 // indirect - golang.org/x/term v0.44.0 // indirect - golang.org/x/text v0.40.0 // indirect + golang.org/x/sync v0.24.0 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/term v0.47.0 // indirect + golang.org/x/text v0.43.0 // indirect golang.org/x/time v0.15.0 // indirect - golang.org/x/tools v0.47.0 // indirect + golang.org/x/tools v0.51.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/api v0.286.0 // indirect google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260622175928-b703f567277d // indirect - google.golang.org/grpc v1.82.1 // indirect - google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/grpc v1.85.0 // indirect + google.golang.org/protobuf v1.36.12 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect @@ -309,23 +294,21 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gorm.io/gorm v1.31.2 // indirect - k8s.io/api v0.36.1 // indirect - k8s.io/apiextensions-apiserver v0.36.1 // indirect + k8s.io/api v0.36.2 // indirect k8s.io/apimachinery v0.36.2 // indirect - k8s.io/client-go v0.36.1 // indirect + k8s.io/client-go v0.36.2 // indirect k8s.io/klog/v2 v2.140.0 // indirect - k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect + k8s.io/kube-openapi v0.0.0-20260501160325-927ab1f70cd6 // indirect k8s.io/streaming v0.36.2 // indirect - k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect + k8s.io/utils v0.0.0-20260626114624-be93311217bd // indirect layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf // indirect modernc.org/libc v1.74.1 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect modernc.org/sqlite v1.55.0 // indirect - sigs.k8s.io/gateway-api v1.5.1 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.4.0 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/sdk/captain/go.sum b/sdk/captain/go.sum index dc70891..575ee10 100644 --- a/sdk/captain/go.sum +++ b/sdk/captain/go.sum @@ -1,15 +1,11 @@ -ariga.io/atlas v0.38.0 h1:MwbtwVtDWJFq+ECyeTAz2ArvewDnpeiw/t/sgNdDsdo= -ariga.io/atlas v0.38.0/go.mod h1:D7XMK6ei3GvfDqvzk+2VId78j77LdqHrqPOWamn51/s= -cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= -cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0= +cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= -cloud.google.com/go/cloudsqlconn v1.22.1 h1:c4HkWMSV4pDL8kJid+nIuF+6iE07pR5Mn/sRVY17wf4= -cloud.google.com/go/cloudsqlconn v1.22.1/go.mod h1:p7l+u0ThOzSvC5a4fkywi1hEyD8S709X1zEqux9tsq0= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= @@ -30,12 +26,12 @@ dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 h1:aokoqcHvaGjiM3VpjKDfMMnF/8epJ+Q1HLJ7CudztqE= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0/go.mod h1:/WYEx9pcM9Y+Dd/APJaNlSvVSvzl54rrMdZT5+Oi2LM= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= -github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= -github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 h1:zvXfGJCWvywnCA814d8ZiVyt+fm9nnTE8xSb99zRyfo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 h1:u93s+zU2JD62im61Bm5CZIc1ZrOJaIAWEg0WOrMVkEo= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1/go.mod h1:oXtinPO4OLj9d1DOTrqrL1oRwGhcqadvAmrl6wTeGlk= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0 h1:xFaZZ+IubdftrDHnGGwZ6QvQ3KHTtWl2MCK+GMt2vxs= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0/go.mod h1:mCBhUhlMjLLJKr5aqw2TNS/VqJOie8MzWq3DAMJeKso= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 h1:m/sWOGCREuSBqg2htVQTBY8nOZpyajYztF0vUvSZTuM= @@ -48,24 +44,22 @@ github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfg github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= -github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= -github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI= github.com/ClickHouse/ch-go v0.71.0 h1:bUdZ/EZj/LcVHsMqaRUP2holqygrPWQKeMjc6nZoyRM= github.com/ClickHouse/ch-go v0.71.0/go.mod h1:NwbNc+7jaqfY58dmdDUbG4Jl22vThgx1cYjBw0vtgXw= github.com/ClickHouse/clickhouse-go/v2 v2.46.0 h1:s3eRy+hYmu5uzotB6ZhDofgHu8kDgGN/fpmjxRkqSpk= github.com/ClickHouse/clickhouse-go/v2 v2.46.0/go.mod h1:giJfUVlMkcfUEPVfRpt51zZaGEx9i17gCos8gBl392c= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 h1:rIkQfkCOVKc1OiRCNcSDD8ml5RJlZbH/Xsq7lbpynwc= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0/go.mod h1:RD2SsorTmYhF6HkTmDw7KmPYQk8OBYwTkuasChwv7R4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 h1:bN1gA3of5bXtbnLsRPrwfmbbe7A5UWFlcTHseujLnpc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0/go.mod h1:Yj5vHEz/aAepZGliRJsA6uvHAVAQyEwajq9ORCHPxzM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= -github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= -github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= @@ -79,8 +73,6 @@ github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBi github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE= github.com/TomOnTime/utfutil v1.0.0 h1:/0Ivgo2OjXJxo8i7zgvs7ewSFZMLwCRGm3P5Umowb90= github.com/TomOnTime/utfutil v1.0.0/go.mod h1:l9lZmOniizVSuIliSkEf87qivMRlSNzbdBFKjuLRg1c= -github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= -github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/chroma/v2 v2.23.1 h1:nv2AVZdTyClGbVQkIzlDm/rnhk1E9bU9nXwmZ/Vk/iY= @@ -99,8 +91,6 @@ github.com/antchfx/xpath v1.3.6 h1:s0y+ElRRtTQdfHP609qFu0+c6bglDv20pqOViQjjdPI= github.com/antchfx/xpath v1.3.6/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= -github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= -github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= @@ -143,8 +133,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6 h1:yLr03zQE/5Eu5l3QU0Si+xMb github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6/go.mod h1:Q5N6icH+KJZDLh+ESNwzdv6cZ6vLFF/egy3IOxWhmz4= github.com/aws/aws-sdk-go-v2/service/sts v1.43.3 h1:VrIhKRCSK1umelSgB9RghvA9RTUYeQffyAS5ApXehNI= github.com/aws/aws-sdk-go-v2/service/sts v1.43.3/go.mod h1:r8wkDOuLaaMFqFiYAb8dGY2A3gJCOujMc6CFOVC4Zhc= -github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= -github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.27.3 h1:F3Zb497UhhskkfpJmfkXswyo+t0sh9OTBnIHjogWbVY= +github.com/aws/smithy-go v1.27.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymanbagabas/go-udiff v0.4.1 h1:OEIrQ8maEeDBXQDoGCbbTTXYJMYRCRO1fnodZ12Gv5o= @@ -153,8 +143,6 @@ github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPn github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= -github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs= github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -165,8 +153,6 @@ github.com/catppuccin/go v0.3.0 h1:d+0/YicIq+hSTo5oPuRi5kOpqkVA5tAsU6dNhvRu+aY= github.com/catppuccin/go v0.3.0/go.mod h1:8IHJuMGaUUjQM82qBrGNBv7LFq6JI3NnQCF6MOlZjpc= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= -github.com/cert-manager/cert-manager v1.20.0 h1:czZamsFJ1YdKPhpv+SopJZN9t3W68vQBnFYUevSHGqY= -github.com/cert-manager/cert-manager v1.20.0/go.mod h1:3oparI7R5JyJMNXntYod9p6DucIZ84st7y/9kL6cbBE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 h1:JFgG/xnwFfbezlUnFMJy0nusZvytYysV4SCS2cYbvws= @@ -183,20 +169,12 @@ github.com/charmbracelet/x/ansi v0.11.6 h1:GhV21SiDz/45W9AnV2R61xZMRri5NlLnl6CVF github.com/charmbracelet/x/ansi v0.11.6/go.mod h1:2JNYLgQUsyqaiLovhU2Rv/pb8r6ydXKS3NIttu3VGZQ= github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI= github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q= -github.com/charmbracelet/x/conpty v0.1.0 h1:4zc8KaIcbiL4mghEON8D72agYtSeIgq8FSThSPQIb+U= -github.com/charmbracelet/x/conpty v0.1.0/go.mod h1:rMFsDJoDwVmiYM10aD4bH2XiRgwI7NYJtQgl5yskjEQ= -github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86 h1:JSt3B+U9iqk37QUU2Rvb6DSBYRLtWqFqfxf8l5hOZUA= -github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86/go.mod h1:2P0UgXMEa6TsToMSuFqKFQR+fZTO9CNGUNokkPatT/0= github.com/charmbracelet/x/exp/golden v0.0.0-20250806222409-83e3a29d542f h1:pk6gmGpCE7F3FcjaOEKYriCvpmIN4+6OS/RD0vm4uIA= github.com/charmbracelet/x/exp/golden v0.0.0-20250806222409-83e3a29d542f/go.mod h1:IfZAMTHB6XkZSeXUqriemErjAWCCzT0LwjKFYCZyw0I= -github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 h1:qko3AQ4gK1MTS/de7F5hPGx6/k1u0w4TeYmBFwzYVP4= -github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0/go.mod h1:pBhA0ybfXv6hDjQUZ7hk1lVxBiUbupdw5R31yPUViVQ= +github.com/charmbracelet/x/exp/strings v0.0.0-20260122224438-b01af16209d9 h1:JevRYfkTT0sN9OIXAOncYNC0cTP1Gml/0mCSnsmRkRk= +github.com/charmbracelet/x/exp/strings v0.0.0-20260122224438-b01af16209d9/go.mod h1:/ehtMPNh9K4odGFkqYJKpIYyePhdp1hLBRvyY4bWkH8= github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= -github.com/charmbracelet/x/termios v0.1.1 h1:o3Q2bT8eqzGnGPOYheoYS8eEleT5ZVNYNy8JawjaNZY= -github.com/charmbracelet/x/termios v0.1.1/go.mod h1:rB7fnv1TgOPOyyKRJ9o+AsTU/vK5WHJ2ivHeut/Pcwo= -github.com/charmbracelet/x/xpty v0.1.3 h1:eGSitii4suhzrISYH50ZfufV3v085BXQwIytcOdFSsw= -github.com/charmbracelet/x/xpty v0.1.3/go.mod h1:poPYpWuLDBFCKmKLDnhBp51ATa0ooD8FhypRwEFtH3Y= github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8= github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0= github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= @@ -234,38 +212,34 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= -github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= -github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/envoy v1.39.1-0.20260819172001-e6e3fd93e4be h1:SWe0x6yfglnxuvOiYgTTnNq7QD/yvqthOh1RBI8Bj8w= +github.com/envoyproxy/go-control-plane/envoy v1.39.1-0.20260819172001-e6e3fd93e4be/go.mod h1:PYEOlng9XcrulfyWpm49jECTPV0LT4q8cO7fLW/xwgk= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= -github.com/exaring/otelpgx v0.11.1 h1:pE79fIg/qh/Lpu00kvswFC5dKfqyJJhMJ4Y4N3w5Lj4= -github.com/exaring/otelpgx v0.11.1/go.mod h1:3OojrUKhhy3lTbYIMBijP3YjMey/jo14eHAW5cXcUdk= github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= -github.com/fergusstrange/embedded-postgres v1.34.0 h1:c6RKhPKFsLVU+Tdxsx8q0UxCHsvZZ/iShAnljRBXs6s= -github.com/fergusstrange/embedded-postgres v1.34.0/go.mod h1:w0YvnCgf19o6tskInrOOACtnqfVlOvluz3hlNLY7tRk= -github.com/flanksource/captain v0.0.73 h1:kxV+OkhanWTr0DcnWeUlALHSSY8pn1FNg5cq39JBkuw= -github.com/flanksource/captain v0.0.73/go.mod h1:M6+b1SjeyS88NSrLZKrzuVt/LHUUH/hqzDCcwVKLKpY= -github.com/flanksource/clicky v1.21.88-0.20261027204456-34e65b99d69d h1:S8zFjrGZZj+HoOPt0YnUMgOt7YvLY1T4v9ifpsQphW8= -github.com/flanksource/clicky v1.21.88-0.20261027204456-34e65b99d69d/go.mod h1:2hh1pw/svkWomN+Ah946CD7zcwuGxvRSnR0PTSqMe/Y= -github.com/flanksource/commons v1.63.0 h1:gI5DecATKbay5Q1o1hYRxQUSnbC+yWP7BmdHQjrpvTY= -github.com/flanksource/commons v1.63.0/go.mod h1:RlVSQ+Jq96jKaJWf3WnwVO4DmDTdzkm3sremzJuYuVw= -github.com/flanksource/commons-db v0.1.61-0.20261027210213-a7b138ebccde h1:QfMh6xZcg9/ksRl5k1Zpd1Enov932Z/xltj8hbRbLkY= -github.com/flanksource/commons-db v0.1.61-0.20261027210213-a7b138ebccde/go.mod h1:81yea1KIjO52xC2uoehwTd8BfnxSN+S6aoGYVDLuLp4= +github.com/flanksource/captain v0.0.83 h1:QeHvs/OodVPethKOFpN6IuEiyrkoYngSaolRSWLL5Lc= +github.com/flanksource/captain v0.0.83/go.mod h1:6Hg3PYn8rARRpYpi8E+EWCO5kgFrEeLD6xNaexTCkpE= +github.com/flanksource/clicky v1.21.95 h1:cBuTKkvbWHjpDRswJ0iOPJzNT5QYiyqt31qmJOA7rBg= +github.com/flanksource/clicky v1.21.95/go.mod h1:H4HeUqMEzAz14ewqX5CNu/VnlaZHrTK/1otCgs1O1m4= +github.com/flanksource/commons v1.64.1 h1:qm6ZnsoTw9JxzzH3R3WJs9kKig0+nfMBXJomWgCi/VU= +github.com/flanksource/commons v1.64.1/go.mod h1:XauWx50brQc4kK6t9OwK9JVMXy/Aq99mX6w6txASBag= +github.com/flanksource/commons-db v0.1.69 h1:oxNc0ChQqASBEA4JaVJGPirOV82ZltDJ1mxQmNX9ZR4= +github.com/flanksource/commons-db v0.1.69/go.mod h1:grmVCAlfxqk8OuQRi1Jmgs9k3QELNVCDDJl3I3EsNAQ= github.com/flanksource/gomplate/v3 v3.24.90 h1:MAUlwU7REhSL62i4n74yvo+T4LA7HuRpJ9tKH9dbMzk= github.com/flanksource/gomplate/v3 v3.24.90/go.mod h1:tcjpBIaHboWVPUYt973dGLCWgTD1sOHdiFgA/AaPBPQ= -github.com/flanksource/is-healthy v1.0.92 h1:15lC+p1RmyO1kOdZdpHdheqysgIK+hQD2ftSQ64gDg0= -github.com/flanksource/is-healthy v1.0.92/go.mod h1:Gm46NfdDQsK9yDZNUNvkCm7jNnbl8R5vfcUYG3BgPXg= +github.com/flanksource/is-healthy v1.0.93 h1:hpXkPfMXG+zawNs0HTjc/3ZIjrW7obAPDR/VoTJk59Q= +github.com/flanksource/is-healthy v1.0.93/go.mod h1:Ee0lzss7zeLlfYOcr4r3eOUJqTD3fjVnxT5cv79UVF8= github.com/flanksource/kubectl-neat v1.0.4 h1:t5/9CqgE84oEtB0KitgJ2+WIeLfD+RhXSxYrqb4X8yI= github.com/flanksource/kubectl-neat v1.0.4/go.mod h1:Un/Voyh3cmiZNKQrW/TkAl28nAA7vwnwDGVjRErKjOw= -github.com/flanksource/sqlglot-go v1.0.4-0.20261022123344-3264e440e925 h1:sQGUeHC+LP/rhh9lNBbQ9our+JYY1cgHQYPSTZKiMSw= -github.com/flanksource/sqlglot-go v1.0.4-0.20261022123344-3264e440e925/go.mod h1:lLkHKgcKwoLDcZ+VchYEp3mNic7fJml4bc8HXyh4vzY= -github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= -github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/flanksource/sqlglot-go v1.1.1 h1:9AlEb41+Ll+W/PQwl7ctZMKo3phXF5f2jznsAcUmt9Y= +github.com/flanksource/sqlglot-go v1.1.1/go.mod h1:aDnAJW6L2s2EXgiM8blbZ+ldV76Rdyu87KkPZJxFQIY= +github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ= +github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64= github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= @@ -295,52 +269,50 @@ github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMj github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY= github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuwuB/jWEHjk= -github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= -github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-jose/go-jose/v4 v4.1.9 h1:pKfeQkig1NmgK4hJrZYXLEYURXfoJB61Larw0uIW4gY= +github.com/go-jose/go-jose/v4 v4.1.9/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= -github.com/go-openapi/inflect v0.21.5 h1:M2RCq6PPS3YbIaL7CXosGL3BbzAcmfBAT0nC3YfesZA= -github.com/go-openapi/inflect v0.21.5/go.mod h1:GypUyi6bU880NYurWaEH2CmH84zFDNd+EhhmzroHmB4= -github.com/go-openapi/jsonpointer v0.22.5 h1:8on/0Yp4uTb9f4XvTrM2+1CPrV05QPZXu+rvu2o9jcA= -github.com/go-openapi/jsonpointer v0.22.5/go.mod h1:gyUR3sCvGSWchA2sUBJGluYMbe1zazrYWIkWPjjMUY0= -github.com/go-openapi/jsonreference v0.21.4 h1:24qaE2y9bx/q3uRK/qN+TDwbok1NhbSmGjjySRCHtC8= -github.com/go-openapi/jsonreference v0.21.4/go.mod h1:rIENPTjDbLpzQmQWCj5kKj3ZlmEh+EFVbz3RTUh30/4= -github.com/go-openapi/swag v0.25.4 h1:OyUPUFYDPDBMkqyxOTkqDYFnrhuhi9NR6QVUvIochMU= -github.com/go-openapi/swag v0.25.4/go.mod h1:zNfJ9WZABGHCFg2RnY0S4IOkAcVTzJ6z2Bi+Q4i6qFQ= -github.com/go-openapi/swag/cmdutils v0.25.4 h1:8rYhB5n6WawR192/BfUu2iVlxqVR9aRgGJP6WaBoW+4= -github.com/go-openapi/swag/cmdutils v0.25.4/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0= -github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4= -github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU= -github.com/go-openapi/swag/fileutils v0.25.4 h1:2oI0XNW5y6UWZTC7vAxC8hmsK/tOkWXHJQH4lKjqw+Y= -github.com/go-openapi/swag/fileutils v0.25.4/go.mod h1:cdOT/PKbwcysVQ9Tpr0q20lQKH7MGhOEb6EwmHOirUk= -github.com/go-openapi/swag/jsonname v0.25.5 h1:8p150i44rv/Drip4vWI3kGi9+4W9TdI3US3uUYSFhSo= -github.com/go-openapi/swag/jsonname v0.25.5/go.mod h1:jNqqikyiAK56uS7n8sLkdaNY/uq6+D2m2LANat09pKU= -github.com/go-openapi/swag/jsonutils v0.25.4 h1:VSchfbGhD4UTf4vCdR2F4TLBdLwHyUDTd1/q4i+jGZA= -github.com/go-openapi/swag/jsonutils v0.25.4/go.mod h1:7OYGXpvVFPn4PpaSdPHJBtF0iGnbEaTk8AvBkoWnaAY= -github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4 h1:IACsSvBhiNJwlDix7wq39SS2Fh7lUOCJRmx/4SN4sVo= -github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4/go.mod h1:Mt0Ost9l3cUzVv4OEZG+WSeoHwjWLnarzMePNDAOBiM= -github.com/go-openapi/swag/loading v0.25.4 h1:jN4MvLj0X6yhCDduRsxDDw1aHe+ZWoLjW+9ZQWIKn2s= -github.com/go-openapi/swag/loading v0.25.4/go.mod h1:rpUM1ZiyEP9+mNLIQUdMiD7dCETXvkkC30z53i+ftTE= -github.com/go-openapi/swag/mangling v0.25.4 h1:2b9kBJk9JvPgxr36V23FxJLdwBrpijI26Bx5JH4Hp48= -github.com/go-openapi/swag/mangling v0.25.4/go.mod h1:6dxwu6QyORHpIIApsdZgb6wBk/DPU15MdyYj/ikn0Hg= -github.com/go-openapi/swag/netutils v0.25.4 h1:Gqe6K71bGRb3ZQLusdI8p/y1KLgV4M/k+/HzVSqT8H0= -github.com/go-openapi/swag/netutils v0.25.4/go.mod h1:m2W8dtdaoX7oj9rEttLyTeEFFEBvnAx9qHd5nJEBzYg= -github.com/go-openapi/swag/stringutils v0.25.4 h1:O6dU1Rd8bej4HPA3/CLPciNBBDwZj9HiEpdVsb8B5A8= -github.com/go-openapi/swag/stringutils v0.25.4/go.mod h1:GTsRvhJW5xM5gkgiFe0fV3PUlFm0dr8vki6/VSRaZK0= -github.com/go-openapi/swag/typeutils v0.25.4 h1:1/fbZOUN472NTc39zpa+YGHn3jzHWhv42wAJSN91wRw= -github.com/go-openapi/swag/typeutils v0.25.4/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE= -github.com/go-openapi/swag/yamlutils v0.25.4 h1:6jdaeSItEUb7ioS9lFoCZ65Cne1/RZtPBZ9A56h92Sw= -github.com/go-openapi/swag/yamlutils v0.25.4/go.mod h1:MNzq1ulQu+yd8Kl7wPOut/YHAAU/H6hL91fF+E2RFwc= -github.com/go-openapi/testify/enable/yaml/v2 v2.4.0 h1:7SgOMTvJkM8yWrQlU8Jm18VeDPuAvB/xWrdxFJkoFag= -github.com/go-openapi/testify/enable/yaml/v2 v2.4.0/go.mod h1:14iV8jyyQlinc9StD7w1xVPW3CO3q1Gj04Jy//Kw4VM= -github.com/go-openapi/testify/v2 v2.4.0 h1:8nsPrHVCWkQ4p8h1EsRVymA2XABB4OT40gcvAu+voFM= -github.com/go-openapi/testify/v2 v2.4.0/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= +github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s= +github.com/go-openapi/jsonpointer v1.0.0/go.mod h1:Z3rw7dWu1p9IgitXCFamSlA5lmDiklEB6vkaxcNZW5Y= +github.com/go-openapi/jsonreference v1.0.0 h1:jlmTr6torcd1YgDQvSfNmRtKzYDO4FGBkrAdlAVWnpY= +github.com/go-openapi/jsonreference v1.0.0/go.mod h1:jtwdyGbJk0Xhe5Y+rwtglQP6Sb1WZST4rT32LWB+sv0= +github.com/go-openapi/swag v0.28.0 h1:xkgbOSKj6DZziNpyqRRAOt3GJGtgjgsd2RoyT30VWuw= +github.com/go-openapi/swag v0.28.0/go.mod h1:4qYnT3Cqr1p1VknOdPo70evN4rgQnAg6jwApHyxSGIg= +github.com/go-openapi/swag/cmdutils v0.28.0 h1:7TOeNtkYru1SG8Y34tDh9WBbLsMqGnptuxWiHREPZ4Q= +github.com/go-openapi/swag/cmdutils v0.28.0/go.mod h1:Sm1MVFMkF6guJJ+pQqHnQA3N0j9qALV3NxzDSv6bETM= +github.com/go-openapi/swag/conv v0.28.0 h1:GtqqbyFe7vR5Y7ehxG9W6/OvrSFdf1OLeTGp40TqxH8= +github.com/go-openapi/swag/conv v0.28.0/go.mod h1:mbUE+mzctnhxi864m0Q07SpN8OowD9JhxmxuYvZZD/k= +github.com/go-openapi/swag/fileutils v0.28.0 h1:Z04XWQD7R8Eq+7GnOrjovBxPPmZzsS4gt2H2GPGIViU= +github.com/go-openapi/swag/fileutils v0.28.0/go.mod h1:VvJFZLTZS0AI854gEQz5tk7dBESdLjiNUMSZ/th2ry8= +github.com/go-openapi/swag/jsonutils v0.28.0 h1:YIch6FwO7RXzeAnbO8Tu7dWBZeUEH+4nA0HXltVTnv4= +github.com/go-openapi/swag/jsonutils v0.28.0/go.mod h1:CYM3WlTUcagR2ZoHdz54di/cbBqt82tuxuXgAjxw+mg= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.28.0 h1:qV+VVUAx5Oro8WjVWpZeql7YReTKhT4smR4zhcOQZr0= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.28.0/go.mod h1:mofwUWx70wvskwESqRJ//k/9kURmCgyJl5m5Ppoh5kY= +github.com/go-openapi/swag/loading v0.28.0 h1:td8QZdZC9MIYGGSnSPKShKiK22I2tU5UQvuUhIBPRLU= +github.com/go-openapi/swag/loading v0.28.0/go.mod h1:rXB0QiQX5mMveXEA7ouM4KiiM9jVJe4K6BVbwhD1M4k= +github.com/go-openapi/swag/mangling v0.28.0 h1:pH8eyeNO9SLYsTMWJrurnNfKmDa28XrlA+HePVD53VM= +github.com/go-openapi/swag/mangling v0.28.0/go.mod h1:jtBE2+V+3pILxOR7Vgce+Cwp6A2PgZbvVqfNntbVs0w= +github.com/go-openapi/swag/netutils v0.28.0 h1:YXN6TALEi2pzts8/8GNm6T61HTAZsieukGZidap989k= +github.com/go-openapi/swag/netutils v0.28.0/go.mod h1:J+WYyFMLtvtCGqa6jLv+YNUmIKI3ZRQRrvfNDMoQoEQ= +github.com/go-openapi/swag/pools v0.28.0 h1:HPMZWSAfce3rdVTFcjFiCIBtDg9h4x2QlRrHipwhxeU= +github.com/go-openapi/swag/pools v0.28.0/go.mod h1:kVQefhSK5RWuRe7BXsL8htgBPAMpN7HDGpGEknqugeE= +github.com/go-openapi/swag/stringutils v0.28.0 h1:ixsc9iYgDPubHL/8nSkbnryEHpD2VRlBMLKpQyPXcDU= +github.com/go-openapi/swag/stringutils v0.28.0/go.mod h1:lzRN95CxXmA03XcDWHLOb6nOMcxCqR5rGY0lOgsfRoM= +github.com/go-openapi/swag/typeutils v0.28.0 h1:nRBKSBXjDgf01VDPB3fWeD9nQuhCOVeIYAkUx2tbkyY= +github.com/go-openapi/swag/typeutils v0.28.0/go.mod h1:Srm0xFNRZ1Y+vCxJclo5qzx8aj+1pAKda/YfFPrG0dQ= +github.com/go-openapi/swag/yamlutils v0.28.0 h1:TV3JXH6DS46KUroDtMLAYHGkdWf5VDq3wVWFirmzROY= +github.com/go-openapi/swag/yamlutils v0.28.0/go.mod h1:x0q/yndZHEgk9Rx3DyDqzFUmHy55KTvIZldvF2dTJXs= +github.com/go-openapi/testify/enable/yaml/v2 v2.6.0 h1:gGHwAJ0R/5jU8BEGDbfRNR3hL68dAVi84WuOApp29B0= +github.com/go-openapi/testify/enable/yaml/v2 v2.6.0/go.mod h1:tY+St1SGq4NFl0QIqdTY4aEdbChAHxhyB77XQi9iJCo= +github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug= +github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw= github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= @@ -371,8 +343,6 @@ github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/cel-go v0.31.0 h1:H0bhpFTqOvmHrBGrWKp7ZlhBm5Hh8PYUEXnwxT1LL7A= github.com/google/cel-go v0.31.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= -github.com/google/dotprompt/go v0.0.0-20260708220100-73beb993ac95 h1:SJdnmyOaT+kZNcUR+a1y2+Oa51j2ctCjYbtexaiXN68= -github.com/google/dotprompt/go v0.0.0-20260708220100-73beb993ac95/go.mod h1:dnlL7KrFwJ7s8EJdsAp1WdLqOalJq1Sx2jWZnQkhFXs= github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -383,12 +353,12 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= -github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg= -github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/pprof v0.0.0-20260507013755-92041b743c96 h1:YDDnaZ9afWajDboPMt9Vikqca/yWAX7KAxVzb4lJU1M= +github.com/google/pprof v0.0.0-20260507013755-92041b743c96/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.1-0.20241114170450-2d3c2a9cc518 h1:UBg1xk+oAsIVbFuGg6hdfAm7EvCv3EL80vFxJNsslqw= +github.com/google/uuid v1.6.1-0.20241114170450-2d3c2a9cc518/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4= github.com/google/wire v0.7.0/go.mod h1:n6YbUQD9cPKTnHXEBN2DXlOp/mVADhVErcMFb0v3J18= github.com/googleapis/enterprise-certificate-proxy v0.3.17 h1:73NfMHdiqo9JFU9+7a5ExpVa10/R29pXfZIaW559nrg= @@ -401,17 +371,14 @@ github.com/gosimple/slug v1.15.0 h1:wRZHsRrRcs6b0XnxMUBM6WK1U1Vg5B0R7VkIf1Xzobo= github.com/gosimple/slug v1.15.0/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ= github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o= github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= github.com/hairyhenderson/toml v0.4.2-0.20210923231440-40456b8e66cf h1:I1sbT4ZbIt9i+hB1zfKw2mE8C12TuGxPiW7YmtLbPa4= github.com/hairyhenderson/toml v0.4.2-0.20210923231440-40456b8e66cf/go.mod h1:jDHmWDKZY6MIIYltYYfW4Rs7hQ50oS4qf/6spSiZAxY= github.com/hairyhenderson/yaml v0.0.0-20220618171115-2d35fca545ce h1:cVkYhlWAxwuS2/Yp6qPtcl0fGpcWxuZNonywHZ6/I+s= github.com/hairyhenderson/yaml v0.0.0-20220618171115-2d35fca545ce/go.mod h1:7TyiGlHI+IO+iJbqRZ82QbFtvgj/AIcFm5qc9DLn7Kc= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I= -github.com/hashicorp/hcl/v2 v2.24.0 h1:2QJdZ454DSsYGoaE6QheQZjtKZSUs9Nh2izTWiwQxvE= -github.com/hashicorp/hcl/v2 v2.24.0/go.mod h1:oGoO1FIQYfn/AgyOhlg9qLC6/nOJPX3qGbkZpYAcqfM= github.com/henvic/httpretty v0.1.4 h1:Jo7uwIRWVFxkqOnErcoYfH90o3ddQyVrSANeS4cxYmU= github.com/henvic/httpretty v0.1.4/go.mod h1:Dn60sQTZfbt2dYsdUSNsCljyF4AfdqnuJFDLJA1I4AM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -479,8 +446,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/labstack/echo/v4 v4.15.2 h1:nnh2sCzGCVYnU+wCisMPiYapEg/QVo/gcI9ePKg5/T4= -github.com/labstack/echo/v4 v4.15.2/go.mod h1:Xzp1Ns1RA2c9fY7nSgUJkpkUZGNbEIVHZbtbOMPktBI= +github.com/labstack/echo/v4 v4.16.4 h1:sQ9yq/FgkY7zWZKaRIvNJAQ1ybWVJrm+z37SWI8keRY= +github.com/labstack/echo/v4 v4.16.4/go.mod h1:V/EQaTVHYJH92QhbQe7kFQ4mjgcjxLQQUQtAN87zZdw= github.com/labstack/gommon v0.5.0 h1:6VSQ2NOzsnEJ5W6+84E0RbcaDDmgB6NIAzWCczTEe6c= github.com/labstack/gommon v0.5.0/go.mod h1:Rzlg7HHy1maLfzBYGg9NZcVuz1sA68HHhLjhcEllYE0= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= @@ -497,8 +464,8 @@ github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 h1:PwQumkgq4/acIi github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= -github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= -github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= @@ -507,14 +474,10 @@ github.com/mattn/go-runewidth v0.0.21 h1:jJKAZiQH+2mIinzCJIaIG9Be1+0NR+5sz/lYEEj github.com/mattn/go-runewidth v0.0.21/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.14.38 h1:tDUzL85kMvOrvpCt8P64SbGgVFtJB11GPi2AdmITgb4= github.com/mattn/go-sqlite3 v1.14.38/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/mbleigh/raymond v0.0.0-20250414171441-6b3a58ab9e0a h1:v2cBA3xWKv2cIOVhnzX/gNgkNXqiHfUgJtA3r61Hf7A= -github.com/mbleigh/raymond v0.0.0-20250414171441-6b3a58ab9e0a/go.mod h1:Y6ghKH+ZijXn5d9E7qGGZBmjitx7iitZdQiIW97EpTU= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/microsoft/go-mssqldb v1.10.0 h1:pHEt+Qz6YFPWqREq10mqSE524QQo+/QremwTCQht7TY= github.com/microsoft/go-mssqldb v1.10.0/go.mod h1:mnG7lGa9iYJbzJqGCXyuQCegStKMr3kogDLD6+bmggg= -github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= -github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -554,8 +517,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c= github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM= -github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= -github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/patrickmn/go-cache v2.1.1-0.20191004192108-46f407853014+incompatible h1:IWzUvJ72xMjmrjR9q3H1PF+jwdN0uNQiR2t1BLNalyo= +github.com/patrickmn/go-cache v2.1.1-0.20191004192108-46f407853014+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/paulmach/orb v0.12.0 h1:z+zOwjmG3MyEEqzv92UN49Lg1JFYx0L9GpGKNVDKk1s= github.com/paulmach/orb v0.12.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= @@ -574,8 +537,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU= github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 h1:S1hI5JiKP7883xBzZAr1ydcxrKNSVNm7+3+JwjxZEsg= +github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25/go.mod h1:ZQntvDG8TkPgljxtA0R9frDoND4QORU1VXz015N5Ks4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -622,18 +585,18 @@ github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiT github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= -github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= +github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= +github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= -github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -678,8 +641,6 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= -github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= @@ -694,53 +655,50 @@ github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -github.com/zclconf/go-cty v1.18.1 h1:yEGE8M4iIZlyKQURZNb2SnEyZlZHUcBCnx6KF81KuwM= -github.com/zclconf/go-cty v1.18.1/go.mod h1:qpnV6EDNgC1sns/AleL1fvatHw72j+S+nS+MJ+T2CSg= -github.com/zclconf/go-cty-yaml v1.2.0 h1:GDyL4+e/Qe/S0B7YaecMLbVvAR/Mp21CXMOSiCTOi1M= -github.com/zclconf/go-cty-yaml v1.2.0/go.mod h1:9YLUH4g7lOhVWqUbctnVlZ5KLpg7JAprQNgxSZ1Gyxs= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmfElKI4d/Myu2OxDFU= -go.opentelemetry.io/contrib/detectors/gcp v1.43.0/go.mod h1:RyaZMFY7yi1kAs45S6mbFGz8O8rqB0dTY14uzvG4LCs= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= -go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= -go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 h1:qazEJlUOQzhCpzQpFETGby7EdqjI1wsd0W+6Gg1SCTU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0/go.mod h1:fOD2Yefuxixkx3ahVNf0O/PERb6r4OlbxfATVnYvzCo= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 h1:lgh3PiVrRUWMLOVSkQicxzZll5NjF1r+AtsX1XRIHw0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0/go.mod h1:5Cnhth3m/AgOeTgE3ex12pPmiu/gGtZit03kSzx9X7s= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0 h1:9jR0ZPRok9ryaOQ2Wx8rg5F7Aon59mxrqbVI60/vlBk= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0/go.mod h1:VSme3o2fvSg5bVg0dRzyHaj4Z5EVhG+g2Fde6LKzmQA= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 h1:oECp5f+hN7nkwjU/8BxQ/q23bGPb8FIrD839owX222E= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0/go.mod h1:DqEFwLumhzMBDQv9PcWbyoDxHI/4lAk6CM4nJBH39sc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= +go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= +go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 h1:UFpxOpYPmMNUtOWhdb+nC1WELIgVf8z9higURrbzYU4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0/go.mod h1:YBGjxe3lt0jtXoEXxGrGhv/jM7oUBUDVK7zhAvNyEjc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 h1:aXZXsZ012wOgVkqaiQv+Z/d8V0xKjmg70F6m4CE94lc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0/go.mod h1:ziu1gUIJhAaxM3EV1e3Ralk5AAyL1UOYz3lJJ4VSI38= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0 h1:TC+BewnDpeiAmcscXbGMfxkO+mwYUwE/VySwvw88PfA= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0/go.mod h1:J/ZyF4vfPwsSr9xJSPyQ4LqtcTPULFR64KwTikGLe+A= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.44.0 h1:bl2S7Ubua0Nms+D/gAmznQTd4dxxMA93aKbcpKqiTCs= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.44.0/go.mod h1:L0hRV50XdVIODHUfWEqGRCXQvj2rV82STVo12FMFBU0= -go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= -go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= -go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA= -go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk= -go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= -go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= -go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= -go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= -go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= -go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= -go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= -go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 h1:lsA/S1bxgdbyFGkTj+3meEdJ6ADVU7QoFstV6MXgE68= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0/go.mod h1:L7u+MirGoB1bjeLH66+xDykF4RC8C3RN7lIFpBiewUo= +go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= +go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= +go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= +go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/metric/x v0.69.0 h1:DjRLr15H83v+hCW7JA9NoJvOkYTtmq5YoDRbe9deYpM= +go.opentelemetry.io/otel/metric/x v0.69.0/go.mod h1:uVvsMPMFFyj/HUQfrUnH3JjnOQ1dwFDorgFLRBasM0k= +go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= +go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= +go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= +go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= +go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= +go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= go.yaml.in/yaml/v4 v4.0.0-rc.4 h1:UP4+v6fFrBIb1l934bDl//mmnoIZEDK0idg1+AIvX5U= go.yaml.in/yaml/v4 v4.0.0-rc.4/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= gocloud.dev v0.46.0 h1:niIuZwSjMtBx8K+ITB2s5kZullB13PGOS2ZoQPZxQ4Q= @@ -755,8 +713,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= -golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= golang.org/x/exp v0.0.0-20260527015227-08cc5374adb3 h1:VHEvKbpgPXcPXn40t9cDTGK3JZwMikIEyF/CTrFfu7k= golang.org/x/exp v0.0.0-20260527015227-08cc5374adb3/go.mod h1:d2fgXJLVs4dYDHUk5lwMIfzRzSrWCfGZb0ZqeLa/Vcw= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -766,8 +724,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= -golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -781,8 +739,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= -golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -795,8 +753,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= -golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -807,7 +765,6 @@ golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -818,8 +775,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -829,8 +786,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= -golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= +golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= +golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -841,8 +798,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -853,8 +810,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= -golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI= +golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -867,16 +824,16 @@ google.golang.org/api v0.286.0 h1:TdTXMvzYKnWV1/lPbCdbXRqBrkDqjPto22H2xeZZ8LI= google.golang.org/api v0.286.0/go.mod h1:NlOlUIr8MPoIhT9Bb/oUnRuHbJOLwxb6JSYJM8Yz+jQ= google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= -google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= -google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260622175928-b703f567277d h1:mpAgMyM9vQHxycBlDq50y1VHpfSfVwzXvrQKtYbXuUY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260622175928-b703f567277d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= -google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc h1:4bNTbnb44EqGVy9HaQxSY2jnafSUdgV3qHtedvNpDKg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.85.0 h1:BPBCfuVRQ1qvBhKDOq9uvQllMvO59rKsYhXpp7Dd8vw= +google.golang.org/grpc v1.85.0/go.mod h1:0rQrOD58yjal/0EaVbGBANindixsCZNQPDV3VKtaKuQ= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= -google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -897,32 +854,26 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= -gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= -gorm.io/driver/sqlserver v1.6.4 h1:kGA9Z0D7dnIz7yVvWp18qLBSPFpUQWGqMA4rnxkScdQ= -gorm.io/driver/sqlserver v1.6.4/go.mod h1:oRtXDKFRYj8MqyMq+JFEdaA+StSQKC4zupU6blIdB0s= gorm.io/gorm v1.31.2 h1:3o8FXNo9v9S858gil+3LlZA1LkCOzgb4g5BL64FgaCo= gorm.io/gorm v1.31.2/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= -k8s.io/api v0.36.1 h1:XbL/EMj8K2aJpJtePmqUyQMsM0D4QI2pvl7YKJ20FTY= -k8s.io/api v0.36.1/go.mod h1:KOWo4ey3TINlXjeHVuwB3i+tXXnu+UcwFBHlI/9dvEo= -k8s.io/apiextensions-apiserver v0.36.1 h1:6JfYmPUsuUIHuN+3QxutXYWj492RqF5fBSx67GYK5Ks= -k8s.io/apiextensions-apiserver v0.36.1/go.mod h1:pLzZin90riwisdzKwv/GoTwENooytoIx5zWJb4Hkby8= +k8s.io/api v0.36.2 h1:TF6YDLIzKfccK7cq9YpTcGX8TJmEkHVRv78DM51fRYY= +k8s.io/api v0.36.2/go.mod h1:F4LbMO4brjZYh7yFkXWhynSvtB7YauxV4c+HHkNRGNg= k8s.io/apimachinery v0.36.2 h1:0PE/W/WNy1UX61NLbXY5TMbJ6UwLL6E6lAPkYrKFxbQ= k8s.io/apimachinery v0.36.2/go.mod h1:fvf/HOLXq9RId0rnDIbN1OEBvHXdQbLMM8nu0LcBUf4= -k8s.io/client-go v0.36.1 h1:FN/K8QIT2CEDt+2WB2HnWrUANZ50AP5GII43/SP2JR0= -k8s.io/client-go v0.36.1/go.mod h1:s6rAnCtTGYDQnpNjEhSaISV+2O8jwruZ6m3QOYBFbtU= +k8s.io/client-go v0.36.2 h1:bfgxmFKc9CgqsgX4xKLAAdmTQlWee7Ob/HlDOrJ5TBI= +k8s.io/client-go v0.36.2/go.mod h1:1vgO4OAlfPnoLcb+Rze2GF5rAr14w8qjrYMoyXJzQj0= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= -k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= -k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= +k8s.io/kube-openapi v0.0.0-20260501160325-927ab1f70cd6 h1:ngxu1nL4SbFuXwu1EY7cSKcVqSjTQPVbYQT6WNjTXaU= +k8s.io/kube-openapi v0.0.0-20260501160325-927ab1f70cd6/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= k8s.io/streaming v0.36.2 h1:NSKthPPg9UFSKsRauVJUVGH2Dvn8fhKmY4qrMkw/p98= k8s.io/streaming v0.36.2/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= -k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= -k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +k8s.io/utils v0.0.0-20260626114624-be93311217bd h1:Ea7fgQ5we8Y9T0OX5o0dAHzQOBRI07D/dEYRaB9ZZEs= +k8s.io/utils v0.0.0-20260626114624-be93311217bd/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf h1:rRz0YsF7VXj9fXRF6yQgFI7DzST+hsI3TeFSGupntu0= layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf/go.mod h1:ivKkcY8Zxw5ba0jldhZCYYQfGdb2K6u9tbYK1AwMIBc= modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc= @@ -953,13 +904,11 @@ modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -sigs.k8s.io/gateway-api v1.5.1 h1:RqVRIlkhLhUO8wOHKTLnTJA6o/1un4po4/6M1nRzdd0= -sigs.k8s.io/gateway-api v1.5.1/go.mod h1:GvCETiaMAlLym5CovLxGjS0NysqFk3+Yuq3/rh6QL2o= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= -sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.4.0 h1:qmp2e3ZfFi1/jJbDGpD4mt3wyp6PE1NfKHCYLqgNQJo= +sigs.k8s.io/structured-merge-diff/v6 v6.4.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/sdk/captain/probe.go b/sdk/captain/probe.go index 291b9c7..c93063a 100644 --- a/sdk/captain/probe.go +++ b/sdk/captain/probe.go @@ -8,7 +8,7 @@ import ( "time" "github.com/flanksource/captain/pkg/ai" - "github.com/flanksource/captain/pkg/ai/middleware" + "github.com/flanksource/captain/pkg/ai/observe" telltale "github.com/flanksource/telltale/sdk" "github.com/flanksource/telltale/sdk/protocol" ) @@ -43,7 +43,7 @@ func (probe) Arm(_ context.Context, raw json.RawMessage, sink telltale.Sink) (fu if _, err := path.Match(p.Models, ""); err != nil { return nil, fmt.Errorf("llm models glob: %w", err) } - release := middleware.ObserveProvider(func(event middleware.ProviderEvent) { + release := observe.ObserveProvider(func(event observe.ProviderEvent) { match, _ := path.Match(p.Models, event.Model) if p.Models != "" && !match { return @@ -58,8 +58,8 @@ func (probe) Arm(_ context.Context, raw json.RawMessage, sink telltale.Sink) (fu return func() error { release(); return nil }, nil } -func makeRecord(event middleware.ProviderEvent, p params) (Record, error) { - r := Record{Phase: event.Phase, Model: event.Model, Backend: event.Backend, Iter: event.Iter, Usage: event.Usage, Cost: event.Cost, DurationMs: float64(event.Duration) / float64(time.Millisecond)} +func makeRecord(event observe.ProviderEvent, p params) (Record, error) { + r := Record{Phase: string(event.Phase), Model: event.Model, Backend: event.Backend, Iter: event.Iter, Usage: event.Usage, Cost: event.Cost, DurationMs: float64(event.Duration) / float64(time.Millisecond)} if event.Err != nil { r.Error = telltale.Mask(event.Err.Error()) } diff --git a/sdk/captain/probe_test.go b/sdk/captain/probe_test.go index d95c53a..eb95e1d 100644 --- a/sdk/captain/probe_test.go +++ b/sdk/captain/probe_test.go @@ -6,7 +6,7 @@ import ( "testing" "github.com/flanksource/captain/pkg/ai" - "github.com/flanksource/captain/pkg/ai/middleware" + "github.com/flanksource/captain/pkg/ai/observe" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -26,7 +26,7 @@ var _ = Describe("llm probe", func() { Expect(err).To(HaveOccurred()) }) It("includes request and response only when requested", func() { - event := middleware.ProviderEvent{Request: &ai.Request{}, Response: &ai.Response{Text: "response"}} + event := observe.ProviderEvent{Request: &ai.Request{}, Response: &ai.Response{Text: "response"}} without, err := makeRecord(event, params{}) Expect(err).NotTo(HaveOccurred()) Expect(without.Prompt).To(BeEmpty()) @@ -39,7 +39,7 @@ var _ = Describe("llm probe", func() { It("masks included prompts and responses before emission", func() { request := &ai.Request{} request.Prompt.User = "password=prompt-secret" - record, err := makeRecord(middleware.ProviderEvent{Request: request, Response: &ai.Response{Text: "token=response-secret"}}, params{IncludePrompts: true, IncludeResponses: true}) + record, err := makeRecord(observe.ProviderEvent{Request: request, Response: &ai.Response{Text: "token=response-secret"}}, params{IncludePrompts: true, IncludeResponses: true}) Expect(err).NotTo(HaveOccurred()) Expect(record.Prompt).NotTo(ContainSubstring("prompt-secret")) Expect(record.Response).NotTo(ContainSubstring("response-secret")) @@ -47,7 +47,7 @@ var _ = Describe("llm probe", func() { It("includes a runner turn prompt only when requested", func() { turn := &ai.TurnStart{} turn.Request.Prompt.User = "turn prompt" - event := middleware.ProviderEvent{Phase: "iteration", Event: &ai.Event{Kind: ai.EventTurnStart, Raw: turn}} + event := observe.ProviderEvent{Phase: observe.PhaseIteration, Event: &ai.Event{Kind: ai.EventTurnStart, Raw: turn}} without, err := makeRecord(event, params{}) Expect(err).NotTo(HaveOccurred()) Expect(without.Prompt).To(BeEmpty()) diff --git a/sdk/clicky/go.mod b/sdk/clicky/go.mod index 772b9a2..34cfbb6 100644 --- a/sdk/clicky/go.mod +++ b/sdk/clicky/go.mod @@ -3,8 +3,8 @@ module github.com/flanksource/telltale/sdk/clicky go 1.27.1 require ( - github.com/flanksource/clicky v1.21.90 - github.com/flanksource/telltale/sdk v0.0.0 + github.com/flanksource/clicky v1.21.94 + github.com/flanksource/telltale/sdk v0.0.2 github.com/onsi/ginkgo/v2 v2.32.1 github.com/onsi/gomega v1.42.1 ) @@ -25,7 +25,7 @@ require ( github.com/dlclark/regexp2 v1.11.5 // indirect github.com/ebitengine/purego v0.10.2 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect - github.com/flanksource/commons v1.63.0 // indirect + github.com/flanksource/commons v1.64.1 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect diff --git a/sdk/clicky/go.sum b/sdk/clicky/go.sum index d8d09cf..668fc88 100644 --- a/sdk/clicky/go.sum +++ b/sdk/clicky/go.sum @@ -76,10 +76,10 @@ github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6 github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= -github.com/flanksource/clicky v1.21.90 h1:8xkBJAwaeVwK2H4O3sfXeZ4LPanNAfd8krfgI35J278= -github.com/flanksource/clicky v1.21.90/go.mod h1:aigyzJt9FSoGV1v7q9nb7D74vQJHaBYJJ3ShMwq8r34= -github.com/flanksource/commons v1.63.0 h1:gI5DecATKbay5Q1o1hYRxQUSnbC+yWP7BmdHQjrpvTY= -github.com/flanksource/commons v1.63.0/go.mod h1:RlVSQ+Jq96jKaJWf3WnwVO4DmDTdzkm3sremzJuYuVw= +github.com/flanksource/clicky v1.21.94 h1:drV+gK5KX8BobOIpmKOs9b7i+1hXutnVYLfyndvA744= +github.com/flanksource/clicky v1.21.94/go.mod h1:aigyzJt9FSoGV1v7q9nb7D74vQJHaBYJJ3ShMwq8r34= +github.com/flanksource/commons v1.64.1 h1:qm6ZnsoTw9JxzzH3R3WJs9kKig0+nfMBXJomWgCi/VU= +github.com/flanksource/commons v1.64.1/go.mod h1:XauWx50brQc4kK6t9OwK9JVMXy/Aq99mX6w6txASBag= github.com/flanksource/gomplate/v3 v3.24.90 h1:MAUlwU7REhSL62i4n74yvo+T4LA7HuRpJ9tKH9dbMzk= github.com/flanksource/gomplate/v3 v3.24.90/go.mod h1:tcjpBIaHboWVPUYt973dGLCWgTD1sOHdiFgA/AaPBPQ= github.com/flanksource/is-healthy v1.0.90 h1:YV4rmL9y1BOgg0FZsWF7FgMBiHyf1FS3oJG6mB8o4VY= diff --git a/sdk/commonsdb/go.mod b/sdk/commonsdb/go.mod index 18decc4..fecbad5 100644 --- a/sdk/commonsdb/go.mod +++ b/sdk/commonsdb/go.mod @@ -3,15 +3,15 @@ module github.com/flanksource/telltale/sdk/commonsdb go 1.27.1 require ( - github.com/flanksource/commons v1.63.0 + github.com/flanksource/commons v1.64.1 github.com/flanksource/commons-db v0.1.70 - github.com/flanksource/telltale/sdk v0.0.0 + github.com/flanksource/telltale/sdk v0.0.2 github.com/onsi/ginkgo/v2 v2.32.1 github.com/onsi/gomega v1.42.1 ) require ( - cel.dev/expr v0.25.2 // indirect + cel.dev/expr v0.25.3 // indirect cloud.google.com/go v0.123.0 // indirect cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect @@ -31,7 +31,7 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 // indirect github.com/ClickHouse/ch-go v0.71.0 // indirect github.com/ClickHouse/clickhouse-go/v2 v2.46.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect @@ -88,7 +88,7 @@ require ( github.com/eko/gocache/store/go_cache/v4 v4.2.5 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect - github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.39.1-0.20260819172001-e6e3fd93e4be // indirect github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect github.com/flanksource/clicky v1.21.90 // indirect @@ -105,23 +105,23 @@ require ( github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.9.0 // indirect github.com/go-git/go-git/v5 v5.19.2 // indirect - github.com/go-jose/go-jose/v4 v4.1.4 // indirect + github.com/go-jose/go-jose/v4 v4.1.9 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/jsonpointer v0.23.1 // indirect - github.com/go-openapi/jsonreference v0.21.5 // indirect - github.com/go-openapi/swag v0.26.0 // indirect - github.com/go-openapi/swag/cmdutils v0.26.0 // indirect - github.com/go-openapi/swag/conv v0.26.0 // indirect - github.com/go-openapi/swag/fileutils v0.26.0 // indirect - github.com/go-openapi/swag/jsonname v0.26.0 // indirect - github.com/go-openapi/swag/jsonutils v0.26.0 // indirect - github.com/go-openapi/swag/loading v0.26.0 // indirect - github.com/go-openapi/swag/mangling v0.26.0 // indirect - github.com/go-openapi/swag/netutils v0.26.0 // indirect - github.com/go-openapi/swag/stringutils v0.26.0 // indirect - github.com/go-openapi/swag/typeutils v0.26.0 // indirect - github.com/go-openapi/swag/yamlutils v0.26.0 // indirect + github.com/go-openapi/jsonpointer v1.0.0 // indirect + github.com/go-openapi/jsonreference v1.0.0 // indirect + github.com/go-openapi/swag v0.28.0 // indirect + github.com/go-openapi/swag/cmdutils v0.28.0 // indirect + github.com/go-openapi/swag/conv v0.28.0 // indirect + github.com/go-openapi/swag/fileutils v0.28.0 // indirect + github.com/go-openapi/swag/jsonutils v0.28.0 // indirect + github.com/go-openapi/swag/loading v0.28.0 // indirect + github.com/go-openapi/swag/mangling v0.28.0 // indirect + github.com/go-openapi/swag/netutils v0.28.0 // indirect + github.com/go-openapi/swag/pools v0.28.0 // indirect + github.com/go-openapi/swag/stringutils v0.28.0 // indirect + github.com/go-openapi/swag/typeutils v0.28.0 // indirect + github.com/go-openapi/swag/yamlutils v0.28.0 // indirect github.com/go-sql-driver/mysql v1.10.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/go-xmlfmt/xmlfmt v1.1.3 // indirect @@ -143,7 +143,7 @@ require ( github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/gosimple/slug v1.15.0 // indirect github.com/gosimple/unidecode v1.0.1 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/hairyhenderson/toml v0.4.2-0.20210923231440-40456b8e66cf // indirect github.com/hairyhenderson/yaml v0.0.0-20220618171115-2d35fca545ce // indirect github.com/henvic/httpretty v0.1.4 // indirect @@ -167,7 +167,7 @@ require ( github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/kr/fs v0.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/labstack/echo/v4 v4.16.3 // indirect + github.com/labstack/echo/v4 v4.16.4 // indirect github.com/labstack/gommon v0.5.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect @@ -216,7 +216,7 @@ require ( github.com/skeema/knownhosts v1.3.2 // indirect github.com/spf13/cobra v1.10.2 // indirect github.com/spf13/pflag v1.0.10 // indirect - github.com/spiffe/go-spiffe/v2 v2.7.0 // indirect + github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.2.0 // indirect github.com/tidwall/pretty v1.2.1 // indirect @@ -232,17 +232,18 @@ require ( github.com/yuin/goldmark v1.7.17 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect - go.opentelemetry.io/otel v1.45.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 // indirect - go.opentelemetry.io/otel/metric v1.45.0 // indirect - go.opentelemetry.io/otel/sdk v1.45.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect - go.opentelemetry.io/otel/trace v1.45.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.45.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect + go.opentelemetry.io/otel v1.47.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 // indirect + go.opentelemetry.io/otel/log v1.47.0 // indirect + go.opentelemetry.io/otel/metric v1.47.0 // indirect + go.opentelemetry.io/otel/sdk v1.47.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.47.0 // indirect + go.opentelemetry.io/otel/trace v1.47.0 // indirect go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect @@ -261,10 +262,10 @@ require ( golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/api v0.286.0 // indirect google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect - google.golang.org/grpc v1.83.2 // indirect - google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/grpc v1.85.0 // indirect + google.golang.org/protobuf v1.36.12 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect diff --git a/sdk/commonsdb/go.sum b/sdk/commonsdb/go.sum index 7a9ab00..6356afd 100644 --- a/sdk/commonsdb/go.sum +++ b/sdk/commonsdb/go.sum @@ -1,5 +1,5 @@ -cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= -cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0= +cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= @@ -52,8 +52,8 @@ github.com/ClickHouse/clickhouse-go/v2 v2.46.0 h1:s3eRy+hYmu5uzotB6ZhDofgHu8kDgG github.com/ClickHouse/clickhouse-go/v2 v2.46.0/go.mod h1:giJfUVlMkcfUEPVfRpt51zZaGEx9i17gCos8gBl392c= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 h1:l7+6kwRMJNwdCvYdDl7Eax+wzEYHSnNY7zrrfbhDdTA= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 h1:bN1gA3of5bXtbnLsRPrwfmbbe7A5UWFlcTHseujLnpc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0/go.mod h1:Yj5vHEz/aAepZGliRJsA6uvHAVAQyEwajq9ORCHPxzM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ= @@ -190,8 +190,8 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= -github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= -github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/envoy v1.39.1-0.20260819172001-e6e3fd93e4be h1:SWe0x6yfglnxuvOiYgTTnNq7QD/yvqthOh1RBI8Bj8w= +github.com/envoyproxy/go-control-plane/envoy v1.39.1-0.20260819172001-e6e3fd93e4be/go.mod h1:PYEOlng9XcrulfyWpm49jECTPV0LT4q8cO7fLW/xwgk= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= @@ -200,8 +200,8 @@ github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeO github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/flanksource/clicky v1.21.90 h1:8xkBJAwaeVwK2H4O3sfXeZ4LPanNAfd8krfgI35J278= github.com/flanksource/clicky v1.21.90/go.mod h1:aigyzJt9FSoGV1v7q9nb7D74vQJHaBYJJ3ShMwq8r34= -github.com/flanksource/commons v1.63.0 h1:gI5DecATKbay5Q1o1hYRxQUSnbC+yWP7BmdHQjrpvTY= -github.com/flanksource/commons v1.63.0/go.mod h1:RlVSQ+Jq96jKaJWf3WnwVO4DmDTdzkm3sremzJuYuVw= +github.com/flanksource/commons v1.64.1 h1:qm6ZnsoTw9JxzzH3R3WJs9kKig0+nfMBXJomWgCi/VU= +github.com/flanksource/commons v1.64.1/go.mod h1:XauWx50brQc4kK6t9OwK9JVMXy/Aq99mX6w6txASBag= github.com/flanksource/commons-db v0.1.70 h1:u5Ch8NCL9c2Gtqzn9OayKUkF6c03t5z7QLF9PE2Yxag= github.com/flanksource/commons-db v0.1.70/go.mod h1:grmVCAlfxqk8OuQRi1Jmgs9k3QELNVCDDJl3I3EsNAQ= github.com/flanksource/gomplate/v3 v3.24.90 h1:MAUlwU7REhSL62i4n74yvo+T4LA7HuRpJ9tKH9dbMzk= @@ -243,47 +243,47 @@ github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMj github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY= github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuwuB/jWEHjk= -github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= -github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-jose/go-jose/v4 v4.1.9 h1:pKfeQkig1NmgK4hJrZYXLEYURXfoJB61Larw0uIW4gY= +github.com/go-jose/go-jose/v4 v4.1.9/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-openapi/jsonpointer v0.23.1 h1:1HBACs7XIwR2RcmItfdSFlALhGbe6S92p0ry4d1GWg4= -github.com/go-openapi/jsonpointer v0.23.1/go.mod h1:iWRmZTrGn7XwYhtPt/fvdSFj1OfNBngqRT2UG3BxSqY= -github.com/go-openapi/jsonreference v0.21.5 h1:6uCGVXU/aNF13AQNggxfysJ+5ZcU4nEAe+pJyVWRdiE= -github.com/go-openapi/jsonreference v0.21.5/go.mod h1:u25Bw85sX4E2jzFodh1FOKMTZLcfifd1Q+iKKOUxExw= -github.com/go-openapi/swag v0.26.0 h1:GVDXCmfvhfu1BxiHo8/FA+BbKmhecHnG3varjON5/RI= -github.com/go-openapi/swag v0.26.0/go.mod h1:82g3193sZJRbocs7bNCqGfIgq8pkuwVwCfhKIRlEQF0= -github.com/go-openapi/swag/cmdutils v0.26.0 h1:iowihOcvq7y4egO8cOq0dmfohz6wfeQ63U1EnuhO2TU= -github.com/go-openapi/swag/cmdutils v0.26.0/go.mod h1:Sm1MVFMkF6guJJ+pQqHnQA3N0j9qALV3NxzDSv6bETM= -github.com/go-openapi/swag/conv v0.26.0 h1:5yGGsPYI1ZCva93U0AoKi/iZrNhaJEjr324YVsiD89I= -github.com/go-openapi/swag/conv v0.26.0/go.mod h1:tpAmIL7X58VPnHHiSO4uE3jBeRamGsFsfdDeDtb5ECE= -github.com/go-openapi/swag/fileutils v0.26.0 h1:WJoPRvsA7QRiiWluowkLJa9jaYR7FCuxmDvnCgaRRxU= -github.com/go-openapi/swag/fileutils v0.26.0/go.mod h1:0WDJ7lp67eNjPMO50wAWYlKvhOb6CQ37rzR7wrgI8Tc= -github.com/go-openapi/swag/jsonname v0.26.0 h1:gV1NFX9M8avo0YSpmWogqfQISigCmpaiNci8cGECU5w= -github.com/go-openapi/swag/jsonname v0.26.0/go.mod h1:urBBR8bZNoDYGr653ynhIx+gTeIz0ARZxHkAPktJK2M= -github.com/go-openapi/swag/jsonutils v0.26.0 h1:FawFML2iAXsPqmERscuMPIHmFsoP1tOqWkxBaKNMsnA= -github.com/go-openapi/swag/jsonutils v0.26.0/go.mod h1:2VmA0CJlyFqgawOaPI9psnjFDqzyivIqLYN34t9p91E= -github.com/go-openapi/swag/jsonutils/fixtures_test v0.26.0 h1:apqeINu/ICHouqiRZbyFvuDge5jCmmLTqGQ9V95EaOM= -github.com/go-openapi/swag/jsonutils/fixtures_test v0.26.0/go.mod h1:AyM6QT8uz5IdKxk5akv0y6u4QvcL9GWERt0Jx/F/R8Y= -github.com/go-openapi/swag/loading v0.26.0 h1:Apg6zaKhCJurpJer0DCxq99qwmhFddBhaMX7kilDcko= -github.com/go-openapi/swag/loading v0.26.0/go.mod h1:dBxQ/6V2uBaAQdevN18VELE6xSpJWZxLX4txe12JwDg= -github.com/go-openapi/swag/mangling v0.26.0 h1:Du2YC4YLA/Y5m/YKQd7AnY5qq0wRKSFZTTt8ktFaXcQ= -github.com/go-openapi/swag/mangling v0.26.0/go.mod h1:jifS7W9vbg+pw63bT+GI53otluMQL3CeemuyCHKwVx0= -github.com/go-openapi/swag/netutils v0.26.0 h1:CmZp+ZT7HrmFwrC3GdGsXBq2+42T1bjKBapcqVpIs3c= -github.com/go-openapi/swag/netutils v0.26.0/go.mod h1:5iK+Ok3ZohWWex1C50BFTPexi03UaPwjW4Oj8kgrpwo= -github.com/go-openapi/swag/stringutils v0.26.0 h1:qZQngLxs5s7SLijc3N2ZO+fUq2o8LjuWAASSrJuh+xg= -github.com/go-openapi/swag/stringutils v0.26.0/go.mod h1:sWn5uY+QIIspwPhvgnqJsH8xqFT2ZbYcvbcFanRyhFE= -github.com/go-openapi/swag/typeutils v0.26.0 h1:2kdEwdiNWy+JJdOvu5MA2IIg2SylWAFuuyQIKYybfq4= -github.com/go-openapi/swag/typeutils v0.26.0/go.mod h1:oovDuIUvTrEHVMqWilQzKzV4YlSKgyZmFh7AlfABNVE= -github.com/go-openapi/swag/yamlutils v0.26.0 h1:H7O8l/8NJJQ/oiReEN+oMpnGMyt8G0hl460nRZxhLMQ= -github.com/go-openapi/swag/yamlutils v0.26.0/go.mod h1:1evKEGAtP37Pkwcc7EWMF0hedX0/x3Rkvei2wtG/TbU= -github.com/go-openapi/testify/enable/yaml/v2 v2.4.2 h1:5zRca5jw7lzVREKCZVNBpysDNBjj74rBh0N2BGQbSR0= -github.com/go-openapi/testify/enable/yaml/v2 v2.4.2/go.mod h1:XVevPw5hUXuV+5AkI1u1PeAm27EQVrhXTTCPAF85LmE= -github.com/go-openapi/testify/v2 v2.4.2 h1:tiByHpvE9uHrrKjOszax7ZvKB7QOgizBWGBLuq0ePx4= -github.com/go-openapi/testify/v2 v2.4.2/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= +github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s= +github.com/go-openapi/jsonpointer v1.0.0/go.mod h1:Z3rw7dWu1p9IgitXCFamSlA5lmDiklEB6vkaxcNZW5Y= +github.com/go-openapi/jsonreference v1.0.0 h1:jlmTr6torcd1YgDQvSfNmRtKzYDO4FGBkrAdlAVWnpY= +github.com/go-openapi/jsonreference v1.0.0/go.mod h1:jtwdyGbJk0Xhe5Y+rwtglQP6Sb1WZST4rT32LWB+sv0= +github.com/go-openapi/swag v0.28.0 h1:xkgbOSKj6DZziNpyqRRAOt3GJGtgjgsd2RoyT30VWuw= +github.com/go-openapi/swag v0.28.0/go.mod h1:4qYnT3Cqr1p1VknOdPo70evN4rgQnAg6jwApHyxSGIg= +github.com/go-openapi/swag/cmdutils v0.28.0 h1:7TOeNtkYru1SG8Y34tDh9WBbLsMqGnptuxWiHREPZ4Q= +github.com/go-openapi/swag/cmdutils v0.28.0/go.mod h1:Sm1MVFMkF6guJJ+pQqHnQA3N0j9qALV3NxzDSv6bETM= +github.com/go-openapi/swag/conv v0.28.0 h1:GtqqbyFe7vR5Y7ehxG9W6/OvrSFdf1OLeTGp40TqxH8= +github.com/go-openapi/swag/conv v0.28.0/go.mod h1:mbUE+mzctnhxi864m0Q07SpN8OowD9JhxmxuYvZZD/k= +github.com/go-openapi/swag/fileutils v0.28.0 h1:Z04XWQD7R8Eq+7GnOrjovBxPPmZzsS4gt2H2GPGIViU= +github.com/go-openapi/swag/fileutils v0.28.0/go.mod h1:VvJFZLTZS0AI854gEQz5tk7dBESdLjiNUMSZ/th2ry8= +github.com/go-openapi/swag/jsonutils v0.28.0 h1:YIch6FwO7RXzeAnbO8Tu7dWBZeUEH+4nA0HXltVTnv4= +github.com/go-openapi/swag/jsonutils v0.28.0/go.mod h1:CYM3WlTUcagR2ZoHdz54di/cbBqt82tuxuXgAjxw+mg= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.28.0 h1:qV+VVUAx5Oro8WjVWpZeql7YReTKhT4smR4zhcOQZr0= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.28.0/go.mod h1:mofwUWx70wvskwESqRJ//k/9kURmCgyJl5m5Ppoh5kY= +github.com/go-openapi/swag/loading v0.28.0 h1:td8QZdZC9MIYGGSnSPKShKiK22I2tU5UQvuUhIBPRLU= +github.com/go-openapi/swag/loading v0.28.0/go.mod h1:rXB0QiQX5mMveXEA7ouM4KiiM9jVJe4K6BVbwhD1M4k= +github.com/go-openapi/swag/mangling v0.28.0 h1:pH8eyeNO9SLYsTMWJrurnNfKmDa28XrlA+HePVD53VM= +github.com/go-openapi/swag/mangling v0.28.0/go.mod h1:jtBE2+V+3pILxOR7Vgce+Cwp6A2PgZbvVqfNntbVs0w= +github.com/go-openapi/swag/netutils v0.28.0 h1:YXN6TALEi2pzts8/8GNm6T61HTAZsieukGZidap989k= +github.com/go-openapi/swag/netutils v0.28.0/go.mod h1:J+WYyFMLtvtCGqa6jLv+YNUmIKI3ZRQRrvfNDMoQoEQ= +github.com/go-openapi/swag/pools v0.28.0 h1:HPMZWSAfce3rdVTFcjFiCIBtDg9h4x2QlRrHipwhxeU= +github.com/go-openapi/swag/pools v0.28.0/go.mod h1:kVQefhSK5RWuRe7BXsL8htgBPAMpN7HDGpGEknqugeE= +github.com/go-openapi/swag/stringutils v0.28.0 h1:ixsc9iYgDPubHL/8nSkbnryEHpD2VRlBMLKpQyPXcDU= +github.com/go-openapi/swag/stringutils v0.28.0/go.mod h1:lzRN95CxXmA03XcDWHLOb6nOMcxCqR5rGY0lOgsfRoM= +github.com/go-openapi/swag/typeutils v0.28.0 h1:nRBKSBXjDgf01VDPB3fWeD9nQuhCOVeIYAkUx2tbkyY= +github.com/go-openapi/swag/typeutils v0.28.0/go.mod h1:Srm0xFNRZ1Y+vCxJclo5qzx8aj+1pAKda/YfFPrG0dQ= +github.com/go-openapi/swag/yamlutils v0.28.0 h1:TV3JXH6DS46KUroDtMLAYHGkdWf5VDq3wVWFirmzROY= +github.com/go-openapi/swag/yamlutils v0.28.0/go.mod h1:x0q/yndZHEgk9Rx3DyDqzFUmHy55KTvIZldvF2dTJXs= +github.com/go-openapi/testify/enable/yaml/v2 v2.6.0 h1:gGHwAJ0R/5jU8BEGDbfRNR3hL68dAVi84WuOApp29B0= +github.com/go-openapi/testify/enable/yaml/v2 v2.6.0/go.mod h1:tY+St1SGq4NFl0QIqdTY4aEdbChAHxhyB77XQi9iJCo= +github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug= +github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw= github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= @@ -342,8 +342,8 @@ github.com/gosimple/slug v1.15.0 h1:wRZHsRrRcs6b0XnxMUBM6WK1U1Vg5B0R7VkIf1Xzobo= github.com/gosimple/slug v1.15.0/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ= github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o= github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= github.com/hairyhenderson/toml v0.4.2-0.20210923231440-40456b8e66cf h1:I1sbT4ZbIt9i+hB1zfKw2mE8C12TuGxPiW7YmtLbPa4= github.com/hairyhenderson/toml v0.4.2-0.20210923231440-40456b8e66cf/go.mod h1:jDHmWDKZY6MIIYltYYfW4Rs7hQ50oS4qf/6spSiZAxY= github.com/hairyhenderson/yaml v0.0.0-20220618171115-2d35fca545ce h1:cVkYhlWAxwuS2/Yp6qPtcl0fGpcWxuZNonywHZ6/I+s= @@ -413,8 +413,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/labstack/echo/v4 v4.16.3 h1:wAzM49wiBxHpZJQ621lminvdxbwfd1Cn6KwRThKV6nA= -github.com/labstack/echo/v4 v4.16.3/go.mod h1:V/EQaTVHYJH92QhbQe7kFQ4mjgcjxLQQUQtAN87zZdw= +github.com/labstack/echo/v4 v4.16.4 h1:sQ9yq/FgkY7zWZKaRIvNJAQ1ybWVJrm+z37SWI8keRY= +github.com/labstack/echo/v4 v4.16.4/go.mod h1:V/EQaTVHYJH92QhbQe7kFQ4mjgcjxLQQUQtAN87zZdw= github.com/labstack/gommon v0.5.0 h1:6VSQ2NOzsnEJ5W6+84E0RbcaDDmgB6NIAzWCczTEe6c= github.com/labstack/gommon v0.5.0/go.mod h1:Rzlg7HHy1maLfzBYGg9NZcVuz1sA68HHhLjhcEllYE0= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= @@ -536,8 +536,8 @@ github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiT github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= -github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= +github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= +github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= @@ -592,34 +592,36 @@ github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7 go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw= -go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= -go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= -go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 h1:fG5MCxGz8+2VtrN/WgqSpJFctVz24gpxj8CxkKmc8Ww= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0/go.mod h1:BmAYTn+3ysbRe+IU2msxmf5Rx3g6DHvex+tWI3LdhYI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 h1:lgh3PiVrRUWMLOVSkQicxzZll5NjF1r+AtsX1XRIHw0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0/go.mod h1:5Cnhth3m/AgOeTgE3ex12pPmiu/gGtZit03kSzx9X7s= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0 h1:9jR0ZPRok9ryaOQ2Wx8rg5F7Aon59mxrqbVI60/vlBk= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0/go.mod h1:VSme3o2fvSg5bVg0dRzyHaj4Z5EVhG+g2Fde6LKzmQA= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 h1:oECp5f+hN7nkwjU/8BxQ/q23bGPb8FIrD839owX222E= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0/go.mod h1:DqEFwLumhzMBDQv9PcWbyoDxHI/4lAk6CM4nJBH39sc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= +go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= +go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 h1:UFpxOpYPmMNUtOWhdb+nC1WELIgVf8z9higURrbzYU4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0/go.mod h1:YBGjxe3lt0jtXoEXxGrGhv/jM7oUBUDVK7zhAvNyEjc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 h1:aXZXsZ012wOgVkqaiQv+Z/d8V0xKjmg70F6m4CE94lc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0/go.mod h1:ziu1gUIJhAaxM3EV1e3Ralk5AAyL1UOYz3lJJ4VSI38= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0 h1:TC+BewnDpeiAmcscXbGMfxkO+mwYUwE/VySwvw88PfA= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0/go.mod h1:J/ZyF4vfPwsSr9xJSPyQ4LqtcTPULFR64KwTikGLe+A= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.44.0 h1:bl2S7Ubua0Nms+D/gAmznQTd4dxxMA93aKbcpKqiTCs= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.44.0/go.mod h1:L0hRV50XdVIODHUfWEqGRCXQvj2rV82STVo12FMFBU0= -go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= -go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= -go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns= -go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI= -go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= -go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= -go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= -go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= -go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= -go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 h1:lsA/S1bxgdbyFGkTj+3meEdJ6ADVU7QoFstV6MXgE68= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0/go.mod h1:L7u+MirGoB1bjeLH66+xDykF4RC8C3RN7lIFpBiewUo= +go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= +go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= +go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= +go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/metric/x v0.69.0 h1:DjRLr15H83v+hCW7JA9NoJvOkYTtmq5YoDRbe9deYpM= +go.opentelemetry.io/otel/metric/x v0.69.0/go.mod h1:uVvsMPMFFyj/HUQfrUnH3JjnOQ1dwFDorgFLRBasM0k= +go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= +go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= +go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= +go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= +go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= +go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -752,16 +754,16 @@ google.golang.org/api v0.286.0 h1:TdTXMvzYKnWV1/lPbCdbXRqBrkDqjPto22H2xeZZ8LI= google.golang.org/api v0.286.0/go.mod h1:NlOlUIr8MPoIhT9Bb/oUnRuHbJOLwxb6JSYJM8Yz+jQ= google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= -google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc= -google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= -google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc h1:4bNTbnb44EqGVy9HaQxSY2jnafSUdgV3qHtedvNpDKg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.85.0 h1:BPBCfuVRQ1qvBhKDOq9uvQllMvO59rKsYhXpp7Dd8vw= +google.golang.org/grpc v1.85.0/go.mod h1:0rQrOD58yjal/0EaVbGBANindixsCZNQPDV3VKtaKuQ= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= -google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/sdk/commonslog/go.mod b/sdk/commonslog/go.mod index 79a5f8d..53d5e64 100644 --- a/sdk/commonslog/go.mod +++ b/sdk/commonslog/go.mod @@ -3,8 +3,8 @@ module github.com/flanksource/telltale/sdk/commonslog go 1.27.1 require ( - github.com/flanksource/commons v1.63.0 - github.com/flanksource/telltale/sdk v0.0.0 + github.com/flanksource/commons v1.64.1 + github.com/flanksource/telltale/sdk v0.0.2 github.com/onsi/ginkgo/v2 v2.32.1 github.com/onsi/gomega v1.42.1 ) diff --git a/sdk/commonslog/go.sum b/sdk/commonslog/go.sum index 2856c98..08d011a 100644 --- a/sdk/commonslog/go.sum +++ b/sdk/commonslog/go.sum @@ -32,8 +32,8 @@ github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeO github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/flanksource/clicky v1.21.82 h1:YwCVk2WAZoY/tLMLmK8Hm5CA7rXZgsVS5514LbEXlNI= github.com/flanksource/clicky v1.21.82/go.mod h1:rLCgem+/vIDDC4r74hHq8WDDmzv8sGF8BIrqraDSzOY= -github.com/flanksource/commons v1.63.0 h1:gI5DecATKbay5Q1o1hYRxQUSnbC+yWP7BmdHQjrpvTY= -github.com/flanksource/commons v1.63.0/go.mod h1:RlVSQ+Jq96jKaJWf3WnwVO4DmDTdzkm3sremzJuYuVw= +github.com/flanksource/commons v1.64.1 h1:qm6ZnsoTw9JxzzH3R3WJs9kKig0+nfMBXJomWgCi/VU= +github.com/flanksource/commons v1.64.1/go.mod h1:XauWx50brQc4kK6t9OwK9JVMXy/Aq99mX6w6txASBag= github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= @@ -106,8 +106,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= -github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= diff --git a/sdk/go.mod b/sdk/go.mod index e407424..c71e623 100644 --- a/sdk/go.mod +++ b/sdk/go.mod @@ -4,7 +4,7 @@ go 1.27.1 require ( github.com/felixge/httpsnoop v1.1.0 - github.com/flanksource/commons v1.63.0 + github.com/flanksource/commons v1.64.1 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 github.com/onsi/ginkgo/v2 v2.32.1 github.com/onsi/gomega v1.42.1 @@ -12,16 +12,44 @@ require ( require ( github.com/Masterminds/semver/v3 v3.5.0 // indirect + github.com/alecthomas/chroma/v2 v2.23.1 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charmbracelet/colorprofile v0.4.3 // indirect + github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 // indirect + github.com/charmbracelet/x/ansi v0.11.6 // indirect + github.com/charmbracelet/x/cellbuf v0.0.15 // indirect + github.com/charmbracelet/x/term v0.2.2 // indirect + github.com/clipperhouse/displaywidth v0.11.0 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/dlclark/regexp2 v1.11.5 // indirect + github.com/flanksource/clicky v1.21.82 // indirect github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/go-xmlfmt/xmlfmt v1.1.3 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect - github.com/tidwall/match v1.2.0 // indirect + github.com/lucasb-eyer/go-colorful v1.3.0 // indirect + github.com/mattn/go-isatty v0.0.22 // indirect + github.com/mattn/go-runewidth v0.0.21 // indirect + github.com/muesli/termenv v0.16.0 // indirect + github.com/patrickmn/go-cache v2.1.0+incompatible // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/mod v0.37.0 // indirect golang.org/x/net v0.56.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect golang.org/x/text v0.40.0 // indirect golang.org/x/tools v0.47.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/sdk/go.sum b/sdk/go.sum index 2856c98..aae6b3a 100644 --- a/sdk/go.sum +++ b/sdk/go.sum @@ -1,9 +1,15 @@ github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= +github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/chroma/v2 v2.23.1 h1:nv2AVZdTyClGbVQkIzlDm/rnhk1E9bU9nXwmZ/Vk/iY= github.com/alecthomas/chroma/v2 v2.23.1/go.mod h1:NqVhfBR0lte5Ouh3DcthuUCTUpDC9cxBOfyMbMQPs3o= +github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= +github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/aymanbagabas/go-udiff v0.4.1 h1:OEIrQ8maEeDBXQDoGCbbTTXYJMYRCRO1fnodZ12Gv5o= +github.com/aymanbagabas/go-udiff v0.4.1/go.mod h1:0L9PGwj20lrtmEMeyw4WKJ/TMyDtvAoK9bf2u/mNo3w= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -18,6 +24,8 @@ github.com/charmbracelet/x/ansi v0.11.6 h1:GhV21SiDz/45W9AnV2R61xZMRri5NlLnl6CVF github.com/charmbracelet/x/ansi v0.11.6/go.mod h1:2JNYLgQUsyqaiLovhU2Rv/pb8r6ydXKS3NIttu3VGZQ= github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI= github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q= +github.com/charmbracelet/x/exp/golden v0.0.0-20250806222409-83e3a29d542f h1:pk6gmGpCE7F3FcjaOEKYriCvpmIN4+6OS/RD0vm4uIA= +github.com/charmbracelet/x/exp/golden v0.0.0-20250806222409-83e3a29d542f/go.mod h1:IfZAMTHB6XkZSeXUqriemErjAWCCzT0LwjKFYCZyw0I= github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8= @@ -32,14 +40,15 @@ github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeO github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/flanksource/clicky v1.21.82 h1:YwCVk2WAZoY/tLMLmK8Hm5CA7rXZgsVS5514LbEXlNI= github.com/flanksource/clicky v1.21.82/go.mod h1:rLCgem+/vIDDC4r74hHq8WDDmzv8sGF8BIrqraDSzOY= -github.com/flanksource/commons v1.63.0 h1:gI5DecATKbay5Q1o1hYRxQUSnbC+yWP7BmdHQjrpvTY= -github.com/flanksource/commons v1.63.0/go.mod h1:RlVSQ+Jq96jKaJWf3WnwVO4DmDTdzkm3sremzJuYuVw= +github.com/flanksource/commons v1.64.1 h1:qm6ZnsoTw9JxzzH3R3WJs9kKig0+nfMBXJomWgCi/VU= +github.com/flanksource/commons v1.64.1/go.mod h1:XauWx50brQc4kK6t9OwK9JVMXy/Aq99mX6w6txASBag= github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -54,6 +63,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg= github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -118,12 +129,18 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.44.0 h1:bl2S7Ubua0Nms+D/gAmznQTd4dxxMA93aKbcpKqiTCs= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.44.0/go.mod h1:L0hRV50XdVIODHUfWEqGRCXQvj2rV82STVo12FMFBU0= go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/exp v0.0.0-20260527015227-08cc5374adb3 h1:VHEvKbpgPXcPXn40t9cDTGK3JZwMikIEyF/CTrFfu7k= +golang.org/x/exp v0.0.0-20260527015227-08cc5374adb3/go.mod h1:d2fgXJLVs4dYDHUk5lwMIfzRzSrWCfGZb0ZqeLa/Vcw= golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= diff --git a/sdk/http.go b/sdk/http.go index adb9792..270509c 100644 --- a/sdk/http.go +++ b/sdk/http.go @@ -49,6 +49,7 @@ func (httpProbe) Arm(ctx context.Context, params json.RawMessage, sink Sink) (fu type httpTransport struct { next http.RoundTripper capture http.RoundTripper + wrap func(http.RoundTripper) http.RoundTripper } func (t httpTransport) RoundTrip(request *http.Request) (*http.Response, error) { @@ -58,6 +59,19 @@ func (t httpTransport) RoundTrip(request *http.Request) (*http.Response, error) return t.capture.RoundTrip(request) } +// Unwrap implements commons http.TransportWrapper so clients can reconfigure +// the transport beneath the capture layer. +func (t httpTransport) Unwrap() http.RoundTripper { return t.next } + +// Rewrap implements commons http.TransportWrapper: it captures through the same +// collector around a reconfigured inner transport. +func (t httpTransport) Rewrap(inner http.RoundTripper) http.RoundTripper { + if inner == nil { + panic("telltale: HTTP transport must not be nil") + } + return httpTransport{next: inner, capture: t.wrap(inner), wrap: t.wrap} +} + // Transport captures requests made through next when the HTTP probe is armed. func Transport(next http.RoundTripper) http.RoundTripper { if next == nil { @@ -69,7 +83,8 @@ func Transport(next http.RoundTripper) http.RoundTripper { Record(context.Background(), "http", entry) return nil }) - return httpTransport{next: next, capture: collector.Middleware()(next)} + wrap := collector.Middleware() + return httpTransport{next: next, capture: wrap(next), wrap: wrap} } // InstrumentDefaultTransport wraps the default HTTP transport at application startup. diff --git a/sdk/http_test.go b/sdk/http_test.go index cc4ead9..2ce829f 100644 --- a/sdk/http_test.go +++ b/sdk/http_test.go @@ -1,6 +1,7 @@ package telltale import ( + "context" "encoding/json" "errors" "io" @@ -10,6 +11,7 @@ import ( "time" "github.com/flanksource/commons/har" + commonshttp "github.com/flanksource/commons/http" "github.com/flanksource/telltale/sdk/protocol" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -84,6 +86,34 @@ var _ = Describe("HTTP transport", func() { Expect(entry.Response.Status).To(Equal(http.StatusCreated)) Expect(entry.Response.Content.Text).NotTo(ContainSubstring("secret-value")) }) + It("captures a commons client configured for TLS after instrumenting the default transport", func() { + previous := http.DefaultTransport + DeferCleanup(func() { http.DefaultTransport = previous }) + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "ok") + })) + defer server.Close() + agent := Loopback() + defer agent.Close() + stream, conn := openAgent(agent) + defer stream.Close() + Expect(conn.Send("arm", protocol.Arm{Session: "tls-test", Probe: "http", Params: json.RawMessage(`{}`)})).To(Succeed()) + _, _, err := conn.Read() + Expect(err).NotTo(HaveOccurred()) + InstrumentDefaultTransport() + response, err := commonshttp.NewClient().InsecureSkipVerify(true).R(GinkgoT().Context()).Get(server.URL + "/widgets") + Expect(err).NotTo(HaveOccurred()) + Expect(response.StatusCode).To(Equal(http.StatusOK)) + frame, _, err := conn.Read() + Expect(err).NotTo(HaveOccurred()) + var batch protocol.Records + Expect(json.Unmarshal(frame.Body, &batch)).To(Succeed()) + Expect(batch.Items).To(HaveLen(1)) + var entry har.Entry + Expect(json.Unmarshal(batch.Items[0], &entry)).To(Succeed()) + Expect(entry.Request.URL).To(Equal(server.URL + "/widgets")) + Expect(entry.Response.Status).To(Equal(http.StatusOK)) + }) It("emits a HAR entry for an armed request", func() { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -114,4 +144,35 @@ var _ = Describe("HTTP transport", func() { Expect(entry.Request.URL).To(Equal(server.URL)) Expect(entry.Response.Status).To(Equal(http.StatusOK)) }) + It("captures a commons client configured for TLS after InstrumentDefaultTransport", func() { + original := http.DefaultTransport + DeferCleanup(func() { http.DefaultTransport = original }) + const path = "/tls-items" + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"result":"ok"}`) + })) + defer server.Close() + agent := Loopback() + defer agent.Close() + stream, conn := openAgent(agent) + defer stream.Close() + Expect(conn.Send("arm", protocol.Arm{Session: "commons-http-test", Probe: "http", Params: json.RawMessage(`{}`)})).To(Succeed()) + _, _, err := conn.Read() + Expect(err).NotTo(HaveOccurred()) + + InstrumentDefaultTransport() + response, err := commonshttp.NewClient().InsecureSkipVerify(true).R(context.Background()).Get(server.URL + path) + Expect(err).NotTo(HaveOccurred()) + Expect(response.AsString()).To(Equal(`{"result":"ok"}`)) + + frame, _, err := conn.Read() + Expect(err).NotTo(HaveOccurred()) + var batch protocol.Records + Expect(json.Unmarshal(frame.Body, &batch)).To(Succeed()) + Expect(batch.Items).To(HaveLen(1)) + var entry har.Entry + Expect(json.Unmarshal(batch.Items[0], &entry)).To(Succeed()) + Expect(entry.Request.URL).To(HaveSuffix(path)) + Expect(entry.Response.Status).To(Equal(http.StatusOK)) + }) }) diff --git a/sdk/java/build.gradle b/sdk/java/build.gradle index 23e98af..7a23953 100644 --- a/sdk/java/build.gradle +++ b/sdk/java/build.gradle @@ -1,24 +1,272 @@ plugins { id 'java' - id 'com.github.johnrengelman.shadow' version '8.1.1' + id 'war' + id 'com.gradleup.shadow' version '8.3.6' } group = 'com.flanksource.telltale' version = providers.gradleProperty('buildVersion').getOrElse('dev') repositories { mavenCentral() } +def observerTargets = new Properties() +file('src/main/resources/com/flanksource/telltale/observer/targets.properties').withInputStream { observerTargets.load(it) } +def observerPin = { String name -> + def value = observerTargets.getProperty(name + '.version') + if (value == null || value.trim().isEmpty()) throw new GradleException('missing observer dependency pin ' + name) + value +} java { sourceCompatibility = JavaVersion.VERSION_1_8; targetCompatibility = JavaVersion.VERSION_1_8 } tasks.withType(JavaCompile).configureEach { if (JavaVersion.current().isJava9Compatible()) options.release = 8 } +sourceSets { + servlet { java.srcDir 'src/war/java' } + harness { java.srcDir 'src/harness/java' } + jvmDemo { java.srcDir '../../hack/jvm-demo/java' } +} +configurations { bridgeRuntime; otelGateAgent; jakartaJmsClient; spring6Jms; spring7Jms; spring6Http; spring7Http; spring6Mvc; spring7Mvc; spring6Sql; spring7Sql; httpClients3Runtime; httpClients4Runtime; h2Fixture; gateH2Java8 } +configurations.testImplementation.extendsFrom(configurations.h2Fixture) +configurations.testCompileClasspath.attributes.attribute(org.gradle.api.attributes.java.TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, JavaVersion.current().majorVersion.toInteger()) +configurations.testRuntimeClasspath.attributes.attribute(org.gradle.api.attributes.java.TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, JavaVersion.current().majorVersion.toInteger()) +configurations.jvmDemoCompileClasspath.attributes.attribute(org.gradle.api.attributes.java.TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, 17) +configurations.jvmDemoRuntimeClasspath.attributes.attribute(org.gradle.api.attributes.java.TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, 17) +tasks.named('compileJvmDemoJava') { options.release = 17 } dependencies { implementation 'com.google.code.gson:gson:2.11.0' implementation 'org.java-websocket:Java-WebSocket:1.6.0' implementation 'tools.profiler:async-profiler:4.6' + implementation 'com.fasterxml.jackson.core:jackson-databind:2.18.3' + implementation 'ognl:ognl:3.3.4' + implementation 'org.benf:cfr:0.152' + implementation 'org.ow2.asm:asm:9.11.1' + implementation 'org.ow2.asm:asm-tree:9.11.1' + implementation 'org.ow2.asm:asm-commons:9.11.1' + implementation 'org.javassist:javassist:3.29.0-GA' + implementation 'org.apache.groovy:groovy:4.0.33' + implementation "io.opentelemetry.proto:opentelemetry-proto:${observerPin('profiles')}" + implementation "com.google.protobuf:protobuf-java:${observerPin('protobuf')}" + implementation "io.opentelemetry.instrumentation:opentelemetry-instrumentation-api-incubator:${observerPin('instrumentation')}-alpha" + implementation 'io.r2dbc:r2dbc-spi:1.0.0.RELEASE' + implementation 'io.r2dbc:r2dbc-proxy:1.1.7.RELEASE' + h2Fixture('com.h2database:h2:2.3.232') { transitive = false } + gateH2Java8('com.h2database:h2:2.2.224') { transitive = false } + testImplementation 'io.r2dbc:r2dbc-h2:1.0.0.RELEASE' + testImplementation 'net.bytebuddy:byte-buddy-agent:1.18.2' + testImplementation "io.opentelemetry:opentelemetry-sdk:${observerPin('sdk')}" + testImplementation "io.opentelemetry:opentelemetry-exporter-otlp:${observerPin('sdk')}" + testImplementation "io.opentelemetry.instrumentation:opentelemetry-instrumentation-api:${observerPin('instrumentation')}" + testImplementation 'org.apache.activemq:artemis-jms-server:2.44.0' + testImplementation 'org.springframework:spring-jms:5.3.39' + testImplementation 'org.springframework:spring-context:5.3.39' + testImplementation 'jakarta.jms:jakarta.jms-api:3.1.0' + testImplementation 'javax.jms:javax.jms-api:2.0.1' + testImplementation 'org.apache.httpcomponents:httpclient:4.5.14' + testImplementation 'org.apache.httpcomponents:httpasyncclient:4.1.5' + testImplementation 'org.apache.httpcomponents.client5:httpclient5:5.2.3' + testImplementation 'com.squareup.okhttp3:okhttp:4.12.0' + testImplementation 'org.springframework:spring-webmvc:5.3.39' + testImplementation 'org.springframework:spring-webflux:5.3.39' + testImplementation 'io.projectreactor.netty:reactor-netty-http:1.0.48' + testImplementation 'org.springframework:spring-jdbc:5.3.39' + testImplementation 'org.springframework:spring-orm:5.3.39' + testImplementation 'org.springframework.data:spring-data-r2dbc:1.5.18' + testImplementation 'org.hibernate:hibernate-core:5.6.15.Final' + testImplementation 'com.zaxxer:HikariCP:4.0.3' + testImplementation 'javax.persistence:javax.persistence-api:2.2' + otelGateAgent "io.opentelemetry.javaagent:opentelemetry-javaagent:${observerPin('instrumentation')}" + jakartaJmsClient('org.apache.activemq:artemis-jakarta-client:2.44.0') { transitive = false } + spring6Jms 'org.springframework:spring-jms:6.2.14' + spring6Jms 'org.springframework:spring-context:6.2.14' + spring7Jms 'org.springframework:spring-jms:7.0.10' + spring7Jms 'org.springframework:spring-context:7.0.10' + spring6Http 'org.springframework:spring-webflux:6.2.14' + spring6Http 'org.springframework:spring-context:6.2.14' + spring6Http 'io.projectreactor.netty:reactor-netty-http:1.2.14' + spring7Http 'org.springframework:spring-webflux:7.0.10' + spring7Http 'org.springframework:spring-context:7.0.10' + spring7Http 'io.projectreactor.netty:reactor-netty-http:1.3.8' + spring6Mvc 'org.springframework:spring-webmvc:6.2.14' + spring6Mvc 'org.springframework:spring-context:6.2.14' + spring6Mvc 'org.apache.tomcat.embed:tomcat-embed-core:10.1.50' + spring6Mvc 'jakarta.annotation:jakarta.annotation-api:2.1.1' + spring7Mvc 'org.springframework:spring-webmvc:7.0.10' + spring7Mvc 'org.springframework:spring-context:7.0.10' + spring7Mvc 'org.apache.tomcat.embed:tomcat-embed-core:11.0.20' + spring7Mvc 'jakarta.annotation:jakarta.annotation-api:3.0.0' + spring6Sql 'org.springframework:spring-jdbc:6.2.14' + spring6Sql 'org.springframework:spring-context:6.2.14' + spring7Sql 'org.springframework:spring-jdbc:7.0.10' + spring7Sql 'org.springframework:spring-context:7.0.10' + httpClients3Runtime 'com.squareup.okhttp3:okhttp:3.14.9' + httpClients4Runtime 'com.squareup.okhttp3:okhttp:4.12.0' testImplementation 'junit:junit:4.13.2' + testImplementation 'org.junit.jupiter:junit-jupiter:5.10.5' + testRuntimeOnly 'org.junit.vintage:junit-vintage-engine:5.10.5' + testImplementation 'org.apache.tomcat.embed:tomcat-embed-core:9.0.123' + testImplementation 'javax.annotation:javax.annotation-api:1.3.2' + servletCompileOnly 'javax.servlet:javax.servlet-api:3.1.0' + harnessImplementation 'org.apache.tomcat.embed:tomcat-embed-core:9.0.123' + harnessImplementation 'javax.annotation:javax.annotation-api:1.3.2' + bridgeRuntime 'com.fasterxml.jackson.core:jackson-databind:2.18.3' + jvmDemoImplementation 'org.springframework:spring-webmvc:5.3.39' + jvmDemoImplementation 'org.springframework:spring-webflux:5.3.39' + jvmDemoImplementation 'org.springframework:spring-jdbc:5.3.39' + jvmDemoImplementation 'org.springframework:spring-jms:5.3.39' + jvmDemoImplementation 'org.springframework.data:spring-data-r2dbc:1.5.18' + jvmDemoImplementation 'org.apache.tomcat.embed:tomcat-embed-core:9.0.123' + jvmDemoImplementation 'javax.annotation:javax.annotation-api:1.3.2' + jvmDemoImplementation 'com.fasterxml.jackson.core:jackson-databind:2.18.3' + jvmDemoImplementation 'com.h2database:h2:2.3.232' + jvmDemoImplementation 'io.r2dbc:r2dbc-h2:1.0.0.RELEASE' + jvmDemoImplementation 'io.projectreactor.netty:reactor-netty-http:1.0.48' + jvmDemoImplementation 'org.apache.activemq:artemis-jms-server:2.44.0' + jvmDemoImplementation 'javax.jms:javax.jms-api:2.0.1' + jvmDemoImplementation "io.opentelemetry:opentelemetry-api:${observerPin('sdk')}" +} +tasks.register('jvmDemo') { + dependsOn 'jvmDemoClasses' + inputs.files sourceSets.jvmDemo.runtimeClasspath, configurations.otelGateAgent + outputs.dir layout.buildDirectory.dir('jvm-demo') + doLast { + if (!JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_17)) throw new GradleException('JVM demo requires a full JDK 17 or newer') + def destination = layout.buildDirectory.dir('jvm-demo').get().asFile + destination.mkdirs() + new File(destination, 'classpath.txt').text = sourceSets.jvmDemo.runtimeClasspath.asPath + '\n' + copy { + from configurations.otelGateAgent + into destination + rename { 'opentelemetry-javaagent.jar' } + } + } } shadowJar { - archiveFileName = 'telltale-agent.jar' - destinationDirectory = file('../../pkg/agentinstall/java/assets') + exclude 'com/flanksource/telltale/observer/bootstrap/**' + archiveFileName = 'telltale-agent-runtime.jar' + destinationDirectory = layout.buildDirectory.dir('libs') + finalizedBy 'agentJar' relocate 'com.google.gson', 'com.flanksource.telltale.shaded.gson' relocate 'org.java_websocket', 'com.flanksource.telltale.shaded.websocket' relocate 'org.slf4j', 'com.flanksource.telltale.shaded.slf4j' - manifest { attributes('Premain-Class':'com.flanksource.telltale.Agent', 'Agent-Class':'com.flanksource.telltale.Agent', 'Main-Class':'com.flanksource.telltale.Attach', 'Implementation-Version':project.version) } + relocate 'com.fasterxml.jackson', 'com.flanksource.telltale.shaded.jackson' + relocate 'org.objectweb.asm', 'com.flanksource.telltale.shaded.asm' + relocate 'ognl', 'com.flanksource.telltale.shaded.ognl' + relocate 'javassist', 'com.flanksource.telltale.shaded.javassist' + relocate 'io.opentelemetry', 'com.flanksource.telltale.shaded.otel' + relocate 'com.google.protobuf', 'com.flanksource.telltale.shaded.protobuf' + dependencies { + exclude(dependency('io.r2dbc:r2dbc-spi:.*')) + exclude(dependency('org.reactivestreams:reactive-streams:.*')) + exclude(dependency('io.projectreactor:reactor-core:.*')) + } + manifest { attributes('Premain-Class':'com.flanksource.telltale.bootstrap.AgentBootstrap', 'Agent-Class':'com.flanksource.telltale.bootstrap.AgentBootstrap', 'Main-Class':'com.flanksource.telltale.Attach', 'Implementation-Version':project.version, 'Can-Redefine-Classes':'true', 'Can-Retransform-Classes':'true') } +} +// The agent the Go deployer embeds (gitignored assets dir). A plain Jar over the runtime shadow +// jar, so the diag agent and observer bootstrap jars nested in it stay whole: shadow would unpack +// them. InstrumentationService stages telltale-diag-agent.jar back out to self-attach. +tasks.register('agentJar', Jar) { + dependsOn shadowJar, 'diagnosticAgentJar', 'observerBootstrapJar', 'testFixtureJars' + archiveFileName = 'telltale-agent.jar' + destinationDirectory = file('../../pkg/agentinstall/java/assets') + from({ zipTree(tasks.shadowJar.archiveFile.get().asFile) }) { exclude 'META-INF/MANIFEST.MF' } + from({ tasks.named('diagnosticAgentJar').get().archiveFile }) + from({ tasks.named('observerBootstrapJar').get().archiveFile }) + manifest.from tasks.shadowJar.manifest +} +tasks.register('testFixtureJars', Copy) { + from configurations.h2Fixture + from configurations.gateH2Java8 + into layout.buildDirectory.dir('libs/integration-fixtures') +} +tasks.register('observerBootstrapJar', Jar) { + archiveFileName = 'telltale-observer-bootstrap.jar' + destinationDirectory = layout.buildDirectory.dir('bridge') + from(sourceSets.main.output) { include 'com/flanksource/telltale/observer/bootstrap/**' } +} +tasks.register('diagnosticAgentJar', Jar) { + dependsOn shadowJar, 'observerBootstrapJar' + archiveFileName = 'telltale-diag-agent.jar' + destinationDirectory = layout.buildDirectory.dir('bridge') + from({ zipTree(tasks.shadowJar.archiveFile.get().asFile) }) { exclude 'META-INF/MANIFEST.MF' } + from({ tasks.named('observerBootstrapJar').get().archiveFile }) + manifest { attributes('Agent-Class':'com.flanksource.telltale.diag.agent.DiagAgent','Premain-Class':'com.flanksource.telltale.diag.agent.DiagAgent','Can-Redefine-Classes':'true','Can-Retransform-Classes':'true') } +} +tasks.named('processTestResources') { + dependsOn 'diagnosticAgentJar' + from({ tasks.named('diagnosticAgentJar').get().archiveFile }) +} +war { + dependsOn 'agentJar', servletClasses + archiveFileName = 'telltale-diag.war' + destinationDirectory = layout.buildDirectory.dir('libs') + classpath = files(sourceSets.servlet.output) + webXml = file('src/war/web.xml') + from(tasks.named('agentJar').flatMap { it.archiveFile }) { into 'WEB-INF/runtime' } + finalizedBy 'embedWar' +} +// The Go deployer embeds the WAR next to telltale-agent.jar (gitignored assets dir). +tasks.register('embedWar') { + def source = tasks.war.archiveFile + def target = file('../../pkg/agentinstall/java/assets/telltale-diag.war') + inputs.file(source) + outputs.file(target) + doLast { + java.nio.file.Files.createDirectories(target.parentFile.toPath()) + java.nio.file.Files.copy(source.get().asFile.toPath(), target.toPath(), java.nio.file.StandardCopyOption.REPLACE_EXISTING) + } +} +tasks.register('tomcatHarnessJar', com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) { + archiveFileName = 'telltale-tomcat-harness.jar' + destinationDirectory = layout.buildDirectory.dir('libs') + configurations = [project.configurations.harnessRuntimeClasspath] + from(sourceSets.harness.output) + exclude 'META-INF/*.SF', 'META-INF/*.DSA', 'META-INF/*.RSA', 'module-info.class' + manifest { attributes('Main-Class':'com.flanksource.telltale.harness.TomcatHarness') } +} +test { + useJUnitPlatform() + dependsOn 'war', 'exampleExtensionJar', 'tomcatHarnessJar', 'diagnosticAgentJar', 'jvmDemoClasses' + systemProperty 'telltale.jvm.catalog', file('../../internal/jvm/catalog.json').absolutePath + // The diag agent attached as it is in a pod, so a test can redefine and + // retransform a live class through the real Instrumentation: how HotSpot + // treats those bytes is behaviour no bytecode assertion can stand in for. + jvmArgumentProviders.add({ + ["-javaagent:" + tasks.named('diagnosticAgentJar').get().archiveFile.get().asFile.absolutePath] + } as CommandLineArgumentProvider) + systemProperty 'telltale.observer.bootstrap', layout.buildDirectory.file('bridge/telltale-observer-bootstrap.jar').get().asFile.absolutePath + systemProperty 'telltale.gate.otel', configurations.otelGateAgent.singleFile.absolutePath + systemProperty 'telltale.gate.agent', tasks.named('agentJar').get().archiveFile.get().asFile.absolutePath + systemProperty 'telltale.gate.classpath', sourceSets.test.runtimeClasspath.minus(sourceSets.main.output).asPath + systemProperty 'telltale.demo.classpath', sourceSets.jvmDemo.runtimeClasspath.asPath + systemProperty 'telltale.gate.java8H2', configurations.gateH2Java8.singleFile.absolutePath + systemProperty 'telltale.jms.jakartaClient', configurations.jakartaJmsClient.singleFile.absolutePath + systemProperty 'telltale.jms.spring6', configurations.spring6Jms.asPath + systemProperty 'telltale.jms.spring7', configurations.spring7Jms.asPath + systemProperty 'telltale.http.spring6', configurations.spring6Http.asPath + systemProperty 'telltale.http.spring7', configurations.spring7Http.asPath + systemProperty 'telltale.http.spring6Mvc', configurations.spring6Mvc.asPath + systemProperty 'telltale.http.spring7Mvc', configurations.spring7Mvc.asPath + systemProperty 'telltale.sql.spring6', configurations.spring6Sql.asPath + systemProperty 'telltale.sql.spring7', configurations.spring7Sql.asPath + systemProperty 'telltale.http.clients3.classpath', configurations.httpClients3Runtime.asPath + systemProperty 'telltale.http.clients4.classpath', configurations.httpClients4Runtime.asPath + for (String key : ['telltale.gate.fixtureJava', 'telltale.gate.expectedJava']) { + if (System.getProperty(key) != null) systemProperty key, System.getProperty(key) + } +} +sourceSets.test.java.srcDir 'examples' +tasks.register('jvmCatalog', JavaExec) { + description = 'Write the built-in JVM operation catalog the Go CLI and OpenAPI are generated from' + classpath = sourceSets.test.runtimeClasspath + mainClass = 'com.flanksource.telltale.CatalogExport' + args file('../../internal/jvm/catalog.json').absolutePath +} +tasks.register('extensionApiJar', Jar) { + archiveFileName = 'telltale-extension-api.jar' + from sourceSets.main.output + include 'com/flanksource/telltale/DiagnosticExtension.class' + include 'com/flanksource/telltale/ExtensionContext*.class' + include 'com/flanksource/telltale/OperationSpec*.class' +} +tasks.register('exampleExtensionJar', Jar) { + dependsOn testClasses + archiveFileName = 'telltale-example-extension.jar' + from(sourceSets.test.output) { include 'example/ExampleExtension.class' } + from('examples/resources') } diff --git a/sdk/java/examples/diagnostics.properties b/sdk/java/examples/diagnostics.properties new file mode 100644 index 0000000..cb3771f --- /dev/null +++ b/sdk/java/examples/diagnostics.properties @@ -0,0 +1,11 @@ +# All paths are on the target JVM host. +# Callers present an RS256 JWT: iss must equal issuer, aud must contain audience, +# and scope must grant jvm:read / jvm:trace / jvm:instrument / jvm:eval / jvm:write per route. +issuer=https://telltale.example.com +audience=example-application +# One of jwks (inline JWKS JSON) or jwksFile (absolute path), and/or jwksUrl. +jwksFile=/var/lib/telltale/jwks.json +#jwksUrl=https://telltale.example.com/.well-known/jwks.json +runtimeDir=/var/lib/telltale/runtime +extensionsDir=/var/lib/telltale/extensions +allowMutation=false diff --git a/sdk/java/examples/example/ExampleExtension.java b/sdk/java/examples/example/ExampleExtension.java new file mode 100644 index 0000000..6db1ea2 --- /dev/null +++ b/sdk/java/examples/example/ExampleExtension.java @@ -0,0 +1,44 @@ +package example; + +import com.flanksource.telltale.DiagnosticExtension; +import com.flanksource.telltale.ExtensionContext; +import com.flanksource.telltale.OperationSpec; +import java.util.*; + +public final class ExampleExtension implements DiagnosticExtension { + public String id() { + return "example"; + } + + public String version() { + return "1.0.0"; + } + + public void start(ExtensionContext context) { + Map schema = new LinkedHashMap<>(); + schema.put("type", "object"); + schema.put( + "properties", Collections.singletonMap("name", Collections.singletonMap("type", "string"))); + schema.put("required", Collections.singletonList("name")); + context.register( + OperationSpec.named("extension.example.greet") + .title("Greet an application operator") + .description( + "Example application-specific operation, discovered without client changes") + .parameters(schema) + .handle( + params -> { + Object name = params.get("name"); + if (!(name instanceof String)) + throw new IllegalArgumentException("name is required"); + return Collections.singletonMap("message", "Hello " + name); + })); + context.bindings( + application -> + Collections.singletonMap( + "loaderType", + application == null ? "bootstrap" : application.getClass().getName())); + } + + public void close() {} +} diff --git a/sdk/java/examples/inspect.groovy b/sdk/java/examples/inspect.groovy new file mode 100644 index 0000000..8ef97e9 --- /dev/null +++ b/sdk/java/examples/inspect.groovy @@ -0,0 +1,3 @@ +println "Inspecting ${args.name}" +println "Application loader: ${application}" +return [runtime: jmx.read('java.lang:type=Runtime', 'Name'), extensions: extensions.keySet()] diff --git a/sdk/java/examples/resources/META-INF/services/com.flanksource.telltale.DiagnosticExtension b/sdk/java/examples/resources/META-INF/services/com.flanksource.telltale.DiagnosticExtension new file mode 100644 index 0000000..d5df76b --- /dev/null +++ b/sdk/java/examples/resources/META-INF/services/com.flanksource.telltale.DiagnosticExtension @@ -0,0 +1 @@ +example.ExampleExtension diff --git a/sdk/java/src/harness/java/com/flanksource/telltale/harness/HarnessWorkload.java b/sdk/java/src/harness/java/com/flanksource/telltale/harness/HarnessWorkload.java new file mode 100644 index 0000000..e780f26 --- /dev/null +++ b/sdk/java/src/harness/java/com/flanksource/telltale/harness/HarnessWorkload.java @@ -0,0 +1,10 @@ +package com.flanksource.telltale.harness; + +/** A method the end-to-end test traces with jvm_trace while the harness keeps calling it. */ +public final class HarnessWorkload { + private HarnessWorkload() {} + + public static double compute(long value) { + return Math.sqrt(value); + } +} diff --git a/sdk/java/src/harness/java/com/flanksource/telltale/harness/TomcatHarness.java b/sdk/java/src/harness/java/com/flanksource/telltale/harness/TomcatHarness.java new file mode 100644 index 0000000..21e7b0c --- /dev/null +++ b/sdk/java/src/harness/java/com/flanksource/telltale/harness/TomcatHarness.java @@ -0,0 +1,62 @@ +package com.flanksource.telltale.harness; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.apache.catalina.startup.Tomcat; + +/** + * Embedded Tomcat that deploys one WAR on 127.0.0.1, prints {@code READY } and runs until + * stdin reaches EOF. Usage: {@code TomcatHarness }. + */ +public final class TomcatHarness { + private TomcatHarness() {} + + public static void main(String[] args) throws Exception { + if (args.length != 2) + throw new IllegalArgumentException("usage: TomcatHarness "); + Path war = Paths.get(args[0]); + if (!war.isAbsolute() || !Files.isRegularFile(war)) + throw new IllegalArgumentException("war must be an absolute path to a file: " + args[0]); + Path base = Files.createTempDirectory("telltale-harness-"); + Files.createDirectories(base.resolve("webapps")); + Tomcat tomcat = new Tomcat(); + tomcat.setAddDefaultWebXmlToWebapp(false); + tomcat.setBaseDir(base.toString()); + tomcat.setPort(0); + tomcat.getConnector().setProperty("address", "127.0.0.1"); + tomcat.addWebapp(args[1], war.toString()); + tomcat.start(); + startWorkload(); + System.out.println("READY " + tomcat.getConnector().getLocalPort()); + System.out.flush(); + awaitEof(System.in); + tomcat.stop(); + tomcat.destroy(); + } + + private static void startWorkload() { + Thread workload = + new Thread( + () -> { + long value = 0; + while (true) { + HarnessWorkload.compute(value++); + try { + Thread.sleep(10); + } catch (InterruptedException stop) { + return; + } + } + }, + "telltale-harness-workload"); + workload.setDaemon(true); + workload.start(); + } + + private static void awaitEof(InputStream input) throws Exception { + byte[] buffer = new byte[256]; + while (input.read(buffer) != -1) {} + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/Agent.java b/sdk/java/src/main/java/com/flanksource/telltale/Agent.java index 541df09..8e644c1 100644 --- a/sdk/java/src/main/java/com/flanksource/telltale/Agent.java +++ b/sdk/java/src/main/java/com/flanksource/telltale/Agent.java @@ -1,6 +1,7 @@ package com.flanksource.telltale; import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import com.google.gson.JsonObject; import java.lang.instrument.Instrumentation; import java.lang.management.ManagementFactory; @@ -15,19 +16,44 @@ import java.util.UUID; public final class Agent { - static final Gson JSON = new Gson(); + static final Gson JSON = new GsonBuilder().serializeNulls().create(); private static Agent instance; final JsonObject config; + final RuntimeClient diagnostics; final String pid = ManagementFactory.getRuntimeMXBean().getName().split("@")[0]; final String started = Instant.now().toString(); final String remoteID = "java/" + UUID.randomUUID(); private final ServerSocket listener; + private HttpListener http; + private final java.util.Set clients = java.util.Collections.synchronizedSet(new java.util.HashSet()); + private final java.util.Set sessions = new java.util.HashSet<>(); + final java.util.concurrent.atomic.AtomicReference stopFailure = new java.util.concurrent.atomic.AtomicReference<>(); + volatile boolean closed; + volatile Hub hub; + private Thread hubThread; private Agent(JsonObject options) throws Exception { config = options; if (!config.has("token") || config.get("token").getAsString().length() < 16) throw new IllegalArgumentException("Agent token is required"); + diagnostics = new RuntimeClient(JSON.fromJson(options, java.util.Map.class)); + com.flanksource.telltale.observer.ObserverRuntime.install(com.flanksource.telltale.diag.hotpatch.InstrumentationService.instrumentation()); listener = new ServerSocket(0, 16, InetAddress.getByName("127.0.0.1")); + try { + if (config.has("http")) http = HttpListener.start(JSON.fromJson(config.get("http"), java.util.Map.class)); + publishDiscovery(); + Thread accept = new Thread(() -> accept(), "telltale-agent"); + accept.setDaemon(true); + accept.start(); + if (config.has("url")) hubThread = Hub.start(this, config.get("url").getAsString()); + } catch (Exception | LinkageError failure) { + try { closeTransports(); } + catch (Exception cleanup) { failure.addSuppressed(cleanup); } + throw failure; + } + } + + private void publishDiscovery() throws Exception { if (config.has("discovery")) { JsonObject endpoint = new JsonObject(); endpoint.addProperty("pid", Integer.parseInt(pid)); @@ -54,10 +80,6 @@ private Agent(JsonObject options) throws Exception { java.nio.file.StandardCopyOption.ATOMIC_MOVE); path.toFile().deleteOnExit(); } - Thread accept = new Thread(() -> accept(), "telltale-agent"); - accept.setDaemon(true); - accept.start(); - if (config.has("url")) Hub.start(this, config.get("url").getAsString()); } private void accept() { @@ -68,18 +90,31 @@ private void accept() { Thread thread = new Thread( () -> { + Session session = null; try (Socket connection = socket; Wire wire = new Wire(connection)) { - new Session(this, wire, false).run(); + session = new Session(this, wire, false); + session.run(); } catch (Exception error) { - System.err.println("telltale local agent: " + error.getClass().getSimpleName()); + if (!closed) System.err.println("telltale local agent: " + error.getClass().getSimpleName()); + else if (session != null && !session.stopped()) stopFailure.compareAndSet(null, error); + } finally { + synchronized (clients) { + clients.remove(socket); + sessions.remove(Thread.currentThread()); + } } }, "telltale-session"); thread.setDaemon(true); - thread.start(); + synchronized (clients) { + if (closed) { socket.close(); return; } + clients.add(socket); + sessions.add(thread); + thread.start(); + } } catch (Exception error) { - System.err.println("telltale accept: " + error.getClass().getSimpleName()); + if (!closed) System.err.println("telltale accept: " + error.getClass().getSimpleName()); return; } } @@ -87,19 +122,82 @@ private void accept() { public static synchronized void agentmain(String args, Instrumentation instrumentation) throws Exception { - start(args); + com.flanksource.telltale.diag.agent.DiagAgent.agentmain(args, instrumentation); + System.setProperty("telltale.runtime.dir", Paths.get(args).toAbsolutePath().getParent().toString()); + start( + JSON.fromJson( + new String(Files.readAllBytes(Paths.get(args)), StandardCharsets.UTF_8), + java.util.Map.class)); + com.flanksource.telltale.observer.ObserverRuntime.install(instrumentation); } public static synchronized void premain(String args, Instrumentation instrumentation) throws Exception { - start(args); + com.flanksource.telltale.diag.agent.DiagAgent.premain(args, instrumentation); + System.setProperty("telltale.runtime.dir", Paths.get(args).toAbsolutePath().getParent().toString()); + start( + JSON.fromJson( + new String(Files.readAllBytes(Paths.get(args)), StandardCharsets.UTF_8), + java.util.Map.class)); + com.flanksource.telltale.observer.ObserverRuntime.install(instrumentation); } - private static void start(String path) throws Exception { - JsonObject config = - JSON.fromJson( - new String(Files.readAllBytes(Paths.get(path)), StandardCharsets.UTF_8), - JsonObject.class); + @SuppressWarnings("unchecked") + public static void validateRuntime(String path) throws Exception { + JsonObject config = JSON.fromJson(new String(Files.readAllBytes(Paths.get(path)), StandardCharsets.UTF_8), JsonObject.class); + if (config == null || !config.has("token") || config.get("token").getAsString().length() < 16) + throw new IllegalArgumentException("Agent token is required"); + AgentResource.from(JSON.fromJson(config, java.util.Map.class)); + if (config.has("http")) HttpListener.validate(JSON.fromJson(config.get("http"), java.util.Map.class)); + int kernel = com.flanksource.telltale.diag.agent.DiagAgent.probeApiVersion(); + if (kernel != com.flanksource.telltale.diag.agent.DiagAgent.PROBE_API_VERSION) + throw new IllegalStateException("incompatible probe bridge API " + kernel + "; restart the JVM to update the bootstrap bridge"); + } + + public static void prepareUpgrade() throws Exception { + RuntimeClient.request("prepareUpgrade", java.util.Collections.emptyMap()); + } + + public static synchronized void stopRuntime() throws Exception { + if (instance == null) { + javax.management.MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + if (server.isRegistered(new javax.management.ObjectName(RuntimeService.NAME))) { + prepareUpgrade(); + RuntimeClient.request("stopRuntime", java.util.Collections.emptyMap()); + } + return; + } + instance.closeTransports(); + RuntimeClient.request("stopRuntime", java.util.Collections.emptyMap()); + instance = null; + } + + private void closeTransports() throws Exception { + closed = true; + listener.close(); + java.util.List stopping; + synchronized (clients) { + for (Socket client : clients) client.close(); + stopping = new java.util.ArrayList<>(sessions); + } + long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(6); + for (Thread session : stopping) { + session.join(Math.max(1, java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(deadline - System.nanoTime()))); + if (session.isAlive()) throw new IllegalStateException("agent session did not stop for upgrade"); + } + if (stopFailure.get() != null) throw new IllegalStateException("agent session cleanup failed", stopFailure.get()); + if (hub != null) hub.close(); + if (hubThread != null) { + hubThread.interrupt(); + hubThread.join(5000); + if (hubThread.isAlive()) throw new IllegalStateException("hub did not stop for upgrade"); + } + if (stopFailure.get() != null) throw new IllegalStateException("hub session cleanup failed", stopFailure.get()); + if (http != null) http.close(); + } + + public static synchronized void start(java.util.Map options) throws Exception { + JsonObject config = JSON.toJsonTree(options).getAsJsonObject(); if (instance != null) { if (!instance.config.equals(config)) throw new IllegalStateException( @@ -109,7 +207,7 @@ private static void start(String path) throws Exception { instance = new Agent(config); } - JsonObject hello(boolean remote) { + JsonObject hello(boolean remote) throws Exception { JsonObject hello = new JsonObject(); hello.addProperty("agentId", remote ? remoteID : pid); hello.addProperty("protocol", 1); @@ -121,18 +219,9 @@ JsonObject hello(boolean remote) { runtime.addProperty("version", System.getProperty("java.version")); runtime.addProperty("vendor", System.getProperty("java.vendor")); hello.add("runtime", runtime); - JsonObject resource = - config.has("resource") ? config.getAsJsonObject("resource").deepCopy() : new JsonObject(); - resource.addProperty("process.pid", pid); - resource.addProperty("service.name", Recording.value(config, "service", "java")); - hello.add("resource", resource); - JsonObject probe = new JsonObject(); - probe.addProperty("name", "jvm_jfr"); - probe.addProperty("title", "JVM JFR"); - probe.addProperty("checkpoints", true); - com.google.gson.JsonArray probes = new com.google.gson.JsonArray(); - probes.add(probe); - hello.add("probes", probes); + hello.add("resource", JSON.toJsonTree(AgentResource.from(JSON.fromJson(config, java.util.Map.class)))); + hello.add("probes", JSON.toJsonTree(RuntimeService.probes())); + hello.add("ops", JSON.toJsonTree(diagnostics.catalog())); return hello; } } diff --git a/sdk/java/src/main/java/com/flanksource/telltale/AgentResource.java b/sdk/java/src/main/java/com/flanksource/telltale/AgentResource.java new file mode 100644 index 0000000..0f19fd9 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/AgentResource.java @@ -0,0 +1,25 @@ +package com.flanksource.telltale; + +import com.flanksource.telltale.observer.ObserverRuntime; +import java.lang.management.ManagementFactory; +import java.util.*; + +final class AgentResource { + private AgentResource() {} + + @SuppressWarnings("unchecked") + static Map from(Map config) { + Object supplied = config.get("resource"); + if (supplied != null && !(supplied instanceof Map)) + throw new IllegalArgumentException("agent resource must be an object"); + Map resource = supplied == null ? new LinkedHashMap<>() + : new LinkedHashMap<>((Map) supplied); + Object service = config.containsKey("service") ? config.get("service") : "java"; + if (!(service instanceof String) || ((String) service).isEmpty()) + throw new IllegalArgumentException("agent service must be a nonempty string"); + resource.put("service.name", service); + resource.put("process.pid", ManagementFactory.getRuntimeMXBean().getName().split("@")[0]); + resource.put("telltale.source.instance", ObserverRuntime.SOURCE); + return resource; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/Attach.java b/sdk/java/src/main/java/com/flanksource/telltale/Attach.java index d9b8741..e915de0 100644 --- a/sdk/java/src/main/java/com/flanksource/telltale/Attach.java +++ b/sdk/java/src/main/java/com/flanksource/telltale/Attach.java @@ -1,18 +1,31 @@ package com.flanksource.telltale; +import java.lang.reflect.InvocationTargetException; +import java.nio.charset.StandardCharsets; +import java.nio.file.*; public final class Attach { private Attach() {} public static void main(String[] args) throws Exception { - if (args.length != 3) - throw new IllegalArgumentException("Expected PID, agent JAR and config file"); + if (args.length != 4) + throw new IllegalArgumentException("Expected PID, agent JAR, config file and upgrade flag"); Class vm = Class.forName("com.sun.tools.attach.VirtualMachine"); Object attached = vm.getMethod("attach", String.class).invoke(null, args[0]); + Path result = Files.createTempFile(Paths.get(args[2]).toAbsolutePath().getParent(), "attach-", ".result"); try { - vm.getMethod("loadAgent", String.class, String.class).invoke(attached, args[1], args[2]); + try { + vm.getMethod("loadAgent", String.class, String.class).invoke(attached, args[1], args[2] + "\n" + args[1] + "\n" + args[3] + "\n" + result); + } catch (InvocationTargetException failure) { + String status = new String(Files.readAllBytes(result), StandardCharsets.UTF_8); + if (status.startsWith("error\n")) throw new IllegalStateException(status.substring(6), failure.getCause()); + throw failure; + } + String status = new String(Files.readAllBytes(result), StandardCharsets.UTF_8); + if (!status.equals("ok")) throw new IllegalStateException("agent did not acknowledge attachment: " + status); } finally { - vm.getMethod("detach").invoke(attached); + try { vm.getMethod("detach").invoke(attached); } + finally { Files.deleteIfExists(result); } } } } diff --git a/sdk/java/src/main/java/com/flanksource/telltale/Capture.java b/sdk/java/src/main/java/com/flanksource/telltale/Capture.java new file mode 100644 index 0000000..d58d594 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/Capture.java @@ -0,0 +1,37 @@ +package com.flanksource.telltale; + +public interface Capture extends AutoCloseable { + byte[] dump() throws Exception; + + String contentType(); + + String fileName(); + + default boolean streaming() { + return false; + } + + default long intervalMillis() { return 250; } + default long dropped() { return 0; } + + default boolean complete() { + return false; + } + + /** + * Figures about the capture as a whole — what it withheld, rather than what it recorded — sent as + * a {@code report} frame after each blob that changed them; null when the capture has none. + */ + default java.util.Map report() { + return null; + } + + default void renew() throws Exception {} + + default void freeze() throws Exception {} + + default byte[] finish() throws Exception { + freeze(); + return dump(); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/DiagnosticExtension.java b/sdk/java/src/main/java/com/flanksource/telltale/DiagnosticExtension.java new file mode 100644 index 0000000..7e917b8 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/DiagnosticExtension.java @@ -0,0 +1,16 @@ +package com.flanksource.telltale; + +/** Implementations are discovered from extension JAR service descriptors. */ +public interface DiagnosticExtension extends AutoCloseable { + String id(); + + String version(); + + default int apiVersion() { + return 1; + } + + void start(ExtensionContext context) throws Exception; + + void close() throws Exception; +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/DiagnosticSchemas.java b/sdk/java/src/main/java/com/flanksource/telltale/DiagnosticSchemas.java new file mode 100644 index 0000000..0b671f8 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/DiagnosticSchemas.java @@ -0,0 +1,268 @@ +package com.flanksource.telltale; + +import java.util.*; + +final class DiagnosticSchemas { + /** A built-in operation's description and the CLI/API command it is served as. */ + private static final Map DOCS = new HashMap<>(); + + private static void doc(String name, String command, String description) { + DOCS.put(name, new String[] {command, description}); + } + + static { + doc("jvm.settings", "jvm settings", "Read OS/JVM properties, environment and resource settings"); + doc("jvm.resources", "jvm resources", "Read filesystem capacity and Linux process and cgroup resources"); + doc("jvm.flags", "jvm flags", "Read effective HotSpot VM flags"); + doc("jvm.extensions.list", "jvm extensions list", "List loaded diagnostic extensions"); + doc("jvm.extensions.install", "jvm extensions install", "Install a diagnostic extension JAR"); + doc("jvm.extensions.replace", "jvm extensions replace", "Replace a loaded diagnostic extension with a new JAR"); + doc("jvm.extensions.unload", "jvm extensions unload", "Unload a diagnostic extension"); + doc("jvm.catalog", "jvm catalog", "List the operations this runtime supports"); + doc("jvm.status", "jvm status", "Report mutation permission, instrumentation and active patches"); + doc("jvm.jmx.search", "jvm jmx search", "Search MBean names matching an ObjectName pattern"); + doc("jvm.jmx.info", "jvm jmx info", "Describe an MBean's attributes, operations and notifications"); + doc("jvm.jmx.read", "jvm jmx read", "Read one MBean attribute, or all of them"); + doc("jvm.jmx.write", "jvm jmx write", "Write an MBean attribute"); + doc("jvm.jmx.invoke", "jvm jmx invoke", "Invoke an MBean operation"); + doc("jvm.metrics", "jvm metrics", "Snapshot memory, GC, thread, class-loading and runtime metrics"); + doc("jvm.telemetry", "jvm telemetry", "Read native activity telemetry from every loaded application without arming it or using JMX"); + doc("jvm.threads", "jvm threads", "Sample thread states and CPU over a short interval"); + doc("jvm.threads.dump", "jvm threads dump", "Dump every thread's stack as text"); + doc("jvm.threads.stack", "jvm threads stack", "Read one thread's stack and lock state"); + doc("jvm.heap.histogram", "jvm heap histogram", "List the heap's classes by retained bytes (GC.class_histogram)"); + doc("jvm.logging.list", "jvm logging list", "List java.util.logging loggers and their levels"); + doc("jvm.logging.set", "jvm logging set", "Set a logger's level, or inherit its parent's with null"); + doc("jvm.classes", "jvm classes", "Page through loaded classes by name and loader"); + doc("jvm.loaders", "jvm loaders", "List class loaders and their identities"); + doc("jvm.groovy.eval", "jvm groovy eval", "Execute a Groovy script once in an application loader"); + doc("jvm.ognl.eval", "jvm ognl eval", "Evaluate an OGNL expression in an application loader"); + doc("jvm.classes.source", "jvm decompile", "Decompile a class with its descriptors and line mappings"); + doc("jvm.classes.bytecode", "jvm decompile bytecode", "Read a class's bytecode as base64"); + doc("jvm.classes.compile", "jvm patch compile", "Preview compiling replacement source without applying it"); + doc("jvm.classes.patch", "jvm patch apply", "Compile and apply replacement source under a restore lease"); + doc("jvm.classes.restore", "jvm patch restore", "Restore a class's original bytecode, or remove one probe"); + doc("jvm.classes.lease", "jvm patch lease", "Renew or change a patched class's restore lease"); + doc("jvm.classes.patches", "jvm patch list", "List active bytecode patches and probes"); + doc("jvm.classes.instrument", "jvm trace instrument", "Add a method probe to a class under a restore lease"); + doc("jvm.probes.list", "jvm trace probes", "List method probe keys"); + doc("jvm.probes.read", "jvm trace read", "Read buffered events of a method probe"); + doc("jvm.probes.freeze", "jvm trace freeze", "Stop a method probe from recording further events"); + doc("jvm.notifications.list", "jvm notifications list", "List JMX notification subscriptions and their queues"); + } + + private static String[] docs(String name) { + String[] docs = DOCS.get(name); + if (docs == null) throw new IllegalStateException("operation has no command or description: " + name); + return docs; + } + + static String command(String name) { + return docs(name)[0]; + } + + static String description(String name) { + return docs(name)[1]; + } + + static Map result(String name) { + Map schema = new LinkedHashMap<>(); + if (name.equals("jvm.jmx.read") || name.equals("jvm.jmx.invoke")) return schema; + if (name.equals("jvm.threads.dump") || name.equals("jvm.flags")) { + schema.put("type", "string"); + return schema; + } + boolean array = + Arrays.asList( + "jvm.jmx.search", + "jvm.logging.list", + "jvm.loaders", + "jvm.classes.patches", + "jvm.extensions.list", + "jvm.catalog", + "jvm.notifications.list") + .contains(name); + schema.put("type", array ? "array" : "object"); + if (array) + schema.put( + "items", + Collections.singletonMap("type", name.equals("jvm.jmx.search") ? "string" : "object")); + if (name.equals("jvm.groovy.eval") || name.equals("jvm.ognl.eval")) { + Map properties = new LinkedHashMap<>(); + properties.put("value", Collections.emptyMap()); + add(properties, "graph", "object"); + if (name.contains("groovy")) { + add(properties, "stdout", "string"); + add(properties, "stderr", "string"); + add(properties, "elapsedMs", "integer"); + } + schema.put("properties", properties); + } + if (name.equals("jvm.heap.histogram")) { + Map properties = new LinkedHashMap<>(); + add(properties, "collectedAt", "string"); + add(properties, "live", "boolean"); + add(properties, "totalInstances", "integer"); + add(properties, "totalBytes", "integer"); + add(properties, "matched", "integer"); + add(properties, "rows", "array"); + schema.put("properties", properties); + } + if (name.equals("jvm.telemetry")) { + Map properties = new LinkedHashMap<>(); + add(properties, "capturedAt", "string"); + Map application = new LinkedHashMap<>(); + add(application, "loader", "string"); + add(application, "label", "string"); + add(application, "status", "object"); + add(application, "histograms", "array"); + Map item = new LinkedHashMap<>(); + item.put("type", "object"); + item.put("properties", application); + Map applications = new LinkedHashMap<>(); + applications.put("type", "array"); + applications.put("items", item); + properties.put("applications", applications); + schema.put("properties", properties); + } + return schema; + } + + static Map parameters(String name) { + Map properties = new LinkedHashMap<>(); + List required = new ArrayList<>(); + switch (name) { + case "jvm.jmx.search": + add(properties, "pattern", "string"); + break; + case "jvm.jmx.info": + case "jvm.jmx.read": + case "jvm.jmx.write": + case "jvm.jmx.invoke": + field(properties, required, "objectName", "string"); + if (name.endsWith("read")) add(properties, "attribute", "string"); + if (name.endsWith("write")) { + field(properties, required, "attribute", "string"); + properties.put("value", Collections.emptyMap()); + required.add("value"); + } + if (name.endsWith("invoke")) { + field(properties, required, "operation", "string"); + add(properties, "args", "array"); + add(properties, "signature", "array"); + } + break; + case "jvm.groovy.eval": + case "jvm.ognl.eval": + field(properties, required, name.contains("groovy") ? "source" : "expression", "string"); + add(properties, "loader", "string"); + if (name.contains("groovy")) { + add(properties, "args", "object"); + add(properties, "timeoutSeconds", "integer"); + } + break; + case "jvm.classes": + add(properties, "query", "string"); + add(properties, "loader", "string"); + add(properties, "limit", "integer"); + add(properties, "offset", "integer"); + add(properties, "includePlatform", "boolean"); + break; + case "jvm.classes.source": + case "jvm.classes.bytecode": + case "jvm.classes.compile": + case "jvm.classes.patch": + case "jvm.classes.instrument": + case "jvm.classes.restore": + case "jvm.classes.lease": + field(properties, required, "class", "string"); + add(properties, "loader", "string"); + add(properties, "method", "string"); + if (name.endsWith("compile") || name.endsWith("patch")) + field(properties, required, "source", "string"); + if (name.endsWith("compile") || name.endsWith("patch")) + add(properties, "classpath", "string"); + if (name.endsWith("instrument")) field(properties, required, "probe", "object"); + if (name.endsWith("restore")) add(properties, "probe", "object"); + if (name.endsWith("lease")) add(properties, "seconds", "integer"); + if (name.endsWith("patch") || name.endsWith("instrument")) + add(properties, "leaseSeconds", "integer"); + break; + case "jvm.probes.read": + field(properties, required, "key", "string"); + add(properties, "since", "integer"); + add(properties, "limit", "integer"); + break; + case "jvm.probes.freeze": + field(properties, required, "key", "string"); + break; + case "jvm.logging.set": + field(properties, required, "name", "string"); + Map level = new LinkedHashMap<>(); + level.put("type", "string"); + level.put("nullable", true); + level.put("title", "level"); + level.put("description", "Use null to inherit the parent logger level."); + properties.put("level", level); + required.add("level"); + break; + case "jvm.threads": + add(properties, "sampleMs", "integer"); + break; + case "jvm.heap.histogram": + add(properties, "top", "integer"); + add(properties, "classPattern", "string"); + add(properties, "live", "boolean"); + describe(properties, "top", "Rows to return, largest first (default 100)."); + describe(properties, "classPattern", "Glob over class names, e.g. com.example.*"); + describe( + properties, "live", "Count only reachable objects. Forces a full GC on this JVM first."); + break; + case "jvm.threads.stack": + field(properties, required, "id", "integer"); + break; + case "jvm.extensions.install": + case "jvm.extensions.replace": + field(properties, required, "jar", "string"); + break; + case "jvm.extensions.unload": + field(properties, required, "id", "string"); + break; + default: + break; + } + Map schema = new LinkedHashMap<>(); + schema.put("type", "object"); + schema.put("properties", properties); + schema.put("required", required); + schema.put("additionalProperties", false); + return schema; + } + + private static void field( + Map properties, List required, String name, String type) { + add(properties, name, type); + required.add(name); + } + + @SuppressWarnings("unchecked") + private static void describe(Map properties, String name, String description) { + ((Map) properties.get(name)).put("description", description); + } + + private static void add(Map properties, String name, String type) { + Map schema = new LinkedHashMap<>(); + schema.put("type", type); + schema.put("title", name); + if (name.equals("source") || name.equals("expression")) schema.put("format", "textarea"); + if (type.equals("array")) + schema.put( + "items", + name.equals("signature") + ? Collections.singletonMap("type", "string") + : Collections.emptyMap()); + if (type.equals("object")) schema.put("additionalProperties", Collections.emptyMap()); + if (name.equals("jar")) schema.put("format", "byte"); + properties.put(name, schema); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/Diagnostics.java b/sdk/java/src/main/java/com/flanksource/telltale/Diagnostics.java new file mode 100644 index 0000000..3d899f6 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/Diagnostics.java @@ -0,0 +1,249 @@ +package com.flanksource.telltale; + +import com.flanksource.telltale.diag.hotpatch.*; +import com.flanksource.telltale.diag.jmx.*; +import com.flanksource.telltale.diag.jvm.*; +import java.util.*; + +/** Shared operation dispatcher for the agent and servlet adapters. */ +public final class Diagnostics { + public interface Operation { + Object call(Map params) throws Exception; + } + + private final Map operations = new LinkedHashMap<>(); + private final Map scopes = new HashMap<>(); + private final boolean allowMutation; + private final JmxService jmx = new JmxService(); + private final Extensions extensions; + private final ClassOperations classes = new ClassOperations(); + + void close() throws Exception { extensions.close(); } + private final NotificationHub notifications = + new NotificationHub(java.lang.management.ManagementFactory.getPlatformMBeanServer()); + + public Diagnostics(Map config) throws Exception { + allowMutation = Boolean.TRUE.equals(config.get("allowMutation")); + extensions = new Extensions(text(config, "extensionsDir", null)); + register("jvm.extensions.list", Scope.READ, p -> extensions.list()); + register("jvm.extensions.install", Scope.INSTRUMENT, p -> extensions.install(p, false)); + register("jvm.extensions.replace", Scope.INSTRUMENT, p -> extensions.install(p, true)); + register("jvm.extensions.unload", Scope.INSTRUMENT, p -> extensions.unload(required(p, "id"))); + register("jvm.catalog", Scope.READ, p -> catalog()); + register( + "jvm.status", + Scope.READ, + p -> { + Map status = new LinkedHashMap<>(); + status.put("allowMutation", allowMutation); + status.put("instrumentation", InstrumentationService.installed()); + status.put("observers", com.flanksource.telltale.observer.ObserverRuntime.capabilities()); + if (InstrumentationService.installed()) + status.put( + "patches", + new PatchService(InstrumentationService.instrumentation()).patches(index())); + return status; + }); + register("jvm.jmx.search", Scope.READ, p -> jmx.search(text(p, "pattern", ""))); + register("jvm.jmx.info", Scope.READ, p -> jmx.info(required(p, "objectName"))); + register( + "jvm.jmx.read", + Scope.READ, + p -> + p.containsKey("attribute") + ? jmx.read(required(p, "objectName"), required(p, "attribute")) + : jmx.readAll(required(p, "objectName"))); + register( + "jvm.jmx.write", + Scope.WRITE, + p -> { + jmx.write(required(p, "objectName"), required(p, "attribute"), p.get("value")); + return Collections.singletonMap("applied", true); + }); + register( + "jvm.jmx.invoke", + Scope.WRITE, + p -> + jmx.invoke( + required(p, "objectName"), + required(p, "operation"), + list(p, "args").toArray(), + list(p, "signature").toArray(new String[0]))); + register("jvm.metrics", Scope.READ, p -> new JvmMetricsService().snapshot()); + register("jvm.telemetry", Scope.READ, p -> new NativeTelemetryService(InstrumentationService.instrumentation()).snapshot()); + register("jvm.settings", Scope.READ, p -> SettingsService.snapshot()); + register("jvm.resources", Scope.READ, p -> ResourceSettings.snapshot()); + register("jvm.flags", Scope.READ, p -> SettingsService.command("vmFlags", "-all")); + register( + "jvm.threads", + Scope.READ, + p -> + new ThreadDumpService() + .snapshot(((Number) p.getOrDefault("sampleMs", 100)).intValue())); + register("jvm.threads.dump", Scope.READ, p -> new ThreadDumpService().dump()); + register("jvm.heap.histogram", Scope.READ, p -> new HeapHistogramService().histogram(p)); + register( + "jvm.threads.stack", + Scope.READ, + p -> { + Object id = p.get("id"); + if (!(id instanceof Number)) throw new IllegalArgumentException("id must be an integer"); + try { + return new ThreadDumpService() + .stack(new java.math.BigDecimal(id.toString()).longValueExact()); + } catch (ArithmeticException | NumberFormatException error) { + throw new IllegalArgumentException("id must be a 64-bit integer", error); + } + }); + register("jvm.logging.list", Scope.READ, p -> new LoggingService().loggers()); + register( + "jvm.logging.set", + Scope.WRITE, + p -> { + if (!(p.get("name") instanceof String)) + throw new IllegalArgumentException("name must be a string (empty names the root)"); + if (!p.containsKey("level")) throw new IllegalArgumentException("level is required"); + new LoggingService().setLevel((String) p.get("name"), text(p, "level", null)); + return Collections.singletonMap("applied", true); + }); + register( + "jvm.classes", + Scope.READ, + p -> + index() + .page( + text(p, "loader", null), + text(p, "query", ""), + ((Number) p.getOrDefault("limit", 200)).intValue(), + ((Number) p.getOrDefault("offset", 0)).intValue(), + Boolean.TRUE.equals(p.get("includePlatform")))); + register("jvm.loaders", Scope.READ, p -> index().loaders()); + register( + "jvm.groovy.eval", + Scope.EVAL, + p -> { + ClassLoader application = loader(p); + try (Extensions.BindingsLease bindings = extensions.bindings(application)) { + return Scripts.evaluate(p, application, jmx, bindings.values); + } + }); + register("jvm.ognl.eval", Scope.EVAL, p -> OgnlEvaluation.evaluate(p, loader(p))); + register("jvm.classes.source", Scope.READ, p -> classes.source(new JsonBody(p))); + register( + "jvm.classes.bytecode", + Scope.READ, + p -> + Collections.singletonMap( + "bytecode", Base64.getEncoder().encodeToString(classes.bytecode(new JsonBody(p))))); + register("jvm.classes.compile", Scope.READ, p -> classes.compile(new JsonBody(p))); + register("jvm.classes.patch", Scope.INSTRUMENT, p -> classes.patch(new JsonBody(p))); + register("jvm.classes.instrument", Scope.INSTRUMENT, p -> classes.instrument(new JsonBody(p))); + register("jvm.classes.restore", Scope.INSTRUMENT, p -> classes.restore(new JsonBody(p))); + register("jvm.classes.lease", Scope.INSTRUMENT, p -> classes.lease(new JsonBody(p))); + register( + "jvm.classes.patches", + Scope.READ, + p -> new PatchService(InstrumentationService.instrumentation()).patches(index())); + register("jvm.probes.read", Scope.READ, p -> classes.probes(new JsonBody(p))); + register("jvm.probes.list", Scope.READ, p -> classes.probeKeys()); + register("jvm.probes.freeze", Scope.TRACE, p -> classes.freezeProbe(new JsonBody(p))); + register("jvm.notifications.list", Scope.READ, p -> notifications.list()); + } + + public synchronized void register(String name, Scope scope, Operation operation) { + if (operations.containsKey(name)) + throw new IllegalArgumentException("duplicate operation: " + name); + operations.put(name, operation); + scopes.put(name, Objects.requireNonNull(scope, "scope")); + } + + public synchronized List> catalog() { + List> result = new ArrayList<>(); + for (String name : operations.keySet()) { + Map spec = + OperationSpec.named(name) + .description(DiagnosticSchemas.description(name)) + .command(DiagnosticSchemas.command(name)) + .parameters(DiagnosticSchemas.parameters(name)) + .result(DiagnosticSchemas.result(name)) + .mutates(scopes.get(name).mutates()) + .descriptor(); + spec.put("scope", scopes.get(name).wire()); + result.add(spec); + } + result.addAll(extensions.catalog()); + return result; + } + + /** The scope a caller needs for an operation; unknown operations fail closed. */ + public Scope scope(String name) { + if (name.startsWith("extension.")) return extensions.scope(name); + synchronized (this) { + Scope scope = scopes.get(name); + if (scope == null) throw new IllegalArgumentException("unknown operation: " + name); + return scope; + } + } + + public Object call(String name, Map params, boolean callerMutation) + throws Exception { + if (name.startsWith("extension.")) { + return extensions.call(name, params, allowMutation && callerMutation); + } + Operation operation; + synchronized (this) { + operation = operations.get(name); + if (operation == null) throw new IllegalArgumentException("unknown operation: " + name); + if (scopes.get(name).mutates() && (!allowMutation || !callerMutation)) + throw new IllegalStateException("mutation disabled for " + name); + } + return operation.call(params); + } + + private ClassIndex index() { + return ClassIndex.from(InstrumentationService.instrumentation()); + } + + Capture capture(String kind, Map params, String key, boolean callerMutation, + Map resource) + throws Exception { + if (RuntimeService.OBSERVATIONS.contains(kind)) + return new ObserverCapture(kind, params, key, resource); + if (kind.equals("jvm_notifications")) return new NotificationCapture(notifications, params); + if (!allowMutation || !callerMutation) + throw new IllegalStateException("mutation disabled for " + kind); + if (kind.equals("jvm_ognl")) return new ExpressionCapture(params, loader(params)); + if (!kind.equals("jvm_trace") && !kind.equals("jvm_redefine")) + throw new IllegalArgumentException("unknown capture kind: " + kind); + return new MethodCapture(classes, kind, params, key); + } + + private ClassLoader loader(Map p) { + if (!p.containsKey("loader")) return ClassLoader.getSystemClassLoader(); + String hash = required(p, "loader"); + for (Class type : InstrumentationService.instrumentation().getAllLoadedClasses()) { + if (ClassIndex.hash(type.getClassLoader()).equals(hash)) return type.getClassLoader(); + } + throw new IllegalArgumentException("unknown loader: " + hash); + } + + static String required(Map p, String key) { + String value = text(p, key, ""); + if (value.trim().isEmpty()) throw new IllegalArgumentException(key + " is required"); + return value; + } + + static String text(Map p, String key, String defaultValue) { + Object value = p.get(key); + if (value == null) return defaultValue; + if (!(value instanceof String)) throw new IllegalArgumentException(key + " must be a string"); + return (String) value; + } + + private static List list(Map p, String key) { + Object value = p.get(key); + if (value == null) return Collections.emptyList(); + if (!(value instanceof List)) throw new IllegalArgumentException(key + " must be an array"); + return (List) value; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/ExpressionCapture.java b/sdk/java/src/main/java/com/flanksource/telltale/ExpressionCapture.java new file mode 100644 index 0000000..0b5fead --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/ExpressionCapture.java @@ -0,0 +1,50 @@ +package com.flanksource.telltale; + +import java.nio.charset.StandardCharsets; +import java.util.*; + +final class ExpressionCapture implements Capture { + private final Map params; + private final ClassLoader loader; + private final long interval; + private long next; + private boolean closed; + + ExpressionCapture(Map params, ClassLoader loader) { + Diagnostics.required(params, "expression"); + this.params = new LinkedHashMap<>(params); + this.loader = loader; + double millis = ((Number) params.getOrDefault("intervalMs", 1000)).doubleValue(); + if (millis < 100 || millis > 3600000 || millis != Math.rint(millis)) + throw new IllegalArgumentException("intervalMs must be an integer between 100 and 3600000"); + interval = (long) millis; + } + + public synchronized byte[] dump() throws Exception { + if (closed) throw new IllegalStateException("expression capture is closed"); + long now = System.currentTimeMillis(); + if (now < next) return "[]".getBytes(StandardCharsets.UTF_8); + Map row = new LinkedHashMap<>(); + row.put("timestamp", java.time.Instant.now().toString()); + row.put("expression", params.get("expression")); + row.put("result", OgnlEvaluation.evaluate(params, loader)); + next = now + interval; + return Agent.JSON.toJson(Collections.singletonList(row)).getBytes(StandardCharsets.UTF_8); + } + + public synchronized void close() { + closed = true; + } + + public boolean streaming() { + return true; + } + + public String contentType() { + return "application/json"; + } + + public String fileName() { + return "expressions.json"; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/ExtensionContext.java b/sdk/java/src/main/java/com/flanksource/telltale/ExtensionContext.java new file mode 100644 index 0000000..06b774d --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/ExtensionContext.java @@ -0,0 +1,53 @@ +package com.flanksource.telltale; + +import java.lang.management.ManagementFactory; +import java.util.*; +import javax.management.MBeanServer; + +public final class ExtensionContext implements AutoCloseable { + public interface Bindings { + Map forLoader(ClassLoader application) throws Exception; + } + + final Map operations = new LinkedHashMap<>(); + private final List resources = new ArrayList<>(); + private Bindings bindings = application -> Collections.emptyMap(); + + public void register(OperationSpec operation) { + if (operation.handler == null) + throw new IllegalArgumentException("operation handler is required: " + operation.name); + if (operations.putIfAbsent(operation.name, operation) != null) + throw new IllegalArgumentException("duplicate operation: " + operation.name); + } + + public void bindings(Bindings provider) { + bindings = Objects.requireNonNull(provider); + } + + public void own(AutoCloseable resource) { + resources.add(Objects.requireNonNull(resource)); + } + + public MBeanServer mbeans() { + return ManagementFactory.getPlatformMBeanServer(); + } + + Map bindings(ClassLoader application) throws Exception { + return bindings.forLoader(application); + } + + public void close() throws Exception { + Exception failure = null; + ListIterator iterator = resources.listIterator(resources.size()); + while (iterator.hasPrevious()) { + try { + iterator.previous().close(); + iterator.remove(); + } catch (Exception error) { + if (failure == null) failure = error; + else failure.addSuppressed(error); + } + } + if (failure != null) throw failure; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/Extensions.java b/sdk/java/src/main/java/com/flanksource/telltale/Extensions.java new file mode 100644 index 0000000..3ed61c8 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/Extensions.java @@ -0,0 +1,340 @@ +package com.flanksource.telltale; + +import java.io.*; +import java.net.*; +import java.nio.file.*; +import java.security.MessageDigest; +import java.util.*; + +final class Extensions implements AutoCloseable { + private final Map loaded = new LinkedHashMap<>(); + private final Path directory; + + private static final class Loaded implements AutoCloseable { + final DiagnosticExtension extension; + final URLClassLoader loader; + final ExtensionContext context; + final String digest; + int active; + boolean failed; + boolean extensionClosed; + boolean contextClosed; + + Loaded( + DiagnosticExtension extension, + URLClassLoader loader, + ExtensionContext context, + String digest) { + this.extension = extension; + this.loader = loader; + this.context = context; + this.digest = digest; + } + + public void close() throws Exception { + failed = true; + if (!extensionClosed) { + extension.close(); + extensionClosed = true; + } + if (!contextClosed) { + context.close(); + contextClosed = true; + } + loader.close(); + } + } + + Extensions(String directory) throws Exception { + this.directory = directory == null ? null : Paths.get(directory); + if (this.directory != null && !this.directory.isAbsolute()) + throw new IllegalArgumentException("extensionsDir must be absolute"); + if (this.directory == null || !Files.exists(this.directory.resolve("active.properties"))) + return; + Properties active = new Properties(); + try (InputStream input = Files.newInputStream(this.directory.resolve("active.properties"))) { + active.load(input); + } + try { + for (String id : active.stringPropertyNames()) { + String digest = active.getProperty(id); + if (!digest.matches("[a-f0-9]{64}")) + throw new IllegalStateException("invalid extension digest for " + id); + Path jar = this.directory.resolve(digest + ".jar"); + if (!digest.equals(digest(Files.readAllBytes(jar)))) + throw new IllegalStateException("extension digest mismatch for " + id); + Loaded item = load(jar, digest); + if (!id.equals(item.extension.id())) { + item.close(); + throw new IllegalStateException("extension identity mismatch for " + id); + } + loaded.put(id, item); + } + } catch (Exception error) { + try { + close(); + } catch (Exception cleanup) { + error.addSuppressed(cleanup); + } + throw error; + } + } + + synchronized List> list() { + List> result = new ArrayList<>(); + for (Loaded item : loaded.values()) { + Map row = new LinkedHashMap<>(); + row.put("id", item.extension.id()); + row.put("version", item.extension.version()); + row.put("digest", item.digest); + row.put("active", item.active); + row.put("status", item.failed ? "failed" : "active"); + row.put("operations", item.context.operations.keySet()); + result.add(row); + } + return result; + } + + synchronized List> catalog() { + List> result = new ArrayList<>(); + for (Loaded item : loaded.values()) + if (!item.failed) + for (OperationSpec operation : item.context.operations.values()) { + Map spec = operation.descriptor(); + spec.put("scope", Scope.forExtension(operation.mutates).wire()); + spec.put("extension", item.extension.id()); + result.add(spec); + } + return result; + } + + synchronized Object install(Map params, boolean replace) throws Exception { + if (directory == null) + throw new IllegalStateException("an absolute extensionsDir must be configured"); + String encoded = Diagnostics.required(params, "jar"); + if (encoded.length() > 24 * 1024 * 1024) + throw new IllegalArgumentException("extension exceeds 16 MiB"); + byte[] bytes = Base64.getDecoder().decode(encoded); + if (bytes.length > 16 * 1024 * 1024) + throw new IllegalArgumentException("extension exceeds 16 MiB"); + Files.createDirectories(directory); + String digest = digest(bytes); + Path jar = directory.resolve(digest + ".jar"); + if (!Files.exists(jar)) Files.write(jar, bytes, StandardOpenOption.CREATE_NEW); + if (!digest.equals(digest(Files.readAllBytes(jar)))) + throw new IllegalStateException("stored extension digest mismatch"); + Loaded candidate = load(jar, digest); + boolean accepted = false; + try { + String id = candidate.extension.id(); + Loaded existing = loaded.get(id); + if (existing != null) { + if (!replace) + throw new IllegalStateException("extension already installed; use replace: " + id); + if (existing.active != 0) + throw new IllegalStateException("extension has active calls or scripts: " + id); + existing.close(); + } + Map planned = new LinkedHashMap<>(loaded); + planned.put(id, candidate); + persist(planned); + loaded.put(id, candidate); + accepted = true; + return Collections.singletonMap("installed", id); + } catch (Exception | LinkageError | ServiceConfigurationError error) { + if (!accepted) + try { + candidate.close(); + } catch (Exception cleanup) { + error.addSuppressed(cleanup); + } + if (error instanceof ServiceConfigurationError || error instanceof LinkageError) + throw new IllegalArgumentException("load extension provider: " + error, error); + throw error; + } + } + + synchronized Object unload(String id) throws Exception { + Loaded item = loaded.get(id); + if (item == null) throw new IllegalArgumentException("unknown extension: " + id); + if (item.active != 0) + throw new IllegalStateException("extension has active calls or scripts: " + id); + item.close(); + Map planned = new LinkedHashMap<>(loaded); + planned.remove(id); + persist(planned); + loaded.remove(id); + return Collections.singletonMap("unloaded", id); + } + + synchronized Scope scope(String name) { + for (Loaded item : loaded.values()) { + OperationSpec operation = item.context.operations.get(name); + if (operation != null) return Scope.forExtension(operation.mutates); + } + throw new IllegalArgumentException("unknown operation: " + name); + } + + Object call(String name, Map params, boolean allowed) throws Exception { + Loaded owner = null; + OperationSpec operation = null; + synchronized (this) { + for (Loaded item : loaded.values()) { + operation = item.context.operations.get(name); + if (operation != null) { + if (item.failed) + throw new IllegalStateException("extension cleanup failed: " + item.extension.id()); + if (operation.mutates && !allowed) + throw new IllegalStateException("mutation disabled for " + name); + owner = item; + item.active++; + break; + } + } + } + if (owner == null) throw new IllegalArgumentException("unknown operation: " + name); + try { + return operation.handler.call(params); + } finally { + synchronized (this) { + owner.active--; + } + } + } + + final class BindingsLease implements AutoCloseable { + final Map values = new LinkedHashMap<>(); + final List owners = new ArrayList<>(); + + public void close() { + synchronized (Extensions.this) { + for (Loaded item : owners) item.active--; + owners.clear(); + } + } + } + + synchronized BindingsLease bindings(ClassLoader application) throws Exception { + BindingsLease lease = new BindingsLease(); + try { + for (Loaded item : loaded.values()) { + if (item.failed) + throw new IllegalStateException("extension cleanup failed: " + item.extension.id()); + item.active++; + lease.owners.add(item); + lease.values.put(item.extension.id(), item.context.bindings(application)); + } + return lease; + } catch (Exception error) { + lease.close(); + throw error; + } + } + + public synchronized void close() throws Exception { + Exception failure = null; + for (Loaded item : loaded.values()) + try { + if (item.active != 0) + throw new IllegalStateException("extension is busy: " + item.extension.id()); + item.close(); + } catch (Exception error) { + if (failure == null) failure = error; + else failure.addSuppressed(error); + } + if (failure != null) throw failure; + loaded.clear(); + } + + private Loaded load(Path jar, String digest) throws Exception { + URLClassLoader loader = + new URLClassLoader( + new URL[] {jar.toUri().toURL()}, DiagnosticExtension.class.getClassLoader()) { + public Enumeration getResources(String name) throws IOException { + if (name.equals("META-INF/services/" + DiagnosticExtension.class.getName())) + return findResources(name); + return super.getResources(name); + } + + protected synchronized Class loadClass(String name, boolean resolve) + throws ClassNotFoundException { + if (name.startsWith("java.") + || name.startsWith("javax.") + || name.startsWith("com.flanksource.telltale.")) + return super.loadClass(name, resolve); + Class type = findLoadedClass(name); + if (type == null) + try { + type = findClass(name); + } catch (ClassNotFoundException absent) { + type = super.loadClass(name, false); + } + if (resolve) resolveClass(type); + return type; + } + }; + ExtensionContext context = new ExtensionContext(); + DiagnosticExtension extension = null; + try { + Iterator providers = + ServiceLoader.load(DiagnosticExtension.class, loader).iterator(); + if (!providers.hasNext()) + throw new IllegalArgumentException("JAR has no DiagnosticExtension provider"); + extension = providers.next(); + if (providers.hasNext()) + throw new IllegalArgumentException("JAR must contain exactly one extension"); + if (extension.getClass().getClassLoader() != loader) + throw new IllegalArgumentException("extension provider must be defined in its JAR"); + if (extension.apiVersion() != 1 + || !extension.id().matches("[a-z][a-z0-9-]*") + || extension.version().trim().isEmpty()) + throw new IllegalArgumentException("invalid extension API, identity or version"); + extension.start(context); + for (String name : context.operations.keySet()) + if (!name.startsWith("extension." + extension.id() + ".")) + throw new IllegalArgumentException("extension operation must use its namespace"); + return new Loaded(extension, loader, context, digest); + } catch (Exception | LinkageError | ServiceConfigurationError error) { + if (extension != null) + try { + extension.close(); + } catch (Exception cleanup) { + error.addSuppressed(cleanup); + } + try { + context.close(); + loader.close(); + } catch (Exception cleanup) { + error.addSuppressed(cleanup); + } + if (error instanceof ServiceConfigurationError || error instanceof LinkageError) + throw new IllegalArgumentException("load extension provider: " + error, error); + throw error; + } + } + + private void persist(Map planned) throws IOException { + Properties active = new Properties(); + for (Loaded item : planned.values()) active.setProperty(item.extension.id(), item.digest); + Path staged = Files.createTempFile(directory, "active-", ".properties"); + try { + try (OutputStream output = Files.newOutputStream(staged)) { + active.store(output, "Telltale active extensions"); + } + Files.move( + staged, + directory.resolve("active.properties"), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } finally { + Files.deleteIfExists(staged); + } + } + + private static String digest(byte[] bytes) throws Exception { + StringBuilder result = new StringBuilder(); + for (byte value : MessageDigest.getInstance("SHA-256").digest(bytes)) + result.append(String.format("%02x", value & 255)); + return result.toString(); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/GroovyEngine.java b/sdk/java/src/main/java/com/flanksource/telltale/GroovyEngine.java new file mode 100644 index 0000000..eb056d0 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/GroovyEngine.java @@ -0,0 +1,87 @@ +package com.flanksource.telltale; + +import groovy.lang.Binding; +import groovy.lang.GroovyShell; +import groovy.transform.ThreadInterrupt; +import groovy.transform.TimedInterrupt; +import java.io.*; +import java.util.*; +import org.codehaus.groovy.control.CompilerConfiguration; +import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer; + +public final class GroovyEngine { + public static Map evaluate( + Map params, ClassLoader loader, Object jmx, Map extensions) + throws Exception { + Object configured = params.getOrDefault("timeoutSeconds", 30); + if (!(configured instanceof Number)) + throw new IllegalArgumentException("timeoutSeconds must be an integer"); + long timeout; + try { + timeout = new java.math.BigDecimal(configured.toString()).longValueExact(); + } catch (ArithmeticException | NumberFormatException error) { + throw new IllegalArgumentException("timeoutSeconds must be an integer", error); + } + if (timeout <= 0 || timeout > 3600) + throw new IllegalArgumentException("timeoutSeconds must be between one and 3600"); + CompilerConfiguration compiler = new CompilerConfiguration(); + compiler.addCompilationCustomizers( + new ASTTransformationCustomizer( + Collections.singletonMap("value", timeout), TimedInterrupt.class)); + compiler.addCompilationCustomizers(new ASTTransformationCustomizer(ThreadInterrupt.class)); + BoundedWriter output = new BoundedWriter(); + BoundedWriter error = new BoundedWriter(); + Binding bindings = new Binding(); + bindings.setVariable("args", params.getOrDefault("args", Collections.emptyMap())); + bindings.setVariable("out", new PrintWriter(output)); + bindings.setVariable("err", new PrintWriter(error)); + bindings.setVariable("jmx", jmx); + bindings.setVariable("application", loader); + bindings.setVariable("extensions", extensions); + ClassLoader previous = Thread.currentThread().getContextClassLoader(); + GroovyShell shell = new GroovyShell(GroovyEngine.class.getClassLoader(), bindings, compiler); + long start = System.nanoTime(); + try { + Thread.currentThread().setContextClassLoader(loader); + Object value = shell.evaluate((String) params.get("source")); + Map result = new LinkedHashMap<>(); + if (value == null + || value instanceof String + || value instanceof Number + || value instanceof Boolean) result.put("value", value); + else + result.put( + "graph", + new com.flanksource.telltale.diag.graph.ValueGraph( + com.flanksource.telltale.diag.graph.GraphLimits.probe( + 4, 100, 1000, 4096, 1048576)) + .root(value, "result", null)); + result.put("stdout", output.toString()); + result.put("stderr", error.toString()); + result.put("elapsedMs", (System.nanoTime() - start) / 1000000); + return result; + } finally { + Thread.currentThread().setContextClassLoader(previous); + shell.getClassLoader().clearCache(); + shell.getClassLoader().close(); + } + } + + private static final class BoundedWriter extends Writer { + private final StringBuilder text = new StringBuilder(); + + public void write(char[] chars, int offset, int length) { + if (length > 1048576 - text.length()) + throw new IllegalStateException("script output exceeds 1 MiB"); + text.append(chars, offset, length); + } + + public void flush() {} + + public void close() {} + + public String toString() { + return text.toString(); + } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/HttpListener.java b/sdk/java/src/main/java/com/flanksource/telltale/HttpListener.java new file mode 100644 index 0000000..7365acd --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/HttpListener.java @@ -0,0 +1,239 @@ +package com.flanksource.telltale; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.util.*; +import java.util.concurrent.*; + +/** The attached agent's HTTP adapter: the WAR's routes for JVMs that have no servlet container. */ +final class HttpListener implements AutoCloseable { + private static final int MAX_REQUEST = 24 * 1024 * 1024; + private static final Set KEYS = + new HashSet<>( + Arrays.asList( + "bind", "port", "portFile", "issuer", "audience", "jwks", "jwksUrl", "configDigest")); + + private final String owner = UUID.randomUUID().toString(); + private final String configDigest; + private final int runtimeApiVersion; + private final ThreadPoolExecutor executor = + new ThreadPoolExecutor( + 4, + 4, + 0, + TimeUnit.SECONDS, + new ArrayBlockingQueue(64), + runnable -> { + Thread thread = new Thread(runnable, "telltale-http"); + thread.setDaemon(true); + return thread; + }, + new ThreadPoolExecutor.CallerRunsPolicy()); + private final HttpServer server; + private boolean registered; + private boolean started; + + private HttpListener(Map config) throws Exception { + for (String key : config.keySet()) + if (!KEYS.contains(key)) throw new IllegalArgumentException("unknown http option " + key); + configDigest = Diagnostics.text(config, "configDigest", ""); + Path portFile = portFile(config); + runtimeApiVersion = RuntimeClient.apiVersion(); + server = HttpServer.create(address(config), 16); + try { + if (runtimeApiVersion == RuntimeService.API_VERSION) register(config); + server.setExecutor(executor); + server.createContext("/", this::exchange); + startDaemon(); + if (portFile != null) writePort(portFile, server.getAddress().getPort()); + } catch (Exception error) { + try { + close(); + } catch (Exception cleanup) { + error.addSuppressed(cleanup); + } + throw error; + } + } + + static HttpListener start(Map config) throws Exception { + return new HttpListener(config); + } + + static void validate(Map config) throws Exception { + for (String key : config.keySet()) + if (!KEYS.contains(key)) throw new IllegalArgumentException("unknown http option " + key); + address(config); + portFile(config); + new com.flanksource.telltale.auth.JwtVerifier( + Diagnostics.required(config, "issuer"), Diagnostics.required(config, "audience"), + config.containsKey("jwks") ? Agent.JSON.toJson(config.get("jwks")) : null, + Diagnostics.text(config, "jwksUrl", null)); + } + + int port() { + return server.getAddress().getPort(); + } + + private void register(Map config) throws Exception { + Map request = new LinkedHashMap<>(); + request.put("owner", owner); + request.put("issuer", config.get("issuer")); + request.put("audience", config.get("audience")); + request.put("jwks", config.get("jwks")); + request.put("jwksUrl", config.get("jwksUrl")); + RuntimeClient.request("registerAdapter", request); + registered = true; + } + + private static InetSocketAddress address(Map config) throws IOException { + String bind = Diagnostics.required(config, "bind"); + Object port = config.get("port"); + if (!(port instanceof Number)) throw new IllegalArgumentException("http port is required"); + double value = ((Number) port).doubleValue(); + if (value != Math.rint(value) || value < 0 || value > 65535) + throw new IllegalArgumentException("http port must be an integer between 0 and 65535"); + return new InetSocketAddress(InetAddress.getByName(bind), (int) value); + } + + /** The dispatcher thread inherits daemon status, so it must not keep the target JVM alive. */ + private void startDaemon() throws Exception { + FutureTask start = new FutureTask<>(server::start, null); + Thread starter = new Thread(start, "telltale-http-start"); + starter.setDaemon(true); + starter.start(); + try { + start.get(); + } catch (ExecutionException error) { + throw new IllegalStateException("start diagnostic HTTP listener", error.getCause()); + } + started = true; + } + + private static Path portFile(Map config) { + String file = Diagnostics.text(config, "portFile", null); + if (file == null) return null; + Path path = Paths.get(file); + if (!path.isAbsolute()) throw new IllegalArgumentException("http portFile must be absolute"); + return path; + } + + private static void writePort(Path path, int port) throws IOException { + Files.createDirectories(path.getParent()); + Path staged = Files.createTempFile(path.getParent(), "port-", ".stage"); + Files.write(staged, Integer.toString(port).getBytes(StandardCharsets.UTF_8)); + Files.move(staged, path, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } + + private void exchange(HttpExchange exchange) throws IOException { + try { + respond(exchange, reply(exchange)); + } catch (IllegalArgumentException error) { + respond(exchange, RuntimeHttp.error(400, error.getMessage())); + } catch (Exception error) { + respond(exchange, RuntimeHttp.error(500, error.toString())); + } finally { + exchange.close(); + } + } + + private Map reply(HttpExchange exchange) throws Exception { + String method = exchange.getRequestMethod(), path = exchange.getRequestURI().getPath(); + boolean health = method.equals("GET") && path.equals("/health"); + if (runtimeApiVersion != RuntimeService.API_VERSION) + return health ? outdatedHealth() : RuntimeHttp.error(503, outdatedMessage()); + if (health) return health(); + Map envelope = new LinkedHashMap<>(); + envelope.put("method", method); + envelope.put("path", path); + envelope.put("owner", owner); + envelope.put("body", new String(read(exchange.getRequestBody()), StandardCharsets.UTF_8)); + envelope.put("authorization", exchange.getRequestHeaders().getFirst("Authorization")); + envelope.put( + "telltaleAuthorization", + exchange.getRequestHeaders().getFirst(com.flanksource.telltale.auth.JwtVerifier.HEADER)); + @SuppressWarnings("unchecked") + Map reply = (Map) RuntimeClient.request("http", envelope); + return reply; + } + + private Map health() throws Exception { + Map body = new LinkedHashMap<>(); + body.put("status", "ok"); + body.put("apiVersion", RuntimeService.API_VERSION); + body.put("configDigest", configDigest); + Map runtime = (Map) RuntimeClient.request("describe", Collections.emptyMap()); + for (String key : new String[] {"upgradeApiVersion", "runtimeVersion", "artifactSHA256"}) + body.put(key, runtime.get(key)); + return jsonReply(200, body); + } + + private Map outdatedHealth() { + Map body = new LinkedHashMap<>(); + body.put("status", "outdated"); + body.put("apiVersion", runtimeApiVersion); + body.put("required", RuntimeService.API_VERSION); + body.put("error", outdatedMessage()); + return jsonReply(503, body); + } + + private String outdatedMessage() { + return "diagnostic runtime API " + + runtimeApiVersion + + " is loaded in this JVM; this adapter requires " + + RuntimeService.API_VERSION + + "; restart the JVM to replace it"; + } + + private static Map jsonReply(int status, Object body) { + Map reply = new LinkedHashMap<>(); + reply.put("status", status); + reply.put("contentType", RuntimeHttp.JSON_TYPE); + reply.put("body", Agent.JSON.toJson(body).getBytes(StandardCharsets.UTF_8)); + reply.put("headers", Collections.emptyMap()); + return reply; + } + + private static void respond(HttpExchange exchange, Map reply) throws IOException { + byte[] body = (byte[]) reply.get("body"); + String contentType = (String) reply.get("contentType"); + if (contentType != null && !contentType.isEmpty()) + exchange.getResponseHeaders().set("Content-Type", contentType); + for (Map.Entry header : ((Map) reply.get("headers")).entrySet()) + exchange.getResponseHeaders().set((String) header.getKey(), (String) header.getValue()); + exchange.sendResponseHeaders((Integer) reply.get("status"), body.length == 0 ? -1 : body.length); + if (body.length > 0) + try (OutputStream output = exchange.getResponseBody()) { + output.write(body); + } + } + + private static byte[] read(InputStream input) throws IOException { + ByteArrayOutputStream result = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int count; + while ((count = input.read(buffer)) != -1) { + if (count > MAX_REQUEST - result.size()) + throw new IllegalArgumentException("request exceeds 24 MiB size limit"); + result.write(buffer, 0, count); + } + return result.toByteArray(); + } + + public void close() throws Exception { + if (started) server.stop(0); + executor.shutdownNow(); + if (!executor.awaitTermination(5, TimeUnit.SECONDS)) + throw new IllegalStateException("HTTP workers did not stop for upgrade"); + if (registered) + RuntimeClient.request("closeOwner", Collections.singletonMap("owner", owner)); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/Hub.java b/sdk/java/src/main/java/com/flanksource/telltale/Hub.java index c350c37..5d1a96a 100644 --- a/sdk/java/src/main/java/com/flanksource/telltale/Hub.java +++ b/sdk/java/src/main/java/com/flanksource/telltale/Hub.java @@ -40,12 +40,13 @@ public void onError(Exception error) { failure = error; } - static void start(Agent agent, String address) { + static Thread start(Agent agent, String address) { Thread thread = new Thread( () -> { int delay = 1; - while (true) { + while (!agent.closed) { + Session session = null; try { URI base = URI.create(address); String scheme = @@ -62,16 +63,20 @@ static void start(Agent agent, String address) { null, null); Hub hub = new Hub(uri, agent.config.get("token").getAsString()); + agent.hub = hub; if (!hub.connectBlocking(10, TimeUnit.SECONDS)) throw new IllegalStateException("Hub connection timed out"); - try (Session session = new Session(agent, hub.wire(), true)) { - session.run(); + try (Session active = new Session(agent, hub.wire(), true)) { + session = active; + active.run(); } finally { hub.close(); } } catch (Exception error) { - System.err.println("telltale hub: " + error.getClass().getSimpleName()); + if (!agent.closed) System.err.println("telltale hub: " + error.getClass().getSimpleName()); + else if (session != null && !session.stopped()) agent.stopFailure.compareAndSet(null, error); } + if (agent.closed) return; try { Thread.sleep(delay * 1000L); } catch (InterruptedException stop) { @@ -84,6 +89,7 @@ static void start(Agent agent, String address) { "telltale-hub"); thread.setDaemon(true); thread.start(); + return thread; } private Wire wire() { diff --git a/sdk/java/src/main/java/com/flanksource/telltale/MethodCapture.java b/sdk/java/src/main/java/com/flanksource/telltale/MethodCapture.java new file mode 100644 index 0000000..6a9889a --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/MethodCapture.java @@ -0,0 +1,192 @@ +package com.flanksource.telltale; + +import com.flanksource.telltale.diag.hotpatch.*; +import java.nio.charset.StandardCharsets; +import java.util.*; + +final class MethodCapture implements Capture { + private final ClassOperations classes; + private final String key; + private final List> targets = new ArrayList<>(); + private final boolean redefine; + private long cursor; + private boolean closed; + private boolean complete; + private boolean more; + private final List> results = new ArrayList<>(); + private final boolean filtered; + private long filteredCalls; + private long omittedEvents; + + @SuppressWarnings("unchecked") + MethodCapture(ClassOperations classes, String kind, Map params, String key) + throws Exception { + this.classes = classes; + this.key = key; + this.redefine = kind.equals("jvm_redefine"); + this.filtered = !redefine && ClassOperations.filterOptions(params) != null; + Object selected = redefine ? Collections.singletonList(params) : params.get("targets"); + if (!(selected instanceof List) + || ((List) selected).isEmpty() + || ((List) selected).size() > 8) + throw new IllegalArgumentException("targets must contain between one and eight methods"); + try { + synchronized (classes) { + for (Object raw : (List) selected) { + if (!(raw instanceof Map)) + throw new IllegalArgumentException("each target must be an object"); + Map target = new LinkedHashMap<>((Map) raw); + target.put("leaseSeconds", 120); + if (redefine) { + target.put("source", Diagnostics.required(params, "source")); + Map patched = classes.patch(new JsonBody(target)); + if (!Boolean.TRUE.equals(patched.get("applied"))) + throw new IllegalArgumentException("compilation failed: " + patched); + target.put("loader", patched.get("loader")); + Map result = new LinkedHashMap<>(patched); + result.put("class", target.get("class")); + result.put("applied", true); + results.add(result); + } else { + Map options = new LinkedHashMap<>(params); + options.remove("targets"); + options.remove("target"); + options.put("key", key); + options.put("join", !targets.isEmpty()); + target.put("probe", options); + Map armed = classes.instrument(new JsonBody(target)); + target.put("loader", armed.get("loader")); + } + targets.add(target); + } + } + } catch (Exception error) { + try { + close(); + } catch (Exception cleanup) { + error.addSuppressed(cleanup); + } + throw error; + } + } + + public synchronized byte[] dump() throws Exception { + if (closed) throw new IllegalStateException("capture is closed"); + if (redefine) return patchResults(false); + Map batch = AgentBridge.drain(key, cursor, 10000); + if (batch.containsKey("error")) + throw new IllegalStateException(String.valueOf(batch.get("error"))); + if (Boolean.TRUE.equals(batch.get("closed"))) + throw new IllegalStateException("probe was closed before capture finished: " + key); + List lines = (List) batch.get("events"); + List events = new ArrayList<>(); + long lost = ((Number) batch.get("dropped")).longValue(); + if (lost > 0) { + Map loss = new LinkedHashMap<>(); + loss.put("kind", "dropped"); + loss.put("key", key); + loss.put("ts", System.currentTimeMillis() * 1000000L); + loss.put("dropped", lost); + events.add(loss); + } + for (Object line : lines) { + com.google.gson.JsonObject event = + Agent.JSON.fromJson((String) line, com.google.gson.JsonObject.class); + event.add("probeSeq", event.remove("seq")); + events.add(event); + } + filteredCalls = ((Number) batch.get("filtered")).longValue(); + omittedEvents = ((Number) batch.get("omitted")).longValue(); + long next = ((Number) batch.get("nextSeq")).longValue(); + more = next < ((Number) batch.get("writeSeq")).longValue(); + complete = + Boolean.TRUE.equals(batch.get("complete")) + && next >= ((Number) batch.get("writeSeq")).longValue(); + byte[] result = Agent.JSON.toJson(events).getBytes(StandardCharsets.UTF_8); + cursor = next; + return result; + } + + public synchronized void renew() throws Exception { + if (closed) return; + synchronized (classes) { + for (Map target : targets) { + Map renewal = new LinkedHashMap<>(target); + renewal.put("seconds", 120); + if (!Boolean.TRUE.equals(classes.lease(new JsonBody(renewal)).get("patched"))) + throw new IllegalStateException("capture lost its patch on " + target.get("class")); + } + } + } + + public synchronized void close() throws Exception { + if (closed) return; + Exception failure = null; + synchronized (classes) { + ListIterator> iterator = targets.listIterator(targets.size()); + while (iterator.hasPrevious()) { + try { + classes.restore(new JsonBody(iterator.previous())); + iterator.remove(); + } catch (Exception error) { + if (failure == null) failure = error; + else failure.addSuppressed(error); + } + } + } + closed = targets.isEmpty(); + if (failure != null) throw failure; + } + + /** What the root-call filter withheld, as of the last read; null for an unfiltered capture. */ + public synchronized Map report() { + if (!filtered) return null; + Map report = new LinkedHashMap<>(); + report.put("filtered", filteredCalls); + report.put("omitted", omittedEvents); + return report; + } + + public String contentType() { + return "application/json"; + } + + public byte[] finish() throws Exception { + if (!redefine) { + freeze(); + com.google.gson.JsonArray events = new com.google.gson.JsonArray(); + do { + events.addAll( + Agent.JSON.fromJson( + new String(dump(), StandardCharsets.UTF_8), com.google.gson.JsonArray.class)); + } while (more); + return Agent.JSON.toJson(events).getBytes(StandardCharsets.UTF_8); + } + close(); + return patchResults(true); + } + + private byte[] patchResults(boolean restored) { + for (Map result : results) { + result.put("restored", restored); + result.put("timestamp", java.time.Instant.now().toString()); + } + return Agent.JSON.toJson(results).getBytes(StandardCharsets.UTF_8); + } + + public void freeze() { + if (!redefine) AgentBridge.freezeProbe(key); + } + + public String fileName() { + return "events.json"; + } + + public boolean streaming() { + return !redefine; + } + + public synchronized boolean complete() { + return complete; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/NotificationCapture.java b/sdk/java/src/main/java/com/flanksource/telltale/NotificationCapture.java new file mode 100644 index 0000000..a3d9cad --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/NotificationCapture.java @@ -0,0 +1,72 @@ +package com.flanksource.telltale; + +import com.flanksource.telltale.diag.jmx.NotificationHub; +import java.nio.charset.StandardCharsets; +import java.util.*; + +final class NotificationCapture implements Capture { + private final NotificationHub hub; + private final String id; + private long cursor; + private boolean closed; + + @SuppressWarnings("unchecked") + NotificationCapture(NotificationHub hub, Map params) throws Exception { + this.hub = hub; + Object filters = params.getOrDefault("types", Collections.emptyList()); + if (!(filters instanceof List)) + throw new IllegalArgumentException("types must be an array of strings"); + for (Object filter : (List) filters) + if (!(filter instanceof String)) + throw new IllegalArgumentException("types must be an array of strings"); + double capacity = ((Number) params.getOrDefault("capacity", 500)).doubleValue(); + if (capacity != Math.rint(capacity)) + throw new IllegalArgumentException("capacity must be an integer"); + id = + hub.register( + Diagnostics.required(params, "mbean"), + (List) filters, + NotificationHub.checkCapacity((long) capacity)); + } + + @SuppressWarnings("unchecked") + public synchronized byte[] dump() { + if (closed) throw new IllegalStateException("notification capture is closed"); + Map batch = hub.read(id, cursor, NotificationHub.MAX_CAPACITY); + List> events = + new ArrayList<>((List>) batch.get("events")); + long lost = ((Number) batch.get("dropped")).longValue(); + if (lost > 0) { + Map loss = new LinkedHashMap<>(); + loss.put("timeStamp", System.currentTimeMillis()); + loss.put("type", "telltale.dropped"); + loss.put("dropped", lost); + events.add(0, loss); + } + cursor = ((Number) batch.get("nextSeq")).longValue(); + return Agent.JSON.toJson(events).getBytes(StandardCharsets.UTF_8); + } + + public void freeze() { + hub.freeze(id); + } + + public synchronized void close() { + if (!closed) { + hub.deregister(id); + closed = true; + } + } + + public boolean streaming() { + return true; + } + + public String contentType() { + return "application/json"; + } + + public String fileName() { + return "notifications.json"; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/ObserverCapture.java b/sdk/java/src/main/java/com/flanksource/telltale/ObserverCapture.java new file mode 100644 index 0000000..a2a267b --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/ObserverCapture.java @@ -0,0 +1,84 @@ +package com.flanksource.telltale; + +import com.flanksource.telltale.observer.ObserverRuntime; +import com.flanksource.telltale.observer.ObserverOptions; +import com.flanksource.telltale.observer.profiles.ProfileEncoder; +import java.nio.charset.StandardCharsets; +import java.util.*; + +final class ObserverCapture implements Capture { + private final Object subscription; + private final String kind, id, source; + private final boolean top; + private final int limit; + private final long interval; + private final Map resource = new LinkedHashMap<>(); + private long sequence; + private boolean frozen; + + ObserverCapture(String kind, Map params, String id, Map configuredResource) throws Exception { + this.kind = kind; this.id = id; + top = kind.endsWith("_top"); + limit = ObserverOptions.positiveInt(params, "limit", 50, 10000); + interval = ObserverOptions.positiveInt(params, "pollIntervalMs", 5000, 3600000); + resource.putAll(configuredResource); + java.lang.instrument.Instrumentation instrumentation = com.flanksource.telltale.diag.hotpatch.InstrumentationService.instrumentation(); + Class owner = Class.forName("com.flanksource.telltale.observer.ObserverRuntime", true, RuntimeOwner.loader()); + owner.getMethod("install", java.lang.instrument.Instrumentation.class).invoke(null, instrumentation); + source = (String) owner.getField("SOURCE").get(null); + resource.put("telltale.source.instance", source); + subscription = owner.getMethod("subscribe", String.class, String.class, Map.class).invoke(null, kind, id, params); + } + + public synchronized byte[] dump() throws Exception { + if (!top) return ObserverRuntime.json(call("drain")).getBytes(StandardCharsets.UTF_8); + long begin = ObserverRuntime.epochNanos(); + List> rows = new ArrayList<>((List>) call("snapshot")); + int observed = rows.size(); + rows.sort((first, second) -> { + if (kind.equals("jms_top")) { + int count = Integer.compare(((Number)((Map)second.get("attributes")).get("activeCount")).intValue(), ((Number)((Map)first.get("attributes")).get("activeCount")).intValue()); + if (count != 0) return count; + } + return Long.compare(((Number) second.get("elapsedNs")).longValue(), ((Number) first.get("elapsedNs")).longValue()); + }); + if (rows.size() > limit) rows = new ArrayList<>(rows.subList(0, limit)); + Map snapshot = new LinkedHashMap<>(); + snapshot.put("kind", kind); snapshot.put("captureId", id); + snapshot.put("source", source); snapshot.put("profileId", UUID.randomUUID().toString()); + snapshot.put("sequence", ++sequence); snapshot.put("startedAtNanos", begin); + snapshot.put("completedAtNanos", ObserverRuntime.epochNanos()); + snapshot.put("observed", observed); snapshot.put("limit", limit); snapshot.put("dropped", dropped()); + snapshot.put("complete", true); snapshot.put("pollIntervalMs", interval); + snapshot.put("resource", resource); snapshot.put("rows", rows); + return ProfileEncoder.encode(snapshot); + } + + public String contentType() { return top ? ProfileEncoder.CONTENT_TYPE : "application/json"; } + public String fileName() { return top ? "snapshot.pb" : "records.json"; } + public boolean streaming() { return true; } + public long intervalMillis() { return top ? interval : 250; } + public long dropped() { return ((Number) call("dropped")).longValue(); } + public void renew() { call("renew"); } + public boolean complete() { return Boolean.TRUE.equals(call("expired")); } + public Map report() { return (Map) call("report"); } + public synchronized void freeze() throws Exception { + if (frozen) return; + call("close"); + frozen = true; + } + public synchronized byte[] finish() throws Exception { + if (top) { byte[] last = dump(); freeze(); return last; } + freeze(); return dump(); + } + public void close() throws Exception { freeze(); } + + private Object call(String method) { + try { return subscription.getClass().getMethod(method).invoke(subscription); } + catch (java.lang.reflect.InvocationTargetException error) { + Throwable cause = error.getCause(); + if (cause instanceof RuntimeException) throw (RuntimeException) cause; + throw new IllegalStateException("observer capture " + method, cause); + } catch (ReflectiveOperationException error) { throw new IllegalStateException("observer capture " + method, error); } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/OgnlEvaluation.java b/sdk/java/src/main/java/com/flanksource/telltale/OgnlEvaluation.java new file mode 100644 index 0000000..4953689 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/OgnlEvaluation.java @@ -0,0 +1,57 @@ +package com.flanksource.telltale; + +import java.lang.reflect.*; +import java.util.Map; +import ognl.*; + +final class OgnlEvaluation { + static Object evaluate(Map params, ClassLoader loader) throws Exception { + ClassResolver resolver = + new ClassResolver() { + public Class classForName(String name, Map context) throws ClassNotFoundException { + return Class.forName(name, true, loader); + } + }; + MemberAccess access = + new MemberAccess() { + public boolean isAccessible(Map context, Object target, Member member, String property) { + return true; + } + + public Object setup(Map context, Object target, Member member, String property) { + if (member instanceof AccessibleObject && !((AccessibleObject) member).isAccessible()) { + ((AccessibleObject) member).setAccessible(true); + return Boolean.FALSE; + } + return null; + } + + public void restore( + Map context, Object target, Member member, String property, Object state) { + if (state != null) ((AccessibleObject) member).setAccessible((Boolean) state); + } + }; + OgnlContext context = new OgnlContext(resolver, null, access); + ClassLoader previous = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(loader); + Object value = + Ognl.getValue( + Ognl.parseExpression(Diagnostics.required(params, "expression")), + context, + context.getRoot()); + if (value == null + || value instanceof String + || value instanceof Number + || value instanceof Boolean) return java.util.Collections.singletonMap("value", value); + return java.util.Collections.singletonMap( + "graph", + new com.flanksource.telltale.diag.graph.ValueGraph( + com.flanksource.telltale.diag.graph.GraphLimits.probe( + 4, 100, 1000, 4096, 1048576)) + .root(value, "result", null)); + } finally { + Thread.currentThread().setContextClassLoader(previous); + } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/OperationSpec.java b/sdk/java/src/main/java/com/flanksource/telltale/OperationSpec.java new file mode 100644 index 0000000..c4a9b9f --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/OperationSpec.java @@ -0,0 +1,78 @@ +package com.flanksource.telltale; + +import java.util.*; + +/** JDK-only operation definition shared by extensions and all transports. */ +public final class OperationSpec { + public interface Handler { + Object call(Map parameters) throws Exception; + } + + final String name; + String title; + String description = ""; + String command; + Map parameters = Collections.singletonMap("type", "object"); + Map result = Collections.emptyMap(); + boolean mutates; + Handler handler; + + private OperationSpec(String name) { + if (name == null || name.trim().isEmpty()) + throw new IllegalArgumentException("operation name is required"); + this.name = name; + this.title = name; + } + + public static OperationSpec named(String name) { + return new OperationSpec(name); + } + + public OperationSpec title(String value) { + title = Objects.requireNonNull(value); + return this; + } + + public OperationSpec description(String value) { + description = Objects.requireNonNull(value); + return this; + } + + /** The space-separated CLI command, and API path, a built-in operation is served as. */ + OperationSpec command(String value) { + command = Objects.requireNonNull(value); + return this; + } + + public OperationSpec parameters(Map schema) { + parameters = new LinkedHashMap<>(schema); + return this; + } + + public OperationSpec result(Map schema) { + result = new LinkedHashMap<>(schema); + return this; + } + + public OperationSpec mutates(boolean value) { + mutates = value; + return this; + } + + public OperationSpec handle(Handler value) { + handler = Objects.requireNonNull(value); + return this; + } + + Map descriptor() { + Map value = new LinkedHashMap<>(); + value.put("name", name); + value.put("title", title); + value.put("description", description); + if (command != null) value.put("command", command); + value.put("paramsSchema", parameters); + value.put("resultSchema", result); + value.put("mutates", mutates); + return value; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/Recording.java b/sdk/java/src/main/java/com/flanksource/telltale/Recording.java index eb3b24e..39d86b0 100644 --- a/sdk/java/src/main/java/com/flanksource/telltale/Recording.java +++ b/sdk/java/src/main/java/com/flanksource/telltale/Recording.java @@ -5,7 +5,7 @@ import java.nio.file.Path; import java.util.concurrent.atomic.AtomicBoolean; -final class Recording implements AutoCloseable { +final class Recording implements Capture { private static final AtomicBoolean ASYNC = new AtomicBoolean(); private final Object recorder; private final Class type; @@ -54,6 +54,20 @@ final class Recording implements AutoCloseable { .invoke(null, value(params, "settings", "profile")); type = Class.forName("jdk.jfr.Recording"); recorder = type.getConstructor(configType).newInstance(config); + if (params.has("supportSnapshot") && params.get("supportSnapshot").getAsBoolean()) { + java.util.Map settings = new java.util.HashMap<>(); + java.util.Map original = (java.util.Map) type.getMethod("getSettings").invoke(recorder); + java.util.Set allowed = new java.util.HashSet<>(java.util.Arrays.asList( + "jdk.ExecutionSample", "jdk.NativeMethodSample", "jdk.ObjectAllocationSample", + "jdk.ObjectAllocationInNewTLAB", "jdk.ObjectAllocationOutsideTLAB", + "jdk.JavaMonitorEnter", "jdk.ThreadPark", "jdk.GarbageCollection", "jdk.GCHeapSummary")); + for (java.util.Map.Entry entry : original.entrySet()) { + String key = entry.getKey().toString(); + settings.put(key, key.endsWith("#enabled") && !allowed.contains(key.split("#")[0]) ? "false" : entry.getValue().toString()); + } + type.getMethod("setSettings", java.util.Map.class).invoke(recorder, settings); + type.getMethod("setMaxSize", long.class).invoke(recorder, 60L * 1024 * 1024); + } type.getMethod("start").invoke(recorder); } } catch (Exception error) { @@ -73,7 +87,7 @@ private String execute(String command) throws Exception { return (String) type.getMethod("execute", String.class).invoke(recorder, command); } - synchronized byte[] dump() throws Exception { + public synchronized byte[] dump() throws Exception { if (closed) throw new IllegalStateException("Recording is closed"); if (async) execute("dump,jfr,file=" + file.toString()); else type.getMethod("dump", Path.class).invoke(recorder, file); @@ -83,6 +97,14 @@ synchronized byte[] dump() throws Exception { return Files.readAllBytes(file); } + public String contentType() { + return "application/x-jfr"; + } + + public String fileName() { + return "recording.jfr"; + } + public synchronized void close() throws Exception { if (closed) return; try { diff --git a/sdk/java/src/main/java/com/flanksource/telltale/RuntimeClient.java b/sdk/java/src/main/java/com/flanksource/telltale/RuntimeClient.java new file mode 100644 index 0000000..d854ff8 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/RuntimeClient.java @@ -0,0 +1,124 @@ +package com.flanksource.telltale; + +import java.lang.management.ManagementFactory; +import java.util.*; +import javax.management.*; + +final class RuntimeClient { + private final Map resource; + RuntimeClient(Map config) throws Exception { + resource = AgentResource.from(config); + RuntimeService.acquire(config); + } + + static Object request(String action, Map request) throws Exception { + try { + return ManagementFactory.getPlatformMBeanServer() + .invoke( + new ObjectName(RuntimeService.NAME), + action, + new Object[] {request}, + new String[] {Map.class.getName()}); + } catch (MBeanException failure) { + throw failure.getTargetException(); + } + } + + /** The registered runtime's API version; a runtime without {@code describe} predates it (0). */ + static int apiVersion() throws Exception { + for (MBeanOperationInfo operation : + ManagementFactory.getPlatformMBeanServer() + .getMBeanInfo(new ObjectName(RuntimeService.NAME)) + .getOperations()) + if (operation.getName().equals("describe")) { + Object value = ((Map) request("describe", Collections.emptyMap())).get("apiVersion"); + if (!(value instanceof Number)) + throw new IllegalStateException("diagnostic runtime describe returned no apiVersion"); + return ((Number) value).intValue(); + } + return 0; + } + + @SuppressWarnings("unchecked") + List> catalog() throws Exception { + return (List>) request("catalog", Collections.emptyMap()); + } + + Object call(String operation, Map params, boolean allow) throws Exception { + Map request = new LinkedHashMap<>(); + request.put("operation", operation); + request.put("params", params); + request.put("allowMutation", allow); + return request("call", request); + } + + @SuppressWarnings("unchecked") + Capture capture(String kind, Map params, String id, boolean allow) + throws Exception { + Map request = new LinkedHashMap<>(); + request.put("kind", kind); + request.put("params", params); + request.put("id", id); + request.put("allowMutation", allow); + request.put("resource", resource); + Map description = (Map) request("captureOpen", request); + return new Capture() { + private boolean closed; + + public byte[] dump() throws Exception { + return (byte[]) request("captureDump", Collections.singletonMap("id", id)); + } + + public byte[] finish() throws Exception { + return (byte[]) request("captureFinish", Collections.singletonMap("id", id)); + } + + public void renew() throws Exception { + request("captureRenew", Collections.singletonMap("id", id)); + } + + public void close() throws Exception { + if (!closed) { + request("captureClose", Collections.singletonMap("id", id)); + closed = true; + } + } + + public boolean streaming() { + return Boolean.TRUE.equals(description.get("streaming")); + } + + public long intervalMillis() { return ((Number) description.get("intervalMillis")).longValue(); } + public long dropped() { + try { return ((Number) request("captureDropped", Collections.singletonMap("id", id))).longValue(); } + catch (Exception error) { throw new IllegalStateException("read capture drops", error); } + } + + public boolean complete() { + try { + return Boolean.TRUE.equals( + request("captureComplete", Collections.singletonMap("id", id))); + } catch (Exception error) { + throw new IllegalStateException("read capture completion", error); + } + } + + @SuppressWarnings("unchecked") + public Map report() { + try { + return (Map) request("captureReport", Collections.singletonMap("id", id)); + } catch (Exception error) { + throw new IllegalStateException("read capture report", error); + } + } + + public String contentType() { + return (String) description.get("contentType"); + } + + public String fileName() { + return (String) description.get("fileName"); + } + }; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/RuntimeHttp.java b/sdk/java/src/main/java/com/flanksource/telltale/RuntimeHttp.java new file mode 100644 index 0000000..8c992a9 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/RuntimeHttp.java @@ -0,0 +1,201 @@ +package com.flanksource.telltale; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.flanksource.telltale.auth.AuthException; +import com.flanksource.telltale.auth.JwtVerifier; +import com.flanksource.telltale.auth.Principal; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.*; + +/** + * Authenticates and routes adapter HTTP requests (WAR servlet and agent listener). Every reply is + * a map of status, contentType, body bytes and headers; failures never escape as exceptions. + */ +final class RuntimeHttp { + static final String JSON_TYPE = "application/json; charset=UTF-8"; + static final String COMPLETE_HEADER = "X-Telltale-Complete"; + + /** What the capture withheld (e.g. jvm_trace {filtered, omitted}), as JSON, when it says. */ + static final String REPORT_HEADER = "X-Telltale-Report"; + private static final int MAX_ARTIFACT = 64 * 1024 * 1024; + private static final String CAPTURES = "/captures/"; + + /** A request failure with the HTTP status it maps to. */ + static final class Failure extends Exception { + final int status; + + Failure(int status, String message) { + super(message); + this.status = status; + } + } + + private final RuntimeService runtime; + + RuntimeHttp(RuntimeService runtime) { + this.runtime = runtime; + } + + Map handle(Map envelope) { + try { + return route(envelope); + } catch (AuthException error) { + return error(error.status(), error.getMessage()); + } catch (Failure error) { + return error(error.status, error.getMessage()); + } catch (IllegalArgumentException | JsonProcessingException error) { + return error(400, error.getMessage()); + } catch (Exception error) { + return error(500, error.toString()); + } + } + + private Map route(Map envelope) throws Exception { + String method = Diagnostics.required(envelope, "method"); + String path = Diagnostics.required(envelope, "path"); + String owner = Diagnostics.text(envelope, "owner", null); + JwtVerifier verifier = runtime.verifier(owner); + if (verifier == null) throw new Failure(401, "adapter not registered"); + Principal principal = + verifier.verify( + JwtVerifier.credential( + Diagnostics.text(envelope, "telltaleAuthorization", null), + Diagnostics.text(envelope, "authorization", null))); + if (method.equals("GET") && path.equals("/hello")) { + principal.require(Scope.READ); + return json(200, hello()); + } + if (method.equals("GET") && path.equals("/catalog")) { + principal.require(Scope.READ); + return json(200, runtime.diagnostics.catalog()); + } + Map body = body(envelope); + if (method.equals("POST") && path.equals("/call")) return call(principal, body); + if (method.equals("POST") && path.equals("/captures")) return open(principal, body, owner); + if (path.startsWith(CAPTURES)) return capture(principal, method, path, owner); + throw new Failure(404, "unknown diagnostic route " + method + " " + path); + } + + private Map call(Principal principal, Map body) throws Exception { + String operation = Diagnostics.required(body, "operation"); + principal.require(runtime.diagnostics.scope(operation)); + return json( + 200, + runtime.diagnostics.call( + operation, params(body), Boolean.TRUE.equals(body.get("allowMutation")))); + } + + private Map open(Principal principal, Map body, String owner) + throws Exception { + principal.require(Scope.forCapture(Diagnostics.required(body, "kind"))); + body.put("params", params(body)); + body.put("id", UUID.randomUUID().toString()); + body.put("owner", owner); + body.put("resource", runtime.resource); + return json(200, runtime.openCapture(body)); + } + + private Map capture( + Principal principal, String method, String path, String owner) throws Exception { + String id = path.substring(CAPTURES.length()); + boolean renew = id.endsWith("/renew"); + if (renew) id = id.substring(0, id.length() - "/renew".length()); + if (id.isEmpty() || id.contains("/")) throw new Failure(404, "unknown diagnostic route " + path); + Capture capture = runtime.owned(id, owner); + principal.require(runtime.captureScope(id)); + if (renew && method.equals("POST")) { + runtime.captureAction("captureRenew", id); + return reply(204, "", new byte[0]); + } + if (!renew && method.equals("POST")) { + Map reply = artifact(capture, (byte[]) runtime.captureAction("captureDump", id)); + headers(reply).put(COMPLETE_HEADER, Boolean.toString(capture.complete())); + putReport(reply, capture); + return reply; + } + if (!renew && method.equals("DELETE")) { + try { + Map reply = + artifact(capture, (byte[]) runtime.captureAction("captureFinish", id)); + putReport(reply, capture); + return reply; + } finally { + runtime.captureAction("captureClose", id); + } + } + throw new Failure(404, "unknown diagnostic route " + method + " " + path); + } + + private Map hello() throws Exception { + Map hello = new LinkedHashMap<>(); + hello.put("apiVersion", RuntimeService.API_VERSION); + hello.put("protocol", 1); + hello.put("sdkVersion", RuntimeService.class.getPackage().getImplementationVersion()); + hello.put("started", runtime.started); + Map platform = new LinkedHashMap<>(); + platform.put("lang", "java"); + platform.put("version", System.getProperty("java.version")); + platform.put("vendor", System.getProperty("java.vendor")); + hello.put("runtime", platform); + hello.put("resource", runtime.resource); + hello.put("probes", RuntimeService.probes()); + hello.put("ops", runtime.diagnostics.catalog()); + return hello; + } + + @SuppressWarnings("unchecked") + private static Map body(Map envelope) throws Exception { + String body = Diagnostics.text(envelope, "body", null); + if (body == null || body.trim().isEmpty()) return new LinkedHashMap<>(); + Object parsed = + com.flanksource.telltale.diag.Json.read( + new ByteArrayInputStream(body.getBytes(StandardCharsets.UTF_8)), Object.class); + if (!(parsed instanceof Map)) throw new IllegalArgumentException("request body must be an object"); + return (Map) parsed; + } + + @SuppressWarnings("unchecked") + private static Map params(Map body) { + Object params = body.get("params"); + if (params == null) return new LinkedHashMap<>(); + if (!(params instanceof Map)) throw new IllegalArgumentException("params must be an object"); + return (Map) params; + } + + private static Map artifact(Capture capture, byte[] data) { + if (data.length > MAX_ARTIFACT) throw new IllegalStateException("artifact exceeds 64 MiB"); + Map reply = reply(200, capture.contentType(), data); + headers(reply) + .put("Content-Disposition", "attachment; filename=\"" + capture.fileName() + "\""); + return reply; + } + + /** Carries the capture's report beside the artifact it describes, when it has one. */ + private static void putReport(Map reply, Capture capture) { + Map report = capture.report(); + if (report != null) headers(reply).put(REPORT_HEADER, Agent.JSON.toJson(report)); + } + + private static Map json(int status, Object value) { + return reply(status, JSON_TYPE, Agent.JSON.toJson(value).getBytes(StandardCharsets.UTF_8)); + } + + static Map error(int status, String message) { + return json(status, Collections.singletonMap("error", message == null ? "" : message)); + } + + private static Map reply(int status, String contentType, byte[] body) { + Map reply = new LinkedHashMap<>(); + reply.put("status", status); + reply.put("contentType", contentType); + reply.put("body", body); + reply.put("headers", new LinkedHashMap()); + return reply; + } + + @SuppressWarnings("unchecked") + private static Map headers(Map reply) { + return (Map) reply.get("headers"); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/RuntimeOwner.java b/sdk/java/src/main/java/com/flanksource/telltale/RuntimeOwner.java new file mode 100644 index 0000000..02ce356 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/RuntimeOwner.java @@ -0,0 +1,16 @@ +package com.flanksource.telltale; + +import java.lang.management.ManagementFactory; +import javax.management.*; + +public final class RuntimeOwner { + private RuntimeOwner() {} + + public static ClassLoader loader() { + MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + try { + ObjectName runtime = new ObjectName(RuntimeService.NAME); + return server.isRegistered(runtime) ? server.getClassLoaderFor(runtime) : ClassLoader.getSystemClassLoader(); + } catch (JMException error) { throw new IllegalStateException("resolve JVM runtime owner", error); } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/RuntimeService.java b/sdk/java/src/main/java/com/flanksource/telltale/RuntimeService.java new file mode 100644 index 0000000..a207649 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/RuntimeService.java @@ -0,0 +1,345 @@ +package com.flanksource.telltale; + +import com.flanksource.telltale.auth.JwtVerifier; +import java.lang.management.ManagementFactory; +import java.time.Instant; +import java.util.*; +import javax.management.*; + +/** One JVM-wide owner, reached across application and servlet classloaders using JDK types. */ +public final class RuntimeService implements DynamicMBean { + public static final String NAME = "com.flanksource.telltale:type=Diagnostics"; + /** Bumped whenever an adapter-visible MBean action or its envelope changes. */ + public static final int API_VERSION = 1; + + private static final String[] ACTIONS = { + "configure", "describe", "catalog", "call", "http", "registerAdapter", "closeOwner", + "captureOpen", "captureDump", "captureComplete", "captureDropped", "captureReport", + "captureFinish", "captureRenew", "captureClose", "prepareUpgrade", "stopRuntime" + }; + + /** The protocol captures served by the observer runtime rather than by a probe or recording. */ + static final List OBSERVATIONS = + Collections.unmodifiableList( + Arrays.asList("http", "http_top", "sql", "sql_top", "jms", "jms_top", "spans")); + + final Diagnostics diagnostics; + final String started = Instant.now().toString(); + /** The configured agent resource; every capture opened over HTTP is attributed to it. */ + final Map resource; + private final Map configuration; + private final Map captures = new HashMap<>(); + private final Map owners = new HashMap<>(); + private final Map captureScopes = new HashMap<>(); + private final Map verifiers = new HashMap<>(); + private final RuntimeHttp http = new RuntimeHttp(this); + private final java.util.concurrent.locks.ReentrantReadWriteLock lifecycle = new java.util.concurrent.locks.ReentrantReadWriteLock(true); + private volatile boolean upgrading; + + private RuntimeService(Map config) throws Exception { + configuration = configuration(config); + resource = AgentResource.from(config); + diagnostics = new Diagnostics(config); + } + + public static void acquire(Map config) throws Exception { + MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + ObjectName name = new ObjectName(NAME); + synchronized (server) { + if (!server.isRegistered(name)) server.registerMBean(new RuntimeService(config), name); + } + server.invoke( + name, + "configure", + new Object[] {configuration(config)}, + new String[] {Map.class.getName()}); + } + + private static Map configuration(Map config) { + Map result = new LinkedHashMap<>(); + result.put("allowMutation", Boolean.TRUE.equals(config.get("allowMutation"))); + result.put("extensionsDir", config.get("extensionsDir")); + return result; + } + + /** The probe kinds every adapter advertises, with the scope each capture requires. */ + static List> probes() { + List> probes = new ArrayList<>(); + for (String name : + new String[] {"jvm_jfr", "jvm_trace", "jvm_redefine", "jvm_notifications", "jvm_ognl"}) { + Map probe = new LinkedHashMap<>(); + probe.put("name", name); + probe.put("title", name.equals("jvm_jfr") ? "JVM JFR" : name); + probe.put("checkpoints", true); + probe.put("mutates", !name.equals("jvm_notifications")); + probe.put("reverts", true); + probe.put("scope", Scope.forCapture(name).wire()); + // The generation of what jvm_trace accepts, so a client asking for an option a later + // generation added is refused before it arms rather than handed a capture that ignored it. + // The bundled generation, as describe() reports it: hello must not need the diag agent + // attached, and an older agent already on the system class path is still refused at arm + // time by ClassOperations' requireProbeApiFor. + if (name.equals("jvm_trace")) + probe.put("version", com.flanksource.telltale.diag.agent.DiagAgent.PROBE_API_VERSION); + probes.add(probe); + } + for (String name : OBSERVATIONS) { + Map observation = new LinkedHashMap<>(); + observation.put("name", name); + observation.put("title", name); + observation.put("checkpoints", true); + observation.put("mutates", false); + observation.put("reverts", true); + observation.put("scope", Scope.forCapture(name).wire()); + probes.add(observation); + } + return probes; + } + + @SuppressWarnings("unchecked") + public Object invoke(String action, Object[] values, String[] signature) + throws MBeanException, ReflectionException { + try { + if (action.equals("prepareUpgrade")) { prepareUpgrade(); return null; } + if (action.equals("stopRuntime")) { stopRuntime(); return null; } + lifecycle.readLock().lock(); + try { + if (upgrading && !action.equals("closeOwner") && !action.equals("describe")) + throw new IllegalStateException("agent runtime is upgrading"); + return dispatch(action, (Map) values[0]); + } finally { lifecycle.readLock().unlock(); } + } catch (Exception error) { + throw new MBeanException(error, error.toString()); + } + } + + private void prepareUpgrade() throws Exception { + if (!lifecycle.writeLock().tryLock(10, java.util.concurrent.TimeUnit.SECONDS)) + throw new IllegalStateException("agent has active diagnostic calls; retry the upgrade when they finish"); + try { + synchronized (captures) { + if (!captures.isEmpty()) throw new IllegalStateException("agent has active captures; stop them before upgrading"); + } + if (!com.flanksource.telltale.diag.agent.DiagAgent.ORIGINAL_BYTES.isEmpty()) + throw new IllegalStateException("agent has active patches; restore them before upgrading"); + if (!com.flanksource.telltale.diag.hotpatch.AgentBridge.probeKeys().isEmpty()) + throw new IllegalStateException("agent has active probes; stop them before upgrading"); + upgrading = true; + } finally { lifecycle.writeLock().unlock(); } + } + + private void stopRuntime() throws Exception { + lifecycle.writeLock().lock(); + try { + if (!upgrading) throw new IllegalStateException("prepare the agent upgrade before stopping its runtime"); + com.flanksource.telltale.observer.ObserverRuntime.shutdown(); + com.flanksource.telltale.diag.hotpatch.ProbeTransformations.shutdown(); + com.flanksource.telltale.diag.hotpatch.SourceTransformations.shutdown(); + com.flanksource.telltale.diag.agent.DiagAgent.RESTORER = null; + diagnostics.close(); + ManagementFactory.getPlatformMBeanServer().unregisterMBean(new ObjectName(NAME)); + } finally { lifecycle.writeLock().unlock(); } + } + + @SuppressWarnings("unchecked") + private Object dispatch(String action, Map request) throws Exception { + switch (action) { + case "configure": + if (!configuration.equals(request)) + throw new IllegalStateException( + "diagnostic runtime already initialized with different mutation or extension" + + " configuration"); + return null; + case "describe": + return describe(); + case "catalog": + return diagnostics.catalog(); + case "http": + return http.handle(request); + case "registerAdapter": + registerAdapter(request); + return null; + case "closeOwner": + closeOwner(Diagnostics.required(request, "owner")); + return null; + case "call": + return diagnostics.call( + Diagnostics.required(request, "operation"), + (Map) request.get("params"), + Boolean.TRUE.equals(request.get("allowMutation"))); + case "captureOpen": + return openCapture(request); + default: + return captureAction(action, Diagnostics.required(request, "id")); + } + } + + private Map describe() { + Map result = new LinkedHashMap<>(); + result.put("apiVersion", API_VERSION); + result.put("upgradeApiVersion", 1); + result.put("runtimeVersion", RuntimeService.class.getPackage().getImplementationVersion()); + result.put("artifactSHA256", resource.get("telltale.artifact.sha256")); + result.put("probeApiVersion", com.flanksource.telltale.diag.agent.DiagAgent.PROBE_API_VERSION); + return result; + } + + private void registerAdapter(Map request) { + String owner = Diagnostics.required(request, "owner"); + Object jwks = request.get("jwks"); + if (jwks != null && !(jwks instanceof String)) jwks = Agent.JSON.toJson(jwks); + JwtVerifier verifier = + new JwtVerifier( + Diagnostics.text(request, "issuer", null), + Diagnostics.text(request, "audience", null), + (String) jwks, + Diagnostics.text(request, "jwksUrl", null)); + synchronized (captures) { + if (verifiers.containsKey(owner)) + throw new IllegalStateException("adapter already registered: " + owner); + verifiers.put(owner, verifier); + } + } + + JwtVerifier verifier(String owner) { + synchronized (captures) { + return owner == null ? null : verifiers.get(owner); + } + } + + private void closeOwner(String owner) throws Exception { + List ids = new ArrayList<>(); + synchronized (captures) { + verifiers.remove(owner); + for (Map.Entry entry : owners.entrySet()) + if (entry.getValue().equals(owner)) ids.add(entry.getKey()); + } + Exception failure = null; + for (String id : ids) + try { + captureAction("captureClose", id); + } catch (Exception error) { + if (failure == null) failure = error; + else failure.addSuppressed(error); + } + if (failure != null) throw failure; + } + + @SuppressWarnings("unchecked") + Map openCapture(Map request) throws Exception { + String key = Diagnostics.required(request, "id"), kind = Diagnostics.required(request, "kind"); + Scope scope = Scope.forCapture(kind); + boolean allow = Boolean.TRUE.equals(request.get("allowMutation")); + synchronized (captures) { + if (captures.containsKey(key)) throw new IllegalArgumentException("duplicate capture " + key); + Map params = (Map) request.get("params"); + Map attributed = + request.containsKey("resource") + ? (Map) request.get("resource") + : resource; + Capture capture = + kind.equals("jvm_jfr") + ? new Recording(Agent.JSON.toJsonTree(params).getAsJsonObject()) + : diagnostics.capture(kind, params, key, allow, attributed); + captures.put(key, capture); + captureScopes.put(key, scope); + if (request.containsKey("owner")) owners.put(key, (String) request.get("owner")); + Map result = new LinkedHashMap<>(); + result.put("id", key); + result.put("streaming", capture.streaming()); + result.put("contentType", capture.contentType()); + result.put("fileName", capture.fileName()); + result.put("intervalMillis", capture.intervalMillis()); + return result; + } + } + + /** The capture an adapter may act on, or a {@link RuntimeHttp.Failure} (404/409). */ + Capture owned(String id, String owner) throws RuntimeHttp.Failure { + synchronized (captures) { + Capture capture = captures.get(id); + if (capture == null) throw new RuntimeHttp.Failure(404, "unknown capture " + id); + if (!Objects.equals(owners.get(id), owner)) + throw new RuntimeHttp.Failure(409, "capture does not belong to this adapter"); + return capture; + } + } + + Scope captureScope(String id) { + synchronized (captures) { + return captureScopes.get(id); + } + } + + Object captureAction(String action, String key) throws Exception { + Capture capture; + synchronized (captures) { + capture = captures.get(key); + } + if (capture == null) throw new IllegalArgumentException("unknown capture " + key); + synchronized (capture) { + switch (action) { + case "captureDump": + return capture.dump(); + case "captureComplete": + return capture.complete(); + case "captureDropped": + return capture.dropped(); + case "captureReport": + return capture.report(); + case "captureFinish": + return capture.finish(); + case "captureRenew": + capture.renew(); + return null; + case "captureClose": + capture.close(); + synchronized (captures) { + captures.remove(key); + owners.remove(key); + captureScopes.remove(key); + } + return null; + default: + throw new IllegalArgumentException("unknown runtime action " + action); + } + } + } + + public Object getAttribute(String name) throws AttributeNotFoundException { + throw new AttributeNotFoundException(name); + } + + public void setAttribute(Attribute attribute) throws AttributeNotFoundException { + throw new AttributeNotFoundException(attribute.getName()); + } + + public AttributeList getAttributes(String[] names) { + return new AttributeList(); + } + + public AttributeList setAttributes(AttributeList values) { + return new AttributeList(); + } + + public MBeanInfo getMBeanInfo() { + List operations = new ArrayList<>(); + for (String action : ACTIONS) + operations.add( + new MBeanOperationInfo( + action, + action, + new MBeanParameterInfo[] { + new MBeanParameterInfo("request", Map.class.getName(), "Request") + }, + Object.class.getName(), + MBeanOperationInfo.ACTION)); + return new MBeanInfo( + getClass().getName(), + "Telltale diagnostic runtime", + null, + null, + operations.toArray(new MBeanOperationInfo[0]), + null); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/Scope.java b/sdk/java/src/main/java/com/flanksource/telltale/Scope.java new file mode 100644 index 0000000..ed06dd4 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/Scope.java @@ -0,0 +1,52 @@ +package com.flanksource.telltale; + +/** Authorization scope an adapter caller needs for an operation, capture or route. */ +public enum Scope { + READ("jvm:read"), + TRACE("jvm:trace"), + INSTRUMENT("jvm:instrument"), + EVAL("jvm:eval"), + WRITE("jvm:write"); + + private final String wire; + + Scope(String wire) { + this.wire = wire; + } + + public String wire() { + return wire; + } + + public boolean mutates() { + return this != READ; + } + + public static Scope forCapture(String kind) { + if (kind == null) throw new IllegalArgumentException("capture kind is required"); + switch (kind) { + case "jvm_jfr": + case "jvm_trace": + case "jvm_notifications": + case "http": + case "http_top": + case "sql": + case "sql_top": + case "jms": + case "jms_top": + case "spans": + return TRACE; + case "jvm_redefine": + return INSTRUMENT; + case "jvm_ognl": + return EVAL; + default: + throw new IllegalArgumentException("unknown capture kind: " + kind); + } + } + + /** Scope of an extension operation, which declares only whether it mutates. */ + static Scope forExtension(boolean mutates) { + return mutates ? WRITE : READ; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/Scripts.java b/sdk/java/src/main/java/com/flanksource/telltale/Scripts.java new file mode 100644 index 0000000..256c5be --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/Scripts.java @@ -0,0 +1,63 @@ +package com.flanksource.telltale; + +import com.flanksource.telltale.diag.jmx.JmxService; +import java.lang.reflect.InvocationTargetException; +import java.net.*; +import java.util.*; + +final class Scripts { + @SuppressWarnings("unchecked") + static Map evaluate( + Map params, + ClassLoader application, + JmxService jmx, + Map extensions) + throws Exception { + Diagnostics.required(params, "source"); + if (params.containsKey("args") && !(params.get("args") instanceof Map)) + throw new IllegalArgumentException("args must be an object"); + URL engine = Scripts.class.getProtectionDomain().getCodeSource().getLocation(); + URL groovy = groovy.lang.GroovyShell.class.getProtectionDomain().getCodeSource().getLocation(); + ClassLoader runtime = Scripts.class.getClassLoader(); + try (URLClassLoader isolated = + new URLClassLoader(new URL[] {engine, groovy}, null) { + protected synchronized Class loadClass(String name, boolean resolve) + throws ClassNotFoundException { + Class type = findLoadedClass(name); + if (type == null) { + if (name.startsWith("groovy.") + || name.startsWith("groovyjarjar") + || name.startsWith("org.codehaus.groovy.") + || name.startsWith("org.apache.groovy.") + || name.startsWith("com.flanksource.telltale.GroovyEngine")) + type = findClass(name); + else if (name.startsWith("com.flanksource.telltale.")) type = runtime.loadClass(name); + else + try { + type = + application == null + ? Class.forName(name, false, null) + : application.loadClass(name); + } catch (ClassNotFoundException absent) { + type = runtime.loadClass(name); + } + } + if (resolve) resolveClass(type); + return type; + } + }) { + try { + return (Map) + isolated + .loadClass("com.flanksource.telltale.GroovyEngine") + .getMethod("evaluate", Map.class, ClassLoader.class, Object.class, Map.class) + .invoke(null, params, application, jmx, extensions); + } catch (InvocationTargetException failure) { + Throwable cause = failure.getCause(); + if (cause instanceof Exception) throw (Exception) cause; + if (cause instanceof Error) throw (Error) cause; + throw new IllegalStateException("Groovy execution failed", cause); + } + } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/Session.java b/sdk/java/src/main/java/com/flanksource/telltale/Session.java index 76132cb..43862a3 100644 --- a/sdk/java/src/main/java/com/flanksource/telltale/Session.java +++ b/sdk/java/src/main/java/com/flanksource/telltale/Session.java @@ -10,10 +10,25 @@ final class Session implements AutoCloseable { private final Agent agent; private final Wire wire; private final boolean remote; - private final Map recordings = new HashMap<>(); + private final java.util.concurrent.ThreadPoolExecutor calls = + new java.util.concurrent.ThreadPoolExecutor( + 4, + 4, + 0, + java.util.concurrent.TimeUnit.SECONDS, + new java.util.concurrent.ArrayBlockingQueue(64), + runnable -> { + Thread thread = new Thread(runnable, "telltale-operation"); + thread.setDaemon(true); + return thread; + }); + private final Map recordings = new HashMap<>(); private final Map sequences = new HashMap<>(); private final Map transferred = new HashMap<>(); private final Map byteLimits = new HashMap<>(); + private final Map> reported = new HashMap<>(); + private final Map>> scheduled = + new HashMap<>(); private final java.util.concurrent.ScheduledExecutorService deadlines = java.util.concurrent.Executors.newSingleThreadScheduledExecutor( r -> { @@ -52,6 +67,17 @@ void run() throws Exception { private synchronized void dispatch(JsonObject frame) throws Exception { String kind = frame.get("t").getAsString(); JsonObject body = frame.getAsJsonObject("body"); + if (kind.equals("call")) { + try { + calls.execute(() -> operation(body)); + } catch (java.util.concurrent.RejectedExecutionException error) { + JsonObject result = new JsonObject(); + result.add("callId", body.get("callId")); + result.addProperty("error", "too many active or queued operations"); + wire.send("result", result, null); + } + return; + } if (kind.equals("ping")) { wire.send("pong", new JsonObject(), null); return; @@ -66,7 +92,8 @@ private synchronized void dispatch(JsonObject frame) throws Exception { else throw new IllegalArgumentException("Unsupported frame: " + kind); try { if (kind.equals("arm")) { - if (!body.get("probe").getAsString().equals("jvm_jfr")) + String probe = body.get("probe").getAsString(); + if (RuntimeService.probes().stream().noneMatch(known -> probe.equals(known.get("name")))) throw new IllegalArgumentException("Unsupported probe"); if (recordings.containsKey(id)) throw new IllegalArgumentException("Duplicate session"); long delay = 0; @@ -80,52 +107,99 @@ private synchronized void dispatch(JsonObject frame) throws Exception { && body.getAsJsonObject("limits").has("maxBytes") && body.getAsJsonObject("limits").get("maxBytes").getAsLong() < 0) throw new IllegalArgumentException("Capture byte limit must not be negative"); - recordings.put(id, new Recording(body.getAsJsonObject("params"))); + Capture capture = + agent.diagnostics.capture( + probe, + parameters(body), + id, + body.has("allowMutation") && body.get("allowMutation").getAsBoolean()); + recordings.put(id, capture); + java.util.List> jobs = new java.util.ArrayList<>(); + scheduled.put(id, jobs); + jobs.add( + deadlines.scheduleWithFixedDelay( + () -> { + synchronized (Session.this) { + if (!recordings.containsKey(id)) return; + try { + capture.renew(); + } catch (Exception error) { + failCapture(id, error); + } + } + }, + 30, + 30, + java.util.concurrent.TimeUnit.SECONDS)); + if (capture.streaming()) + jobs.add( + deadlines.scheduleWithFixedDelay( + () -> { + synchronized (Session.this) { + if (!recordings.containsKey(id)) return; + try { + sendBlob(id, "", capture, capture.dump()); + if (capture.complete()) { + finishCapture(id); + JsonObject end = new JsonObject(); + end.addProperty("session", id); + end.addProperty("complete", true); + wire.send("ended", end, null); + } + } catch (Exception error) { + failCapture(id, error); + } + } + }, + capture.intervalMillis(), + capture.intervalMillis(), + java.util.concurrent.TimeUnit.MILLISECONDS)); if (body.has("limits") && body.getAsJsonObject("limits").has("maxBytes")) byteLimits.put(id, body.getAsJsonObject("limits").get("maxBytes").getAsLong()); if (delay > 0) { - deadlines.schedule( - () -> { - synchronized (Session.this) { - if (!recordings.containsKey(id)) return; - JsonObject end = new JsonObject(); - end.addProperty("session", id); - try (Recording r = recordings.remove(id)) { - sendBlob(id, "", r.dump()); - } catch (Exception error) { - end.addProperty("error", error.toString()); - } - try { - wire.send("ended", end, null); - sequences.remove(id); - transferred.remove(id); - byteLimits.remove(id); - } catch (Exception error) { - System.err.println( - "telltale deadline response failed: " + error.getClass().getSimpleName()); - } - } - }, - delay, - java.util.concurrent.TimeUnit.MILLISECONDS); + jobs.add( + deadlines.schedule( + () -> { + synchronized (Session.this) { + if (!recordings.containsKey(id)) return; + JsonObject end = new JsonObject(); + end.addProperty("session", id); + try { + finishCapture(id); + end.addProperty("complete", true); + } catch (Exception error) { + end.addProperty("error", error.toString()); + } + try { + wire.send("ended", end, null); + sequences.remove(id); + transferred.remove(id); + byteLimits.remove(id); + reported.remove(id); + cancelJobs(id); + } catch (Exception error) { + System.err.println( + "telltale deadline response failed: " + + error.getClass().getSimpleName()); + } + } + }, + delay, + java.util.concurrent.TimeUnit.MILLISECONDS)); } } else { - Recording recording = recordings.get(id); + Capture recording = recordings.get(id); if (recording == null) throw new IllegalArgumentException("Unknown session"); if (kind.equals("checkpoint")) { String checkpoint = body.get("id").getAsString(); response.addProperty("id", checkpoint); - sendBlob(id, checkpoint, recording.dump()); + sendBlob(id, checkpoint, recording, recording.dump()); } else { - try (Recording ending = recordings.remove(id)) { - sendBlob(id, "", ending.dump()); - } + finishCapture(id); } } } catch (Exception error) { - while (error instanceof java.lang.reflect.InvocationTargetException - && error.getCause() instanceof Exception) error = (Exception) error.getCause(); - response.addProperty("error", error.toString()); + response.addProperty("error", failureMessage(error)); if (kind.equals("checkpoint") && body.has("id")) response.add("id", body.get("id")); } wire.send(reply, response, null); @@ -133,14 +207,17 @@ private synchronized void dispatch(JsonObject frame) throws Exception { sequences.remove(id); transferred.remove(id); byteLimits.remove(id); + reported.remove(id); + cancelJobs(id); } } - private void sendBlob(String id, String checkpoint, byte[] data) throws Exception { + private void sendBlob(String id, String checkpoint, Capture capture, byte[] data) + throws Exception { long bytes = transferred.containsKey(id) ? transferred.get(id) : 0; long limit = byteLimits.containsKey(id) ? byteLimits.get(id) : 0; if (limit > 0 && data.length > limit - bytes) - throw new IllegalStateException("JFR capture byte limit exceeded"); + throw new IllegalStateException("Capture byte limit exceeded"); transferred.put(id, bytes + data.length); long sequence = sequences.containsKey(id) ? sequences.get(id) + 1 : 1; sequences.put(id, sequence); @@ -148,24 +225,152 @@ private void sendBlob(String id, String checkpoint, byte[] data) throws Exceptio blob.addProperty("session", id); blob.addProperty("checkpoint", checkpoint); blob.addProperty("seq", sequence); - blob.addProperty("name", "recording.jfr"); - blob.addProperty("contentType", "application/x-jfr"); + blob.addProperty("name", capture.fileName()); + blob.addProperty("contentType", capture.contentType()); blob.addProperty("len", data.length); wire.send("blob", blob, data); + sendReport(id, capture); + sendDropped(id, sequence, capture); + } + + /** Sends the capture's report after the blob it describes, when it changed since the last. */ + private void sendReport(String id, Capture capture) throws Exception { + Map summary = capture.report(); + if (summary == null || summary.equals(reported.get(id))) return; + reported.put(id, summary); + JsonObject report = new JsonObject(); + report.addProperty("session", id); + report.add("summary", Agent.JSON.toJsonTree(summary)); + wire.send("report", report, null); + } + + /** Tells the reader how many records the capture dropped, as an empty records frame. */ + private void sendDropped(String id, long sequence, Capture capture) throws Exception { + long dropped = capture.dropped(); + if (dropped <= 0) return; + JsonObject report = new JsonObject(); + report.addProperty("session", id); + report.addProperty("seq", sequence + 1); + report.addProperty("dropped", dropped); + report.add("items", new com.google.gson.JsonArray()); + sequences.put(id, sequence + 1); + wire.send("records", report, null); } - public synchronized void close() throws Exception { - deadlines.shutdownNow(); + private void finishCapture(String id) throws Exception { + try (Capture capture = recordings.remove(id)) { + if (capture == null) throw new IllegalArgumentException("Unknown session " + id); + sendBlob(id, "", capture, capture.finish()); + } finally { + cancelJobs(id); + sequences.remove(id); + transferred.remove(id); + byteLimits.remove(id); + reported.remove(id); + } + } + + public void close() throws Exception { Exception failure = null; - for (Recording r : recordings.values()) { - try { - r.close(); - } catch (Exception e) { - if (failure == null) failure = e; - else failure.addSuppressed(e); + synchronized (this) { + calls.shutdownNow(); + deadlines.shutdownNow(); + for (Capture r : recordings.values()) { + try { r.close(); } + catch (Exception error) { + if (failure == null) failure = error; + else failure.addSuppressed(error); + } } + recordings.clear(); + } + if (!(awaitStopped(calls) & awaitStopped(deadlines))) { + Exception timeout = new IllegalStateException("agent session workers did not stop"); + if (failure == null) failure = timeout; + else failure.addSuppressed(timeout); } - recordings.clear(); if (failure != null) throw failure; } + + boolean stopped() { return calls.isTerminated() && deadlines.isTerminated(); } + + private static boolean awaitStopped(java.util.concurrent.ExecutorService executor) { + boolean interrupted = false; + long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(5); + try { + while (true) { + try { return executor.awaitTermination(Math.max(0, deadline - System.nanoTime()), java.util.concurrent.TimeUnit.NANOSECONDS); } + catch (InterruptedException stop) { interrupted = true; } + } + } finally { + if (interrupted) Thread.currentThread().interrupt(); + } + } + + private void failCapture(String id, Exception error) { + cancelJobs(id); + sequences.remove(id); + transferred.remove(id); + byteLimits.remove(id); + reported.remove(id); + Capture capture = recordings.remove(id); + if (capture != null) { + try { + capture.close(); + } catch (Exception cleanup) { + error.addSuppressed(cleanup); + } + } + JsonObject result = new JsonObject(); + result.addProperty("session", id); + result.addProperty("error", error.toString()); + try { + wire.send("ended", result, null); + } catch (Exception failure) { + System.err.println("telltale capture failure: " + failure); + } + } + + private void cancelJobs(String id) { + java.util.List> jobs = scheduled.remove(id); + if (jobs != null) for (java.util.concurrent.ScheduledFuture job : jobs) job.cancel(false); + } + + private void operation(JsonObject body) { + JsonObject result = new JsonObject(); + result.add("callId", body.get("callId")); + try { + Object value = + agent.diagnostics.call( + body.get("op").getAsString(), + parameters(body), + body.has("allowMutation") && body.get("allowMutation").getAsBoolean()); + result.add("value", Agent.JSON.toJsonTree(value)); + } catch (Exception error) { + result.addProperty("error", failureMessage(error)); + } + try { + wire.send("result", result, null); + } catch (Exception error) { + System.err.println("telltale operation response failed: " + error); + } + } + + static String failureMessage(Throwable error) { + while (error instanceof java.lang.reflect.InvocationTargetException + && error.getCause() != null) error = error.getCause(); + return error.toString(); + } + + @SuppressWarnings("unchecked") + private static Map parameters(JsonObject body) throws Exception { + if (!body.has("params")) return new java.util.LinkedHashMap<>(); + Map params = + com.flanksource.telltale.diag.Json.read( + new java.io.ByteArrayInputStream( + body.get("params").toString().getBytes(StandardCharsets.UTF_8)), + Map.class); + if (params == null) throw new IllegalArgumentException("params must be an object"); + return params; + } } diff --git a/sdk/java/src/main/java/com/flanksource/telltale/auth/AuthException.java b/sdk/java/src/main/java/com/flanksource/telltale/auth/AuthException.java new file mode 100644 index 0000000..2f6bb90 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/auth/AuthException.java @@ -0,0 +1,28 @@ +package com.flanksource.telltale.auth; + +/** An authentication (401) or authorization (403) refusal; the message never contains the token. */ +public final class AuthException extends Exception { + private final int status; + + AuthException(int status, String reason) { + super(reason); + this.status = status; + } + + AuthException(int status, String reason, Throwable cause) { + super(reason, cause); + this.status = status; + } + + public int status() { + return status; + } + + static AuthException unauthorized(String reason) { + return new AuthException(401, reason); + } + + static AuthException forbidden(String reason) { + return new AuthException(403, reason); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/auth/Claims.java b/sdk/java/src/main/java/com/flanksource/telltale/auth/Claims.java new file mode 100644 index 0000000..f615d99 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/auth/Claims.java @@ -0,0 +1,41 @@ +package com.flanksource.telltale.auth; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import java.util.*; + +/** Typed reads of JWT header and claim members; a member of the wrong type is a 401. */ +final class Claims { + private Claims() {} + + static String string(JsonObject object, String name) throws AuthException { + JsonElement value = object.get(name); + if (value == null || value.isJsonNull()) return null; + if (!value.isJsonPrimitive() || !value.getAsJsonPrimitive().isString()) + throw AuthException.unauthorized("token " + name + " must be a string"); + return value.getAsString(); + } + + static Double number(JsonObject object, String name) throws AuthException { + JsonElement value = object.get(name); + if (value == null || value.isJsonNull()) return null; + if (!value.isJsonPrimitive() || !value.getAsJsonPrimitive().isNumber()) + throw AuthException.unauthorized("token " + name + " must be numeric seconds"); + return value.getAsDouble(); + } + + static List audiences(JsonObject claims) throws AuthException { + JsonElement value = claims.get("aud"); + if (value == null || value.isJsonNull()) return Collections.emptyList(); + if (value.isJsonPrimitive() && value.getAsJsonPrimitive().isString()) + return Collections.singletonList(value.getAsString()); + if (!value.isJsonArray()) throw AuthException.unauthorized("token aud must be a string or array"); + List result = new ArrayList<>(); + for (JsonElement item : value.getAsJsonArray()) { + if (!item.isJsonPrimitive() || !item.getAsJsonPrimitive().isString()) + throw AuthException.unauthorized("token aud must contain strings"); + result.add(item.getAsString()); + } + return result; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/auth/Jwks.java b/sdk/java/src/main/java/com/flanksource/telltale/auth/Jwks.java new file mode 100644 index 0000000..fb1970e --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/auth/Jwks.java @@ -0,0 +1,65 @@ +package com.flanksource.telltale.auth; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.math.BigInteger; +import java.security.KeyFactory; +import java.security.PublicKey; +import java.security.spec.RSAPublicKeySpec; +import java.util.*; + +/** Parses a JWKS into RSA signing keys by kid; any unusable key rejects the whole set. */ +final class Jwks { + private Jwks() {} + + static Map parse(String json, String source) { + JsonElement root; + try { + root = JsonParser.parseString(json); + } catch (RuntimeException error) { + throw new IllegalArgumentException(source + " is not valid JSON", error); + } + if (!root.isJsonObject() || !root.getAsJsonObject().has("keys") + || !root.getAsJsonObject().get("keys").isJsonArray()) + throw new IllegalArgumentException(source + " must be an object with a keys array"); + Map keys = new LinkedHashMap<>(); + for (JsonElement item : root.getAsJsonObject().getAsJsonArray("keys")) { + if (!item.isJsonObject()) throw new IllegalArgumentException(source + " keys must be objects"); + JsonObject jwk = item.getAsJsonObject(); + String kid = member(jwk, "kid", source); + if (keys.put(kid, key(jwk, kid, source)) != null) + throw new IllegalArgumentException(source + " has duplicate kid " + kid); + } + return keys; + } + + private static PublicKey key(JsonObject jwk, String kid, String source) { + String where = source + " key " + kid; + if (!"RSA".equals(member(jwk, "kty", where))) + throw new IllegalArgumentException(where + " must have kty RSA"); + if (jwk.has("alg") && !"RS256".equals(member(jwk, "alg", where))) + throw new IllegalArgumentException(where + " alg must be RS256"); + if (jwk.has("use") && !"sig".equals(member(jwk, "use", where))) + throw new IllegalArgumentException(where + " use must be sig"); + try { + return KeyFactory.getInstance("RSA") + .generatePublic( + new RSAPublicKeySpec(integer(member(jwk, "n", where)), integer(member(jwk, "e", where)))); + } catch (Exception error) { + throw new IllegalArgumentException(where + " is not a valid RSA public key", error); + } + } + + private static BigInteger integer(String base64url) { + return new BigInteger(1, Base64.getUrlDecoder().decode(base64url)); + } + + private static String member(JsonObject object, String name, String where) { + JsonElement value = object.get(name); + if (value == null || !value.isJsonPrimitive() || !value.getAsJsonPrimitive().isString() + || value.getAsString().isEmpty()) + throw new IllegalArgumentException(where + " requires string " + name); + return value.getAsString(); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/auth/JwksSource.java b/sdk/java/src/main/java/com/flanksource/telltale/auth/JwksSource.java new file mode 100644 index 0000000..26f9d20 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/auth/JwksSource.java @@ -0,0 +1,57 @@ +package com.flanksource.telltale.auth; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.security.PublicKey; +import java.util.Map; + +/** Fetches a JWKS from an http(s) URL with bounded time and size. */ +final class JwksSource { + private static final int TIMEOUT_MS = 5000; + private static final int MAX_BYTES = 1024 * 1024; + private final URL url; + + JwksSource(String location) { + try { + url = new URL(location); + } catch (IOException error) { + throw new IllegalArgumentException("jwksUrl is not a valid URL: " + location, error); + } + if (!url.getProtocol().equals("http") && !url.getProtocol().equals("https")) + throw new IllegalArgumentException("jwksUrl must use http or https: " + location); + } + + Map fetch() throws IOException { + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setConnectTimeout(TIMEOUT_MS); + connection.setReadTimeout(TIMEOUT_MS); + connection.setInstanceFollowRedirects(false); + connection.setRequestProperty("Accept", "application/json"); + try { + int status = connection.getResponseCode(); + if (status != 200) throw new IOException(url + " answered HTTP " + status); + try (InputStream input = connection.getInputStream()) { + return Jwks.parse(new String(read(input), StandardCharsets.UTF_8), url.toString()); + } catch (IllegalArgumentException invalid) { + throw new IOException(invalid.getMessage(), invalid); + } + } finally { + connection.disconnect(); + } + } + + private static byte[] read(InputStream input) throws IOException { + ByteArrayOutputStream result = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int count; + while ((count = input.read(buffer)) != -1) { + if (count > MAX_BYTES - result.size()) throw new IOException("JWKS exceeds 1 MiB"); + result.write(buffer, 0, count); + } + return result.toByteArray(); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/auth/JwtVerifier.java b/sdk/java/src/main/java/com/flanksource/telltale/auth/JwtVerifier.java new file mode 100644 index 0000000..2edd73e --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/auth/JwtVerifier.java @@ -0,0 +1,168 @@ +package com.flanksource.telltale.auth; + +import com.flanksource.telltale.Scope; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.nio.charset.StandardCharsets; +import java.security.PublicKey; +import java.security.Signature; +import java.util.*; +import java.util.concurrent.TimeUnit; + +/** Verifies RS256 bearer JWTs against a fixed issuer, audience and JWKS; fails closed. */ +public final class JwtVerifier { + static final long LEEWAY_SECONDS = 60; + static final long REFRESH_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(30); + + private final String issuer; + private final String audience; + private final Map configured; + private final JwksSource remote; + private Map fetched = Collections.emptyMap(); + private long lastRefresh; + private boolean refreshed; + + /** + * @param jwks inline JWKS JSON ({@code {"keys":[...]}}) or null + * @param jwksUrl http(s) URL serving a JWKS, or null; at least one key source is required + */ + public JwtVerifier(String issuer, String audience, String jwks, String jwksUrl) { + this.issuer = nonEmpty(issuer, "issuer"); + this.audience = nonEmpty(audience, "audience"); + if (blank(jwks) && blank(jwksUrl)) + throw new IllegalArgumentException("configure jwks, jwksFile or jwksUrl"); + configured = blank(jwks) ? Collections.emptyMap() : Jwks.parse(jwks, "inline jwks"); + remote = blank(jwksUrl) ? null : new JwksSource(jwksUrl); + } + + /** + * The header execution tokens travel in. The Kubernetes apiserver deletes {@code Authorization} + * from every request it proxies, so a token through {@code pods/proxy} or {@code services/proxy} + * must use this header; direct connections may use either. + */ + public static final String HEADER = "X-Telltale-Authorization"; + + /** + * The bearer credential of a request: {@link #HEADER}, else {@code Authorization}. Both present + * with different values is a 400, never a silent pick. + */ + public static String credential(String telltale, String authorization) throws AuthException { + boolean hasTelltale = !blank(telltale), hasAuthorization = !blank(authorization); + if (hasTelltale && hasAuthorization && !telltale.trim().equals(authorization.trim())) + throw new AuthException(400, HEADER + " and Authorization carry different credentials"); + return hasTelltale ? telltale : authorization; + } + + public Principal verify(String authorization, Scope required) throws AuthException { + Principal principal = verify(authorization); + principal.require(required); + return principal; + } + + public Principal verify(String authorization) throws AuthException { + String[] parts = bearer(authorization).split("\\.", -1); + if (parts.length != 3) throw AuthException.unauthorized("malformed bearer token"); + JsonObject header = segment(parts[0], "header"); + checkHeader(header); + checkSignature(key(header.get("kid").getAsString()), parts); + return claims(segment(parts[1], "payload")); + } + + private static String bearer(String authorization) throws AuthException { + if (authorization == null || authorization.trim().isEmpty()) + throw AuthException.unauthorized("missing bearer token"); + if (!authorization.regionMatches(true, 0, "Bearer ", 0, 7)) + throw AuthException.unauthorized("authorization must use the Bearer scheme"); + String token = authorization.substring(7).trim(); + if (token.isEmpty()) throw AuthException.unauthorized("missing bearer token"); + return token; + } + + private static JsonObject segment(String encoded, String name) throws AuthException { + try { + JsonElement value = + JsonParser.parseString( + new String(Base64.getUrlDecoder().decode(encoded), StandardCharsets.UTF_8)); + if (!value.isJsonObject()) throw AuthException.unauthorized("token " + name + " must be an object"); + return value.getAsJsonObject(); + } catch (RuntimeException error) { + throw AuthException.unauthorized("malformed token " + name); + } + } + + private static void checkHeader(JsonObject header) throws AuthException { + String alg = Claims.string(header, "alg"); + if (!"RS256".equals(alg)) + throw AuthException.unauthorized("token alg must be RS256, got " + (alg == null ? "none" : alg)); + if (header.has("crit")) throw AuthException.unauthorized("token crit header is not supported"); + String kid = Claims.string(header, "kid"); + if (kid == null || kid.isEmpty()) throw AuthException.unauthorized("token kid is required"); + } + + private static void checkSignature(PublicKey key, String[] parts) throws AuthException { + boolean valid; + try { + Signature signature = Signature.getInstance("SHA256withRSA"); + signature.initVerify(key); + signature.update((parts[0] + "." + parts[1]).getBytes(StandardCharsets.US_ASCII)); + valid = signature.verify(Base64.getUrlDecoder().decode(parts[2])); + } catch (Exception error) { + throw AuthException.unauthorized("invalid token signature"); + } + if (!valid) throw AuthException.unauthorized("invalid token signature"); + } + + private Principal claims(JsonObject claims) throws AuthException { + long now = System.currentTimeMillis() / 1000; + if (!issuer.equals(Claims.string(claims, "iss"))) + throw AuthException.unauthorized("token issuer is not " + issuer); + if (!Claims.audiences(claims).contains(audience)) + throw AuthException.unauthorized("token audience does not include " + audience); + Double exp = Claims.number(claims, "exp"); + if (exp == null) throw AuthException.unauthorized("token exp is required"); + if (now > exp + LEEWAY_SECONDS) throw AuthException.unauthorized("token expired"); + Double nbf = Claims.number(claims, "nbf"); + if (nbf != null && now < nbf - LEEWAY_SECONDS) + throw AuthException.unauthorized("token is not valid yet"); + Claims.number(claims, "iat"); + String subject = Claims.string(claims, "sub"); + if (subject == null || subject.trim().isEmpty()) + throw AuthException.unauthorized("token sub is required"); + String scope = Claims.string(claims, "scope"); + if (scope == null) throw AuthException.unauthorized("token scope is required"); + return new Principal(subject, new LinkedHashSet<>(Arrays.asList(scope.trim().split("\\s+")))); + } + + private PublicKey key(String kid) throws AuthException { + PublicKey key = configured.get(kid); + if (key != null) return key; + synchronized (this) { + key = fetched.get(kid); + if (key != null) return key; + if (remote == null) throw AuthException.unauthorized("unknown signing key " + kid); + long now = System.nanoTime(); + if (refreshed && now - lastRefresh < REFRESH_INTERVAL_NANOS) + throw AuthException.unauthorized("unknown signing key " + kid + "; JWKS refreshed recently"); + refreshed = true; + lastRefresh = now; + try { + fetched = remote.fetch(); + } catch (Exception error) { + throw new AuthException(401, "refresh JWKS: " + error.getMessage(), error); + } + key = fetched.get(kid); + if (key == null) throw AuthException.unauthorized("unknown signing key " + kid); + return key; + } + } + + private static String nonEmpty(String value, String name) { + if (blank(value)) throw new IllegalArgumentException(name + " is required"); + return value; + } + + static boolean blank(String value) { + return value == null || value.trim().isEmpty(); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/auth/Principal.java b/sdk/java/src/main/java/com/flanksource/telltale/auth/Principal.java new file mode 100644 index 0000000..00682b3 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/auth/Principal.java @@ -0,0 +1,28 @@ +package com.flanksource.telltale.auth; + +import com.flanksource.telltale.Scope; +import java.util.*; + +/** A verified caller: the JWT subject and the scopes it was granted. */ +public final class Principal { + private final String subject; + private final Set scopes; + + Principal(String subject, Set scopes) { + this.subject = subject; + this.scopes = Collections.unmodifiableSet(new LinkedHashSet<>(scopes)); + } + + public String subject() { + return subject; + } + + public Set scopes() { + return scopes; + } + + public void require(Scope scope) throws AuthException { + if (!scopes.contains(scope.wire())) + throw AuthException.forbidden("token lacks required scope " + scope.wire()); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/bootstrap/AgentBootstrap.java b/sdk/java/src/main/java/com/flanksource/telltale/bootstrap/AgentBootstrap.java new file mode 100644 index 0000000..32670d4 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/bootstrap/AgentBootstrap.java @@ -0,0 +1,138 @@ +package com.flanksource.telltale.bootstrap; + +import java.lang.instrument.Instrumentation; +import java.lang.management.ManagementFactory; +import java.lang.reflect.*; +import java.net.*; +import java.nio.file.*; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.jar.JarFile; +import javax.management.ObjectName; + +public final class AgentBootstrap { + private static RuntimeVersion current; + private AgentBootstrap() {} + + public static void agentmain(String options, Instrumentation instrumentation) throws Exception { + String[] args = options.split("\n", -1); + if (args.length != 4 || !(args[2].equals("true") || args[2].equals("false"))) + throw new IllegalArgumentException("Expected configuration path, artifact path, upgrade flag and result path"); + Path result = Paths.get(args[3]); + try { + install(Paths.get(args[1]), Paths.get(args[0]), instrumentation, Boolean.parseBoolean(args[2])); + Files.write(result, "ok".getBytes(StandardCharsets.UTF_8)); + } catch (Exception | LinkageError failure) { + java.io.StringWriter report = new java.io.StringWriter(); + failure.printStackTrace(new java.io.PrintWriter(report)); + try { Files.write(result, ("error\n" + report).getBytes(StandardCharsets.UTF_8)); } + catch (Exception reporting) { failure.addSuppressed(reporting); } + throw failure; + } + } + + public static void premain(String config, Instrumentation instrumentation) throws Exception { + Path artifact = Paths.get(AgentBootstrap.class.getProtectionDomain().getCodeSource().getLocation().toURI()); + install(artifact, Paths.get(config), instrumentation, false); + } + + public static synchronized void install(Path artifact, Path config, Instrumentation instrumentation, boolean upgrade) throws Exception { + String digest = digest(artifact); + if (current != null && !upgrade) { + if (!current.digest.equals(digest)) throw new IllegalStateException("agent artifact changed; use --upgrade to replace the runtime"); + current.invoke("validateRuntime", new Class[] {String.class}, config.toString()); + current.invoke("agentmain", new Class[] {String.class, Instrumentation.class}, config.toString(), instrumentation); + return; + } + RuntimeVersion candidate = new RuntimeVersion(artifact, digest); + RuntimeVersion previous = current; + boolean activated = false; + try { + candidate.invoke("validateRuntime", new Class[] {String.class}, config.toString()); + if (current == null && ManagementFactory.getPlatformMBeanServer().isRegistered(new ObjectName("com.flanksource.telltale:type=Diagnostics"))) + throw new IllegalStateException("the loaded agent predates runtime upgrades; restart the JVM once to install the upgrade-capable launcher"); + if (current != null) { + current.invoke("prepareUpgrade", new Class[0]); + current.invoke("stopRuntime", new Class[0]); + } + try { + candidate.start(config, instrumentation); + } catch (Exception | LinkageError failure) { + current = null; + try { + candidate.invoke("stopRuntime", new Class[0]); + if (previous != null) { + RuntimeVersion restored = new RuntimeVersion(previous.artifact, previous.digest); + Path saved = Files.createTempFile(previous.artifact.getParent(), "rollback-", ".json"); + try { + Files.write(saved, previous.configData); + restored.start(saved, instrumentation); + } finally { Files.deleteIfExists(saved); } + current = restored; + } + } catch (Exception | LinkageError recovery) { failure.addSuppressed(recovery); } + throw new IllegalStateException(current == null ? "agent activation failed; no runtime is active" : "agent upgrade failed; the previous runtime was restored", failure); + } + current = candidate; + activated = true; + } finally { + if (!activated) candidate.loader.close(); + } + } + + private static String digest(Path artifact) throws Exception { + try (JarFile jar = new JarFile(artifact.toFile())) { + if (jar.getJarEntry("com/flanksource/telltale/Agent.class") == null) + throw new java.io.IOException("agent artifact has no runtime entry point"); + } + byte[] hash = MessageDigest.getInstance("SHA-256").digest(Files.readAllBytes(artifact)); + StringBuilder value = new StringBuilder(); + for (byte part : hash) value.append(String.format("%02x", part & 255)); + return value.toString(); + } + + private static final class RuntimeVersion { + final URLClassLoader loader; + final Class entry; + final String digest; + final Path artifact; + byte[] configData; + + RuntimeVersion(Path artifact, String digest) throws Exception { + this.digest = digest; + this.artifact = artifact; + loader = new URLClassLoader(new URL[] {artifact.toUri().toURL()}, ClassLoader.getSystemClassLoader().getParent()) { + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + if (name.startsWith("com.flanksource.telltale.diag.agent.") || name.startsWith("com.flanksource.telltale.bootstrap.")) + return ClassLoader.getSystemClassLoader().loadClass(name); + return super.loadClass(name, resolve); + } + }; + try { + entry = loader.loadClass("com.flanksource.telltale.Agent"); + entry.getMethod("validateRuntime", String.class); + entry.getMethod("agentmain", String.class, Instrumentation.class); + entry.getMethod("prepareUpgrade"); + entry.getMethod("stopRuntime"); + } catch (Exception | LinkageError error) { + loader.close(); + throw error; + } + } + + void start(Path config, Instrumentation instrumentation) throws Exception { + configData = Files.readAllBytes(config); + invoke("agentmain", new Class[] {String.class, Instrumentation.class}, config.toString(), instrumentation); + } + + Object invoke(String method, Class[] types, Object... args) throws Exception { + try { return entry.getMethod(method, types).invoke(null, args); } + catch (InvocationTargetException error) { + Throwable cause = error.getCause(); + if (cause instanceof Error) throw (Error) cause; + if (cause instanceof Exception) throw (Exception) cause; + throw new IllegalStateException("runtime lifecycle failed: " + method, cause); + } + } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/Json.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/Json.java new file mode 100644 index 0000000..0080f0e --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/Json.java @@ -0,0 +1,35 @@ +package com.flanksource.telltale.diag; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import java.io.IOException; +import java.io.InputStream; + +/** + * Thin Jackson wrapper shared by the diagnostic servlets. Jackson is bundled in this WAR's + * WEB-INF/lib (this webapp has its own classloader and cannot rely on the host's copy). + */ +public final class Json { + + private static final ObjectMapper MAPPER = + new ObjectMapper().disable(SerializationFeature.FAIL_ON_EMPTY_BEANS); + + private Json() {} + + public static String write(Object value) { + try { + return MAPPER.writeValueAsString(value); + } catch (IOException e) { + throw new RuntimeException("serialize JSON: " + e.getMessage(), e); + } + } + + public static T read(InputStream in, Class type) throws IOException { + return MAPPER.readValue(in, type); + } + + public static T read(InputStream in, TypeReference type) throws IOException { + return MAPPER.readValue(in, type); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/AttachMain.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/AttachMain.java new file mode 100644 index 0000000..e8b0c5f --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/AttachMain.java @@ -0,0 +1,70 @@ +package com.flanksource.telltale.diag.agent; + +import java.lang.reflect.InvocationTargetException; + +/** + * Loads {@link DiagAgent} into an already-running JVM from the outside. + * + *

This exists because the Attach API has two halves that must come from the same JDK: {@code + * tools.jar} (the classes) and {@code libattach.so} (the native transport, found via {@code + * java.library.path}). A JRE-only application image has neither, and pointing an in-process + * classloader at a side-loaded {@code tools.jar} only supplies the first half — {@code + * VirtualMachine.attach} then dies with {@code UnsatisfiedLinkError: no attach in + * java.library.path}. + * + *

Running the attach in a child process launched from the side-loaded JDK's own {@code bin/java} + * makes both halves line up, because that launcher puts its own {@code jre/lib/} on the + * library path. It also sidesteps the Java 9+ {@code jdk.attach.allowAttachSelf} restriction: from + * here the target is another process, not this one. + * + *

Driven reflectively so this class compiles without {@code tools.jar} on the build's classpath; + * the launcher supplies it via {@code -cp}. + * + *

Usage: {@code java -cp : com.flanksource.telltale.diag.agent.AttachMain + * } + */ +public final class AttachMain { + + private static final String VIRTUAL_MACHINE = "com.sun.tools.attach.VirtualMachine"; + + private AttachMain() {} + + public static void main(String[] args) { + if (args.length != 2) { + System.err.println("usage: AttachMain "); + System.exit(2); + return; + } + try { + Class vmClass = Class.forName(VIRTUAL_MACHINE); + Object vm = vmClass.getMethod("attach", String.class).invoke(null, args[0]); + try { + vmClass.getMethod("loadAgent", String.class).invoke(vm, args[1]); + } finally { + detach(vmClass, vm); + } + System.out.println("attached " + args[1] + " to pid " + args[0]); + } catch (InvocationTargetException e) { + // The interesting failure is always the cause: AttachNotSupportedException, + // AgentLoadException, IOException from the attach socket. + print(e.getCause() == null ? e : e.getCause()); + System.exit(1); + } catch (Throwable t) { + print(t); + System.exit(1); + } + } + + private static void detach(Class vmClass, Object vm) { + try { + vmClass.getMethod("detach").invoke(vm); + } catch (ReflectiveOperationException | RuntimeException ignored) { + // The agent is already loaded; a failed detach must not mask that. + } + } + + private static void print(Throwable t) { + System.err.println(t.getClass().getName() + ": " + t.getMessage()); + t.printStackTrace(); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/DiagAgent.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/DiagAgent.java new file mode 100644 index 0000000..4a5af04 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/DiagAgent.java @@ -0,0 +1,1149 @@ +package com.flanksource.telltale.diag.agent; + +import com.flanksource.telltale.diag.graph.InertTypes; +import com.flanksource.telltale.diag.graph.ValueGraph; +import java.lang.instrument.Instrumentation; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Java agent holding the JVM's {@link Instrumentation} handle, the pre-patch bytes of every class + * the WAR has rewritten, and the events instrumented methods record. + * + *

This class ships in its own jar (the {@code agentJar} task) which {@code + * VirtualMachine.loadAgent} appends to the system class path. Two things follow, and + * between them they explain every design choice in here: + * + *

    + *
  • It outlives a WAR redeploy, which is why the byte stash and the probe buffers live here and + * not in the webapp — a redeployed WAR gets a fresh classloader and would lose them while the + * classes it patched stay patched. + *
  • It is reachable from every classloader in the JVM, which is what lets instrumented + * application code call {@link #probeEnter} by name. + *
+ * + *

The methods injected into application code — {@code probeEnter}, {@code probeReturn}, {@code + * probeThrow}, {@code probeExit} — run inside code that never asked to be observed. They catch + * {@link Throwable}, they do not block, they run none of the application's code, and they allocate + * no more than the event and its bounded value snapshot. Observation must not break the observed. + * + *

The class initialiser is held to the same bar for the same reason: it runs on whichever + * application thread first reaches an instrumented method, so a failure there would be an {@code + * ExceptionInInitializerError} in application's own call path, permanent until the JVM restarts. It + * therefore reads no system properties (a {@code SecurityManager} can deny that), does no I/O, and + * logs nothing. Configuration is resolved in {@link #openProbe}, on a servlet thread, where a + * failure is reportable. + */ +public final class DiagAgent { + + /** + * Probe API generation. The WAR checks this before it redefines anything. + * + *

The agent jar is on the system class path and cannot be reloaded without restarting the JVM, + * so a pod that attached an older agent and then received a newer WAR would otherwise have a + * class rewritten to call a method that is not there — and the {@code NoSuchMethodError} would + * surface inside application's own call path, which is a far worse failure than anything the WAR + * can produce on its own. Bumping this is how that is caught before the redefinition rather than + * after. + * + *

v3 added the descriptor parameter to {@link #probeEnter}. The signature changed rather than + * gaining an overload, because the emitter names exactly one and two would only make it possible + * to inject the wrong one. + * + *

v4 added {@link #releaseProbe} and {@link #probeKeysOwnedBy}, which the WAR needs to restore + * a class held by more than one span. Nothing injected changed, so a v3 agent would run the + * rewritten code perfectly well — it would fail at the restore instead, which is the worse + * moment: the class is already patched and the call that puts it back is the one missing. + * + *

v5 added {@link #probeArmingsOwnedBy} and the armings a ring now keeps, which the WAR needs + * to arm a SECOND span into a class a first span already armed. A v4 agent has no record of the + * first arming, so a WAR that assumed it did would rebuild the class from the new span alone and + * disarm the old one — reporting a clean capture of nothing. That was the live failure: the + * preservation plan arms ActivityDal three ways in one step and only the last-armed of the three + * ever recorded an event. + * + *

v6 added the event budget: a probe records {@code events} of them (default 100) and stops, + * reporting {@code complete} so the span disarms. Nothing injected changed and a v5 agent would + * run the rewritten code perfectly well — it would ignore the budget instead, which is worse than + * it sounds. An unknown option is ignored by design, so a caller who asked for fifty events would + * be given an unbounded capture and told nothing, and an instrumented method on a hot path would + * keep paying the probe cost for the life of its lease. + * + *

v7 changed what an event carries and one injected signature. Arguments and return values are + * object-graph snapshots rather than {@code String.valueOf} text, so the JSON a reader decodes + * changed shape; and {@link #probeEnter} gained the call site's line, so the viewer can show the + * code that made each call. The signature changed rather than gaining an overload, as in v3, and + * a v6 agent would fail the WAR's signature check before anything was rewritten. + * + *

v8 added {@code alloc}: a probe opened with it carries {@code allocBytes} on the events that + * close a frame. Nothing injected changed, so the WAR asks for v8 only when a span asks for + * {@code alloc} — a v7 agent would ignore the unknown option and report a capture with no + * allocation and no reason. + * + *

v9 added {@code gc}: {@code gcCount} and {@code gcPauseMs} on the events that close a frame, + * and the span's figure on every read. Gated the same way: only a span asking for {@code gc} + * needs v9. + * + *

v10 added root-call filters: {@code minCost}, {@code exceptionsOnly} and {@code match}. A + * filtered probe holds each root call on its thread and publishes it only when it qualifies, + * counting the rest as {@code filtered} on every read. Nothing injected changed; gated like + * {@code alloc} and {@code gc}, because a v9 agent would ignore the options and hand back every + * call as though each had matched. + */ + public static final int PROBE_API_VERSION = 10; + + /** Set by {@link #agentmain}; read reflectively from the webapp classloader. */ + public static volatile Instrumentation INSTRUMENTATION; + + /** + * Pre-patch class bytes, keyed {@code |}. + * + *

This lives here, on the system class path, rather than in the WAR: a redeployed WAR gets a + * fresh classloader and would lose the map, while the classes it patched stay patched — leaving a + * JVM running modified code with no way back. Both the key and the value are JDK types, so the + * WAR can read and write this map reflectively across the classloader boundary. + */ + public static final Map ORIGINAL_BYTES = new ConcurrentHashMap(); + + /** + * Patch leases, keyed {@code |} as {@link #ORIGINAL_BYTES} is, valued + * with a {@link System#currentTimeMillis} deadline. A patch whose deadline passes is restored by + * the reaper. + */ + public static final Map LEASES = new ConcurrentHashMap(); + + /** Restores one leased patch. Installed by the WAR, which owns redefinition. */ + public static volatile Restorer RESTORER; + + /** Open probe buffers, keyed by probe key. */ + private static final ConcurrentHashMap PROBES = + new ConcurrentHashMap(); + + /** + * The end state of recently closed probes, so a reader that polls once more after a span ends is + * told the span finished rather than getting a 404 that reads like a mistyped key. Bounded, and + * evicted oldest-first. + */ + private static final ConcurrentHashMap> CLOSED = + new ConcurrentHashMap>(); + + private static final List CLOSED_ORDER = new ArrayList(); + private static final int CLOSED_RETAINED = 32; + + private static final AtomicInteger OPEN_KEYS = new AtomicInteger(); + + /** + * Bumped whenever a probe closes, so a thread holding frames for a probe that has gone drops them + * the next time it records rather than carrying the depth forever. See {@link + * ProbeThreadState#resetIfStale}. + */ + private static final AtomicLong PROBE_EPOCH = new AtomicLong(); + + /** + * Per-thread depth, frame ids and open-invocation start times. Not an inheritable ThreadLocal: a + * child thread starts its own call stack, and inheriting the parent's depth would report nesting + * that never happened. + */ + private static final ThreadLocal STATE = + new ThreadLocal() { + @Override + protected ProbeThreadState initialValue() { + return new ProbeThreadState(Thread.currentThread()); + } + }; + + /** Set by the WAR so it can tell whether the reaper is already running. */ + private static volatile Thread reaper; + + /** How a lapsed lease is undone, without the agent jar depending on the WAR. */ + public interface Restorer { + /** + * @param key {@code |} + */ + void restore(String key) throws Exception; + } + + private DiagAgent() {} + + // ---------------------------------------------------------------- probes + + /** + * @return {@link #PROBE_API_VERSION}, so the WAR can refuse a stale agent. + */ + public static int probeApiVersion() { + return PROBE_API_VERSION; + } + + /** + * Opens a buffer for {@code key}, owned by {@code owner} (the {@code |} + * of the class being instrumented, so a restore or a lapsed lease can close it without a second + * registry). + * + *

Called before the class is rewritten. A refusal here — the key cap is full — therefore + * reaches the operator while the JVM is still untouched, rather than after a redefinition that + * would record into nothing. + */ + public static Map openProbe( + String key, String owner, Map options) { + ProbeConfig config = ProbeConfig.resolve(options); + Map result = new LinkedHashMap(); + result.put("key", key); + if (!config.enabled) { + result.put("opened", Boolean.FALSE); + result.put("error", "probes are disabled on this JVM (telltale.diag.probe.enabled=false)"); + return result; + } + String unmeasurable = unmeasurable(config); + if (unmeasurable != null) { + result.put("opened", Boolean.FALSE); + result.put("error", unmeasurable); + return result; + } + ProbeRing existing = PROBES.get(key); + if (existing != null) { + // A span instrumenting several methods arms them one at a time under + // ONE key, so a second open on a live key can be that span adding + // another class. It has to SAY so: two spans that collided on a key + // would otherwise merge their events into one ring silently, and a + // capture that quietly contains someone else's calls is worse than + // one that refused to start. + // + // Joining adds an owner, and the ring then closes only when the LAST + // of them is restored. Options are not re-read — the first open + // sized the ring and the caps, and resizing under a live writer is + // not something this can do. + if (!config.join) { + result.put("opened", Boolean.FALSE); + result.put( + "error", + "probe key " + + key + + " is already open; pass join:true to record" + + " another method of the same span into it"); + return result; + } + String mismatch = measurementMismatch(key, existing.config, config); + if (mismatch != null) { + result.put("opened", Boolean.FALSE); + result.put("error", mismatch); + return result; + } + if (owner != null) { + existing.owners.add(owner); + existing.arm(owner, options); + } + result.put("opened", Boolean.TRUE); + result.put("joined", Boolean.TRUE); + result.put("generation", Long.valueOf(existing.generation)); + result.put("capacity", Integer.valueOf(existing.capacity)); + putMeasurements(result, config); + return result; + } + if (OPEN_KEYS.get() >= config.maxKeys) { + result.put("opened", Boolean.FALSE); + result.put( + "error", + "this JVM already has " + + OPEN_KEYS.get() + + " probes open (" + + describeOpenKeys() + + ") and the cap is " + + config.maxKeys + + "; close one or raise telltale.diag.probe.keys"); + return result; + } + ProbeRing ring = new ProbeRing(key, owner, config); + if (PROBES.putIfAbsent(key, ring) != null) { + result.put("opened", Boolean.FALSE); + result.put("error", "probe key " + key + " is already open"); + return result; + } + OPEN_KEYS.incrementAndGet(); + CLOSED.remove(key); + ring.arm(owner, options); + result.put("opened", Boolean.TRUE); + result.put("generation", Long.valueOf(ring.generation)); + result.put("capacity", Integer.valueOf(ring.capacity)); + putMeasurements(result, config); + return result; + } + + /** Why an opt-in measurement this open asked for cannot be taken, or null. */ + private static String unmeasurable(ProbeConfig config) { + if (config.allocError != null) { + return "alloc was requested but cannot be measured: " + config.allocError; + } + if (config.gcError != null) { + return "gc was requested but cannot be measured: " + config.gcError; + } + if (config.filterError != null) { + return "the call filter cannot be evaluated: " + config.filterError; + } + return null; + } + + /** + * Refuses a join that disagrees with the ring about what it measures. The ring measures + * allocation and GC or it does not; a join that disagreed would get the ring's answer and believe + * its own. + */ + private static String measurementMismatch(String key, ProbeConfig open, ProbeConfig join) { + if (open.alloc != join.alloc) { + return "probe key " + + key + + " is open with alloc=" + + open.alloc + + " and this join asked for alloc=" + + join.alloc + + "; every target of a span is armed with the same alloc"; + } + if (open.gc != join.gc) { + return "probe key " + + key + + " is open with gc=" + + open.gc + + " and this join asked for gc=" + + join.gc + + "; every target of a span is armed with the same gc"; + } + String opened = open.filter == null ? "none" : open.filter.describe(); + String joining = join.filter == null ? "none" : join.filter.describe(); + if (!opened.equals(joining)) { + return "probe key " + + key + + " is open with filter " + + opened + + " and this join asked for " + + joining + + "; every target of a span is armed with the same filter"; + } + return null; + } + + /** Confirms in an open's answer each opt-in measurement the ring takes. */ + private static void putMeasurements(Map result, ProbeConfig config) { + if (config.alloc) { + result.put("alloc", Boolean.TRUE); + } + if (config.gc) { + result.put("gc", Boolean.TRUE); + } + if (config.filter != null) { + result.put("filter", config.filter.describe()); + } + } + + /** What the filter withheld: calls decided against, and events cut from kept calls. */ + private static void putFiltered(Map result, ProbeRing ring) { + result.put("filtered", Long.valueOf(ring.filteredCount())); + result.put("omitted", Long.valueOf(ring.omittedCount())); + } + + /** + * The collections the JVM ran since a gc probe opened, as {@code gc: {count, pauseMs}}. This is + * the span's figure: frames nest and overlap across threads, so summing their per-frame deltas + * would count one pause many times. Left out when a counter cannot be read, rather than reported + * as zero. + */ + private static void putSpanGc(Map result, ProbeRing ring) { + Map span = + ring.config.gcCounters == null ? null : ring.config.gcCounters.sinceOpen(); + if (span != null) { + result.put("gc", span); + } + } + + /** + * Fixes the span's GC figure when recording stops, so later reads report the span, not the JVM + * since. + */ + private static void stopGc(ProbeRing ring) { + if (ring.config.gcCounters != null) { + ring.config.gcCounters.stop(); + } + } + + /** Closes one probe, releasing its buffer. Safe on a key that is not open. */ + public static Map closeProbe(String key) { + ProbeRing ring = PROBES.remove(key); + if (ring == null) { + Map absent = new LinkedHashMap(); + absent.put("key", key); + absent.put("closed", Boolean.FALSE); + return absent; + } + OPEN_KEYS.decrementAndGet(); + if (ring.freeze()) { + // Any frame this probe had open will never receive its exit, so every + // thread's depth has to forget them rather than stay raised. + PROBE_EPOCH.incrementAndGet(); + } + stopGc(ring); + Map end = new LinkedHashMap(); + end.put("key", key); + end.put("closed", Boolean.TRUE); + end.put("generation", Long.valueOf(ring.generation)); + end.put("writeSeq", Long.valueOf(ring.writeSeq())); + end.put("capacity", Integer.valueOf(ring.capacity)); + putSpanGc(end, ring); + putFiltered(end, ring); + ring.close(); + rememberClosed(key, end); + return end; + } + + /** Stops one probe accepting events while retaining its readable buffer. */ + public static Map freezeProbe(String key) { + ProbeRing ring = key == null ? null : PROBES.get(key); + if (ring == null) { + throw new IllegalArgumentException("no probe is open for key " + key); + } + boolean changed = ring.freeze(); + if (changed) { + PROBE_EPOCH.incrementAndGet(); + } + stopGc(ring); + Map result = new LinkedHashMap(); + result.put("key", key); + result.put("frozen", Boolean.TRUE); + result.put("changed", Boolean.valueOf(changed)); + result.put("generation", Long.valueOf(ring.generation)); + result.put("writeSeq", Long.valueOf(ring.writeSeq())); + result.put("capacity", Integer.valueOf(ring.capacity)); + return result; + } + + /** + * Releases {@code owner}'s claim on every probe recording for it, closing the ones no longer + * claimed by anything. + * + *

This is what a restore and the lease reaper call. Without it a killed client leaks its + * buffer for the life of the JVM: probe keys carry a random per-span suffix, so every abandoned + * span would strand a distinct one that nothing would ever look up again. + * + *

It releases rather than closes because a span may instrument several classes into one ring, + * and they are restored one at a time. Closing on the first would leave the rest instrumented and + * recording into a closed ring — probeEnter early-returns, no counter moves, and the span reports + * a method that never ran. So the ring survives until its last owner is gone. + * + *

Returns only the probes that actually closed, so a caller can report what it ended rather + * than what it merely let go of. + */ + public static List> closeProbesOwnedBy(String owner) { + List> closed = new ArrayList>(); + if (owner == null) { + return closed; + } + for (Iterator> it = PROBES.entrySet().iterator(); it.hasNext(); ) { + Map.Entry entry = it.next(); + ProbeRing ring = entry.getValue(); + if (!ring.owners.remove(owner)) { + continue; + } + if (ring.owners.isEmpty()) { + closed.add(closeProbe(entry.getKey())); + } + } + return closed; + } + + /** + * Releases one probe's claim on {@code owner}, closing its ring if that was the last class it + * recorded for. + * + *

The single-key counterpart of {@link #closeProbesOwnedBy}, and what a restore calls when it + * knows which span is finishing. The difference matters when several spans instrument ONE class: + * releasing by owner alone would take every one of their rings down together, so the first span + * to finish would silently disarm the rest. + * + * @return {@code released} — whether this key was open and did record for that class — and {@code + * closed}, whether that was its last owner. A caller decides from {@link #probeKeysOwnedBy}, + * not from these, whether the class's original bytes may go back. + */ + public static Map releaseProbe(String key, String owner) { + ProbeRing ring = key == null ? null : PROBES.get(key); + if (ring == null || owner == null || !ring.owners.remove(owner)) { + Map nothing = new LinkedHashMap(); + nothing.put("key", key); + nothing.put("released", Boolean.FALSE); + nothing.put("closed", Boolean.FALSE); + return nothing; + } + if (!ring.owners.isEmpty()) { + Map held = new LinkedHashMap(); + held.put("key", key); + held.put("released", Boolean.TRUE); + held.put("closed", Boolean.FALSE); + held.put("owners", new ArrayList(ring.owners)); + return held; + } + Map end = new LinkedHashMap(closeProbe(key)); + end.put("released", Boolean.TRUE); + return end; + } + + /** + * The keys of every open probe still recording for {@code owner}. + * + *

The reverse of the {@code owners} a ring carries, and the question a restore has to answer + * before it puts a class's original bytes back: those bytes are shared by every span instrumented + * into the class, so restoring while another span still holds it disarms that span without + * telling it. + */ + public static List probeKeysOwnedBy(String owner) { + List keys = new ArrayList(); + if (owner == null) { + return keys; + } + for (Map.Entry entry : PROBES.entrySet()) { + if (entry.getValue().owners.contains(owner)) { + keys.add(entry.getKey()); + } + } + return keys; + } + + /** + * Everything armed into {@code owner}'s class that is still recording, in the order it was armed. + * + *

The question the WAR has to answer BEFORE it rewrites a class, and the counterpart of {@link + * #probeKeysOwnedBy}, which answers it before a restore. A class is redefined whole, so arming a + * second method replaces the first method's rewritten bytes along with everything else — the + * second span has to rebuild the union, and this is where it reads what the union is. + * + *

Each row is the {@code options} the WAR opened the probe with, plus the {@code owner} it + * armed and the {@code key} it records under. The agent puts nothing of its own in and reads + * nothing out: what the rows mean is the WAR's business, which is what keeps a change to the + * rewriting from needing an agent that can only be replaced by restarting the JVM. + */ + public static List> probeArmingsOwnedBy(String owner) { + List> armings = new ArrayList>(); + if (owner == null) { + return armings; + } + for (Map.Entry entry : PROBES.entrySet()) { + for (Map armed : entry.getValue().armings) { + if (!owner.equals(armed.get("owner"))) { + continue; + } + Map row = new LinkedHashMap(armed); + row.put("key", entry.getKey()); + armings.add(row); + } + } + Collections.sort( + armings, + new Comparator>() { + @Override + public int compare(Map left, Map right) { + return Long.valueOf(armedSeq(left)).compareTo(Long.valueOf(armedSeq(right))); + } + }); + return armings; + } + + private static long armedSeq(Map arming) { + Object seq = arming.get("armedSeq"); + return seq instanceof Number ? ((Number) seq).longValue() : 0L; + } + + /** + * Closes every open probe. + * + *

The operator's blunt instrument for "stop recording, whatever is going on" — a probe keeps + * its buffer until something closes it, and this closes all of them without needing to know their + * keys. It does not restore any class: an instrumented method whose probe is closed simply + * records nothing, which is why this is safe to call at any time. + */ + public static List> closeAllProbes() { + List> closed = new ArrayList>(); + for (String key : new ArrayList(PROBES.keySet())) { + closed.add(closeProbe(key)); + } + return closed; + } + + /** Every open probe, for diagnosing what a JVM is still recording. */ + public static List> probeKeys() { + List> keys = new ArrayList>(); + for (Map.Entry entry : PROBES.entrySet()) { + ProbeRing ring = entry.getValue(); + Map row = new LinkedHashMap(); + row.put("key", entry.getKey()); + // A list, not the first owner under the old singular name: a reader + // deciding whether a restore will interrupt a capture needs every + // class the ring records for, and one of several is a wrong answer + // that looks like a right one. This route is a live read with nothing + // persisted behind it, so the rename is clean. + row.put("owners", new ArrayList(ring.owners)); + row.put("generation", Long.valueOf(ring.generation)); + row.put("capacity", Integer.valueOf(ring.capacity)); + row.put("writeSeq", Long.valueOf(ring.writeSeq())); + row.put("oldestSeq", Long.valueOf(ring.oldestSeq())); + row.put("degraded", Boolean.valueOf(ring.isDegraded())); + keys.add(row); + } + return keys; + } + + /** + * Reads forward from {@code sinceSeq} without removing anything. + * + *

Every value crossing back is a JDK type, because the caller is the WAR reaching in through + * the system classloader. The events themselves are already-rendered JSON lines: the WAR forwards + * them without understanding them and the CLI appends them to its archive verbatim. + * + *

Non-destructive is deliberate. It lets a following CLI and a browser watch one span at once, + * and it removes the race a destructive drain has, where an event written between the read and + * the removal is lost with nothing to show it ever existed. + */ + public static Map drain(String key, long sinceSeq, int limit) { + Map result = new LinkedHashMap(); + result.put("key", key); + ProbeRing ring = PROBES.get(key); + if (ring == null) { + Map end = CLOSED.get(key); + if (end != null) { + result.putAll(end); + result.put("active", Boolean.FALSE); + result.put("events", new ArrayList()); + return result; + } + result.put("active", Boolean.FALSE); + result.put("closed", Boolean.FALSE); + result.put("error", "no probe is open for key " + key); + return result; + } + ProbeRing.Batch batch = ring.read(sinceSeq, limit <= 0 ? 1000 : limit); + List lines = new ArrayList(batch.events.size()); + for (ProbeEvent event : batch.events) { + lines.add(ProbeJson.line(event)); + } + result.put("generation", Long.valueOf(ring.generation)); + result.put("events", lines); + result.put("nextSeq", Long.valueOf(batch.nextSeq)); + result.put("oldestSeq", Long.valueOf(batch.oldestSeq)); + result.put("writeSeq", Long.valueOf(batch.writeSeq)); + result.put("dropped", Long.valueOf(batch.dropped)); + result.put("truncated", Boolean.valueOf(batch.truncated)); + result.put("suppressed", Long.valueOf(ring.suppressedCount())); + putFiltered(result, ring); + result.put("degraded", Boolean.valueOf(ring.isDegraded())); + result.put("capacity", Integer.valueOf(ring.capacity)); + putSpanGc(result, ring); + result.put("active", Boolean.valueOf(ring.isActive())); + result.put("closed", Boolean.FALSE); + // Reported alongside active rather than folded into it: a complete probe + // has stopped recording but still holds everything it recorded, which is + // the opposite of what a reader should conclude from active:false. It is + // also the signal to disarm — the client that opened the span is the only + // party that can restore the class without closing buffers nobody has + // read yet, so this is how it is told there is nothing left to wait for. + result.put("complete", Boolean.valueOf(ring.isComplete())); + result.put("maxEvents", Integer.valueOf(ring.config.maxEvents)); + return result; + } + + // ------------------------------------------------- injected probe points + + /** + * Opens a frame for one invocation and records that it started. + * + *

Returns the frame id the matching {@link #probeExit}, {@link #probeReturn} and {@link + * #probeThrow} carry back, or 0 when nothing is recording — the injected code passes whatever + * comes back along without inspecting it, so an unopened probe costs one map lookup and nothing + * else. + * + * @param descriptor the raw JVM descriptor of the method entered, pushed as a constant by the + * emitter. It is what tells two overloads apart once one probe key covers a set of methods, + * which {@code method} on its own cannot. + * @param line for a call site, the caller's source line the call was made from, pushed as a + * constant; 0 for a method's own entry and for a class compiled without line numbers. + */ + public static long probeEnter( + String key, String className, String method, String descriptor, int line, Object[] args) { + try { + ProbeRing ring = PROBES.get(key); + if (ring == null || !ring.beginWrite()) { + return 0L; + } + try { + ProbeThreadState state = state(); + ProbeFilter filter = ring.config.filter; + ProbePending held = filter == null ? null : state.pending(ring); + int depth = state.depth(); + // A filtered probe's root renders its arguments even when the ring is + // degraded: the predicate needs them, and only a kept call is retained. + // A call already rejected renders nothing more. + boolean root = filter != null && (held == null || depth <= held.rootDepth); + boolean rejected = held != null && !root && held.rejected; + ValueGraph.Node[] rendered = + renderArgs(ring, state, rejected ? null : args, root && filter.hasArgPredicates()); + long frame = state.nextFrame(); + long parent = state.parentFrame(ring.config.maxDepth); + // After the arguments are rendered, so the snapshot is not charged + // to the method it describes. + long startAlloc = allocated(ring); + ProbeGc gc = ring.config.gcCounters; + long startGcCount = gc == null ? -1L : gc.collections(); + long startGcMillis = gc == null ? -1L : gc.collectionMillis(); + long startNanos = System.nanoTime(); + state.push( + frame, startNanos, startAlloc, startGcCount, startGcMillis, ring.config.maxDepth); + ProbeEvent event = + ProbeEvent.enter( + ring.now(), + state.threadId, + state.threadName, + key, + depth, + frame, + parent, + className, + method, + descriptor, + line, + rendered, + state.renderedBytes, + state.truncated); + if (filter == null) { + ring.add(event); + } else if (!root) { + held.hold(event); + } else { + holdRoot(ring, state, held, event, filter.matchesArgs(rendered)); + } + return frame; + } finally { + ring.endWrite(); + } + } catch (Throwable ignored) { + // See the class javadoc: observation must never break the observed. + return 0L; + } + } + + /** + * Records a returned value and hands it back unchanged, so the instrumented method's own {@code + * return} keeps its meaning. + * + *

The identity of the value matters more than it looks: a probe that normalised, interned or + * copied what it observed would be a probe that changed the program it was measuring. + */ + public static T probeReturn(String key, long frame, T value) { + try { + ProbeRing ring = PROBES.get(key); + if (ring != null && frame != 0L && ring.beginWrite()) { + try { + ProbeThreadState state = state(); + if (ring.config.allocation != null) { + // Before the value is snapshotted: the exit that follows + // closes the frame on this reading, not on one that + // includes rendering what the method returned. + state.markReturn(frame, allocated(ring)); + } + ProbeFilter filter = ring.config.filter; + ProbePending held = filter == null ? null : state.pending(ring); + boolean root = held != null && held.rootFrame == frame; + if (filter == null || (held != null && !held.rejected)) { + ValueGraph.Node rendered = + renderValue(ring, state, value, root && filter.hasReturnPredicates()); + ProbeEvent event = + ProbeEvent.returned( + ring.now(), + state.threadId, + state.threadName, + key, + state.depth(), + frame, + rendered, + state.renderedBytes, + state.truncated); + if (filter == null) { + ring.add(event); + } else if (root) { + held.returnMatched = filter.matchesReturn(rendered); + held.close(event); + } else { + held.hold(event); + } + } + } finally { + ring.endWrite(); + } + } + } catch (Throwable ignored) { + // Fall through: the value is returned either way. + } + return value; + } + + /** Records an exception on its way out, closing the frame it escaped. */ + public static void probeThrow(String key, long frame, Throwable error) { + try { + ProbeRing ring = PROBES.get(key); + if (ring == null || frame == 0L || !ring.beginWrite()) { + return; + } + try { + ProbeThreadState state = state(); + long endAlloc = allocated(ring); + ProbeGc gc = ring.config.gcCounters; + long endGcCount = gc == null ? -1L : gc.collections(); + long endGcMillis = gc == null ? -1L : gc.collectionMillis(); + long startNanos = state.popTo(frame); + long allocBytes = ProbeAllocation.delta(state.poppedAlloc(), endAlloc); + String rendered = describeThrowable(ring, state, error); + publishClose( + ring, + state, + ProbeEvent.thrown( + ring.now(), + state.threadId, + state.threadName, + key, + state.depth(), + frame, + rendered, + elapsed(startNanos), + allocBytes, + ProbeAllocation.delta(state.poppedGcCount(), endGcCount), + ProbeAllocation.delta(state.poppedGcMillis(), endGcMillis), + state.lastUnwound(), + state.truncated), + true); + } finally { + ring.endWrite(); + } + } catch (Throwable ignored) { + // As above. + } + } + + /** Closes a frame that finished normally, with how long it took. */ + public static void probeExit(String key, long frame) { + try { + ProbeRing ring = PROBES.get(key); + if (ring == null || frame == 0L || !ring.beginWrite()) { + return; + } + try { + ProbeThreadState state = state(); + long endAlloc = state.takeReturnAlloc(frame); + if (endAlloc < 0L) { + endAlloc = allocated(ring); + } + ProbeGc gc = ring.config.gcCounters; + long endGcCount = gc == null ? -1L : gc.collections(); + long endGcMillis = gc == null ? -1L : gc.collectionMillis(); + long startNanos = state.popTo(frame); + publishClose( + ring, + state, + ProbeEvent.exited( + ring.now(), + state.threadId, + state.threadName, + key, + state.depth(), + frame, + elapsed(startNanos), + ProbeAllocation.delta(state.poppedAlloc(), endAlloc), + ProbeAllocation.delta(state.poppedGcCount(), endGcCount), + ProbeAllocation.delta(state.poppedGcMillis(), endGcMillis), + state.lastUnwound()), + false); + } finally { + ring.endWrite(); + } + } catch (Throwable ignored) { + // As above. + } + } + + // ---------------------------------------------------------- call filters + + /** + * Starts holding a filtered probe's root call on this thread. A call the probe was still holding + * here is one whose close was lost (an exception escaping uninstrumented code, a probe opened + * mid-call): it can never be decided, so it is counted as filtered and let go. + */ + private static void holdRoot( + ProbeRing ring, + ProbeThreadState state, + ProbePending abandoned, + ProbeEvent enter, + boolean argsMatched) { + if (abandoned != null) { + state.release(abandoned); + ring.filter(); + } + ProbePending call = new ProbePending(ring, enter.frame, enter.depth, !argsMatched); + state.hold(call); + call.hold(enter); + } + + /** + * Publishes an event that closes a frame. For a filtered probe it is held with its call, and the + * root's own close decides the whole call: published in order, or counted as {@code filtered}. + */ + private static void publishClose( + ProbeRing ring, ProbeThreadState state, ProbeEvent event, boolean threw) { + ProbeFilter filter = ring.config.filter; + if (filter == null) { + ring.add(event); + return; + } + ProbePending held = state.pending(ring); + if (held == null) { + // A frame entered before this probe opened: there is no call to decide. + return; + } + if (event.frame != held.rootFrame) { + if (event.depth > held.rootDepth) { + held.hold(event); + return; + } + // Unwound past the root: its own close was lost and never comes. + state.release(held); + ring.filter(); + return; + } + held.close(event); + state.release(held); + if (!held.rejected && filter.keeps(event.durationNs, threw, held.returnMatched)) { + held.publish(); + } else { + ring.filter(); + } + } + + // ------------------------------------------------------ value rendering + + /** This thread's state, with frames from a closed probe already forgotten. */ + private static ProbeThreadState state() { + ProbeThreadState state = STATE.get(); + state.resetIfStale(PROBE_EPOCH.get()); + return state; + } + + private static long elapsed(long startNanos) { + return startNanos < 0L ? 0L : System.nanoTime() - startNanos; + } + + /** The thread's allocation counter when the ring measures it, else -1. */ + private static long allocated(ProbeRing ring) { + ProbeAllocation allocation = ring.config.allocation; + return allocation == null ? -1L : allocation.currentThreadBytes(); + } + + /** + * Snapshots the arguments, all of them through ONE walk: they share the event's node and text + * budgets, and an object passed twice is shown once. + */ + private static ValueGraph.Node[] renderArgs( + ProbeRing ring, ProbeThreadState state, Object[] args, boolean evenIfDegraded) { + state.truncated = false; + state.renderedBytes = 0L; + if (args == null + || ProbeConfig.ARGS_NONE.equals(ring.config.argMode) + || (ring.isDegraded() && !evenIfDegraded)) { + return null; + } + int count = Math.min(args.length, ring.config.maxArgs); + ValueGraph.Node[] rendered = new ValueGraph.Node[count + (count < args.length ? 1 : 0)]; + ValueGraph graph = new ValueGraph(ring.config.graph); + for (int i = 0; i < count; i++) { + rendered[i] = describe(ring, state, graph, args[i], "arg" + i); + } + if (count < args.length) { + rendered[count] = ValueGraph.marker("(truncated)", (args.length - count) + " more args"); + state.truncated = true; + } + state.truncated |= graph.truncated(); + state.renderedBytes = graph.bytes(); + return rendered; + } + + private static ValueGraph.Node renderValue( + ProbeRing ring, ProbeThreadState state, Object value, boolean evenIfDegraded) { + state.truncated = false; + state.renderedBytes = 0L; + if (ProbeConfig.ARGS_NONE.equals(ring.config.argMode) + || (ring.isDegraded() && !evenIfDegraded)) { + return null; + } + ValueGraph graph = new ValueGraph(ring.config.graph); + ValueGraph.Node rendered = describe(ring, state, graph, value, "return"); + state.truncated = graph.truncated(); + state.renderedBytes = graph.bytes(); + return rendered; + } + + /** + * Snapshots one value for reporting. + * + *

The snapshot is an object graph built by reading fields, never by calling the value's own + * methods: {@code toString()} on a lazily initialised persistence proxy can fire a query or dirty + * a session, and one that throws or reaches another probe would make the diagnostic the failure. + * application's DCLs do not override {@code toString()} anyway, so text would only have said + * {@code ActivityDcl@d7c405bf}. The walk is bounded in depth, children, nodes and characters, + * because what it builds sits in the ring until it is read — an application {@code XMLData} + * payload is the obvious value that would otherwise fill the heap of the JVM being diagnosed. + * + *

{@link ProbeConfig#ARGS_TYPE} still exists for the cheapest capture: the class and identity + * only, reading nothing at all. + */ + private static ValueGraph.Node describe( + ProbeRing ring, ProbeThreadState state, ValueGraph graph, Object value, String label) { + if (value != null && ProbeConfig.ARGS_TYPE.equals(ring.config.argMode)) { + return ValueGraph.opaque(label, value); + } + if (state.inProbe) { + ring.suppress(); + return ValueGraph.marker(label, "(not rendered: already inside a probe)"); + } + state.inProbe = true; + try { + return graph.root(value, label, null); + } catch (Throwable t) { + // The walk catches what a field read can throw; this is for anything + // else — an OutOfMemoryError building a node — so the method still runs. + return ValueGraph.marker(label, "(not rendered: " + t.getClass().getName() + ")"); + } finally { + state.inProbe = false; + } + } + + /** + * The exception as its class and message, read without calling a subclass's getMessage(): that is + * application's code, on application's thread. + */ + private static String describeThrowable(ProbeRing ring, ProbeThreadState state, Throwable error) { + state.truncated = false; + if (error == null) { + return null; + } + return truncate(state, InertTypes.throwableText(error), ring.config.argChars); + } + + /** + * Cuts at the cap and says how much it cut, so a truncation is never silent. Marks {@code state} + * when it cuts, so the event can say it was truncated without the reader having to spot the + * marker in the text. + */ + private static String truncate(ProbeThreadState state, String value, int max) { + if (value == null || value.length() <= max) { + return value; + } + state.truncated = true; + return value.substring(0, max) + "…(+" + (value.length() - max) + " chars)"; + } + + private static String describeOpenKeys() { + StringBuilder open = new StringBuilder(); + for (String key : PROBES.keySet()) { + if (open.length() > 0) { + open.append(", "); + } + open.append(key); + } + return open.toString(); + } + + private static synchronized void rememberClosed(String key, Map end) { + CLOSED.put(key, end); + CLOSED_ORDER.add(key); + while (CLOSED_ORDER.size() > CLOSED_RETAINED) { + CLOSED.remove(CLOSED_ORDER.remove(0)); + } + } + + // ---------------------------------------------------------------- leases + + /** Extends a patch's lease. */ + public static void lease(String key, long millis) { + LEASES.put(key, System.currentTimeMillis() + millis); + } + + /** + * Starts the daemon that restores patches whose lease has lapsed, once. + * + *

This is what makes a killed client safe. The span that installed a patch renews its lease + * while it runs; if the process holding it dies — a killed CLI, a dropped exec channel, a Ctrl-C + * — nothing renews, and the JVM puts the original bytes back on its own instead of serving + * patched code indefinitely. + */ + public static synchronized void startReaper() { + if (reaper != null && reaper.isAlive()) { + return; + } + Thread thread = + new Thread( + new Runnable() { + @Override + public void run() { + while (true) { + try { + Thread.sleep(5000L); + reapExpired(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } catch (Throwable ignored) { + // Keep reaping: one class that cannot be restored must + // not strand every other lease behind it. + } + } + } + }, + "telltale-diag-lease-reaper"); + thread.setDaemon(true); + thread.start(); + reaper = thread; + } + + /** Restores every patch whose deadline has passed. Visible for testing. */ + public static void reapExpired() { + Restorer restorer = RESTORER; + if (restorer == null) { + return; + } + long now = System.currentTimeMillis(); + for (Map.Entry entry : + new ArrayList>(LEASES.entrySet())) { + if (entry.getValue() == null || entry.getValue() > now) { + continue; + } + // Drop the lease first: a restore that fails must not be retried + // every five seconds for the life of the JVM. + LEASES.remove(entry.getKey()); + // Free the buffers before the restore, so a restore that fails still + // releases the memory. The class is coming back either way, and a + // probe on un-instrumented code would never record again regardless. + closeProbesOwnedBy(entry.getKey()); + try { + restorer.restore(entry.getKey()); + } catch (Throwable ignored) { + // The class may have gone with its classloader (a webapp + // redeploy), which makes the stale lease harmless. + } + } + } + + public static void agentmain(String args, Instrumentation instrumentation) { + INSTRUMENTATION = instrumentation; + } + + public static void premain(String args, Instrumentation instrumentation) { + INSTRUMENTATION = instrumentation; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeAllocation.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeAllocation.java new file mode 100644 index 0000000..85c1509 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeAllocation.java @@ -0,0 +1,80 @@ +package com.flanksource.telltale.diag.agent; + +import java.lang.management.ManagementFactory; + +/** + * The calling thread's allocation counter, for a probe opened with {@code alloc: true}. + * + *

It reads HotSpot's per-thread allocated-bytes counter through {@code + * com.sun.management.ThreadMXBean}: everything the thread allocated, TLAB and outside it, since it + * started. A frame's measurement is the difference between two readings, so it counts what the + * thread allocated between entry and exit — callees included, other threads excluded, and never + * what is still reachable afterwards. + * + *

The {@code com.sun.management} type is confined to this class on purpose. {@link #resolve} + * runs on the servlet thread that opens the probe, so a JVM without the extension fails there, as a + * refusal the operator reads, and never as a linkage error inside an instrumented method. + */ +final class ProbeAllocation { + + private final com.sun.management.ThreadMXBean threads; + + private ProbeAllocation(com.sun.management.ThreadMXBean threads) { + this.threads = threads; + } + + /** + * The counter, checked usable before any probe depends on it. + * + * @throws IllegalStateException naming why this JVM cannot measure per-thread allocation. It is + * never enabled on the caller's behalf: that switch is JVM-global, and a diagnostic must not + * change the JVM it observes. + */ + static ProbeAllocation resolve() { + java.lang.management.ThreadMXBean bean = ManagementFactory.getThreadMXBean(); + if (!(bean instanceof com.sun.management.ThreadMXBean)) { + throw new IllegalStateException( + "this JVM's ThreadMXBean (" + + bean.getClass().getName() + + ") is not a com.sun.management.ThreadMXBean, so it cannot report per-thread" + + " allocation"); + } + com.sun.management.ThreadMXBean threads = (com.sun.management.ThreadMXBean) bean; + if (!threads.isThreadAllocatedMemorySupported()) { + throw new IllegalStateException( + "this JVM does not support per-thread allocation measurement"); + } + if (!threads.isThreadAllocatedMemoryEnabled()) { + throw new IllegalStateException( + "per-thread allocation measurement is disabled on this JVM" + + " (ThreadMXBean.setThreadAllocatedMemoryEnabled(false))"); + } + ProbeAllocation allocation = new ProbeAllocation(threads); + if (allocation.currentThreadBytes() < 0L) { + throw new IllegalStateException( + "this JVM reported no allocation counter for the current thread"); + } + return allocation; + } + + /** + * Bytes the calling thread has allocated so far, or -1 when the JVM cannot say — measurement + * switched off after the probe opened, or the read failed. A frame with a -1 at either end + * reports no allocation rather than a wrong one, and never throws into the method it observes. + */ + long currentThreadBytes() { + try { + return threads.getThreadAllocatedBytes(Thread.currentThread().getId()); + } catch (Throwable unreadable) { + return -1L; + } + } + + /** + * The difference between two readings of a monotonic counter — allocated bytes here, collection + * counts and milliseconds for {@link ProbeGc} — or -1 when either reading is unknown. + */ + static long delta(long start, long end) { + return start < 0L || end < 0L ? -1L : end - start; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeConfig.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeConfig.java new file mode 100644 index 0000000..5fbceff --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeConfig.java @@ -0,0 +1,309 @@ +package com.flanksource.telltale.diag.agent; + +import com.flanksource.telltale.diag.graph.GraphLimits; +import java.util.Map; + +/** + * The bounds one probe records under, resolved once when the probe is opened. + * + *

Resolved once, and on the servlet thread that opened the probe, for two reasons. + * {@code System.getProperty} takes the {@code Properties} monitor and allocates, which has no + * business on a path an instrumented method runs; and a {@code SecurityManager} can deny it, which + * is reportable on a request thread and an {@code ExceptionInInitializerError} in application's + * call path anywhere else. Everything the hot path needs is a final field here. + * + *

Every bound takes a JVM-wide default from a system property and can be overridden per probe. A + * probe on a method taking a large DTO wants tighter caps than the JVM default, and a restart-only + * knob would be no use at all. + */ +final class ProbeConfig { + + /** + * Values are snapshotted as object graphs — the default. Fields are read, not getters called, and + * only JDK value types are turned into text, so the snapshot runs none of the application's code; + * see {@code ValueGraph}. + */ + static final String ARGS_VALUE = "value"; + + /** Only the type and identity hash, reading nothing from the value. */ + static final String ARGS_TYPE = "type"; + + /** Arguments and return values are not recorded at all. */ + static final String ARGS_NONE = "none"; + + private static final String PREFIX = "telltale.diag.probe."; + + /** Pre-v2 name for the ring capacity, still honoured. */ + private static final String LEGACY_LIMIT = "telltale.diag.probe-limit"; + + final int ringCapacity; + + /** + * Events one probe records before it stops recording, arthas' {@code -n}. + * + *

A capture is a sample, not a tap. Left running, an instrumented method on a hot path pays + * the probe cost on every call for the life of its lease and produces a ring holding the last few + * seconds of a run nobody is reading — the useful part having scrolled past long before. Stopping + * at a budget is what makes "instrument this and see" a safe thing to do on a busy JVM. + * + *

Distinct from {@link #ringCapacity}, which is how much is KEPT. This is how much is ever + * WRITTEN, and reaching it ends the recording rather than evicting anything. + */ + final int maxEvents; + + final int maxKeys; + final int maxArgs; + final int maxDepth; + final String argMode; + final boolean enabled; + final long budgetBytes; + + /** + * How far a value snapshot may go: {@code expand} levels, {@code children} per container, and — + * across one whole event, all of its arguments together — {@code nodes} and {@code event-chars} + * characters, each string capped at {@code arg-chars} from its head and tail. + * + *

Generous by default, because a capture that cannot be read is a capture that was paid for + * and wasted: 512 characters cut the preservation run's query builder off before its {@code NOT + * IN}, which was the one part that changed between calls. + */ + final GraphLimits graph; + + /** Per-string cap, also applied to an exception's text. */ + final int argChars; + + /** + * The caller means to record into a key that is already open. + * + *

Per-open intent, never a system property: a JVM-wide default would turn off the one check + * that tells "this span is arming its second method" apart from "two spans landed on one key", + * and the second silently merges someone else's calls into a capture. A span arming a set of + * methods sets it; every other caller leaves it false and a repeat open is refused. + */ + final boolean join; + + /** + * The caller asked for {@code alloc}: each frame's exit carries the bytes its thread allocated + * since the entry. Per-open intent, never a system property — it is a call into the JVM on every + * entry and exit of the observed method. + */ + final boolean alloc; + + /** The counter {@link #alloc} reads, or null when it was not asked for or is unusable. */ + final ProbeAllocation allocation; + + /** Why {@link #alloc} was asked for and cannot be honoured; the open is refused with it. */ + final String allocError; + + /** + * The caller asked for {@code gc}: each frame's exit carries the collections that ran JVM-wide + * while it was open, and every read the span's total. + */ + final boolean gc; + + /** The counters {@link #gc} reads, or null when it was not asked for or is unusable. */ + final ProbeGc gcCounters; + + /** Why {@link #gc} was asked for and cannot be honoured; the open is refused with it. */ + final String gcError; + + /** + * Which root calls are published ({@code minCost}, {@code exceptionsOnly}, {@code match}), or + * null when every call is. Per-open intent, never a system property. + */ + final ProbeFilter filter; + + /** Why the filter asked for cannot be evaluated; the open is refused with it. */ + final String filterError; + + private ProbeConfig( + int ringCapacity, + int maxEvents, + int maxKeys, + int maxArgs, + int maxDepth, + String argMode, + boolean enabled, + long budgetBytes, + boolean join, + GraphLimits graph, + int argChars, + boolean alloc, + boolean gc, + Map options) { + this.ringCapacity = ringCapacity; + this.maxEvents = maxEvents; + this.maxKeys = maxKeys; + this.maxArgs = maxArgs; + this.maxDepth = maxDepth; + this.argMode = argMode; + this.enabled = enabled; + this.budgetBytes = budgetBytes; + this.join = join; + this.graph = graph; + this.argChars = argChars; + this.alloc = alloc; + ProbeAllocation resolved = null; + String error = null; + if (alloc) { + try { + resolved = ProbeAllocation.resolve(); + } catch (RuntimeException | LinkageError unusable) { + error = reason(unusable); + } + } + this.allocation = resolved; + this.allocError = error; + this.gc = gc; + ProbeGc counters = null; + String gcFailure = null; + if (gc) { + try { + counters = ProbeGc.resolve(); + } catch (RuntimeException | LinkageError unusable) { + gcFailure = reason(unusable); + } + } + this.gcCounters = counters; + this.gcError = gcFailure; + ProbeFilter resolvedFilter = null; + String filterFailure = null; + try { + resolvedFilter = ProbeFilter.resolve(options); + } catch (IllegalArgumentException invalid) { + filterFailure = reason(invalid); + } + this.filter = resolvedFilter; + this.filterError = filterFailure; + } + + private static String reason(Throwable unusable) { + return unusable.getMessage() != null ? unusable.getMessage() : unusable.getClass().getName(); + } + + /** + * Resolves the JVM defaults, then applies whatever the caller overrode. + * + *

{@code options} may be null; unknown entries are ignored rather than rejected, so an older + * CLI talking to a newer agent still opens a probe. + */ + static ProbeConfig resolve(Map options) { + int capacity = + ProbeRing.roundUpToPowerOfTwo(positive(pick(options, "ring", legacyCapacity()), 4096)); + int argChars = positive(pick(options, "arg-chars", prefixed("arg-chars", 16384)), 16384); + GraphLimits graph = + GraphLimits.probe( + positive(pick(options, "expand", prefixed("expand", 3)), 3), + positive(pick(options, "children", prefixed("children", 100)), 100), + positive(pick(options, "nodes", prefixed("nodes", 2000)), 2000), + argChars, + positive(pick(options, "event-chars", prefixed("event-chars", 262144)), 262144)); + return new ProbeConfig( + capacity, + positive(pick(options, "events", prefixed("events", 1000)), 1000), + positive(pick(options, "keys", prefixed("keys", 8)), 8), + positive(pick(options, "max-args", prefixed("max-args", 16)), 16), + positive(pick(options, "depth", prefixed("depth", 256)), 256), + argMode(options), + boolOption(options, "enabled", boolProperty("enabled", true)), + positive( + pick(options, "budget-bytes", prefixed("budget-bytes", 256 * 1024 * 1024)), + 256 * 1024 * 1024), + boolOption(options, "join", false), + graph, + argChars, + boolOption(options, "alloc", false), + boolOption(options, "gc", false), + options); + } + + /** + * The ring capacity's JVM default, honouring the pre-v2 property name. + * + *

{@code telltale.diag.probe-limit} bounded a list that dropped the newest frame; it now sizes + * a ring that drops the oldest. The number means the same thing — how much is kept — so an + * operator who set it keeps getting what they asked for rather than silently reverting to the + * default. + */ + private static int legacyCapacity() { + int current = intProperty(PREFIX + "ring", -1); + if (current > 0) { + return current; + } + return intProperty(LEGACY_LIMIT, 4096); + } + + private static String argMode(Map options) { + Object raw = options == null ? null : options.get("arg-mode"); + String mode = raw == null ? System.getProperty(PREFIX + "arg-mode") : String.valueOf(raw); + if (ARGS_TYPE.equalsIgnoreCase(mode)) { + return ARGS_TYPE; + } + if (ARGS_NONE.equalsIgnoreCase(mode)) { + return ARGS_NONE; + } + // Anything unrecognised means "render values", the documented default. A + // typo must not silently disable capture — that reads as a quiet method. + return ARGS_VALUE; + } + + private static int pick(Map options, String name, int fallback) { + Object raw = options == null ? null : options.get(name); + if (raw instanceof Number) { + return ((Number) raw).intValue(); + } + if (raw != null) { + try { + return Integer.parseInt(String.valueOf(raw).trim()); + } catch (RuntimeException ignored) { + return fallback; + } + } + return fallback; + } + + private static boolean boolOption(Map options, String name, boolean fallback) { + Object raw = options == null ? null : options.get(name); + if (raw instanceof Boolean) { + return ((Boolean) raw).booleanValue(); + } + if (raw != null) { + return Boolean.parseBoolean(String.valueOf(raw).trim()); + } + return fallback; + } + + /** Reads {@code telltale.diag.probe.}. */ + private static int prefixed(String name, int fallback) { + return intProperty(PREFIX + name, fallback); + } + + private static int intProperty(String fullName, int fallback) { + try { + String configured = System.getProperty(fullName); + if (configured != null) { + return Integer.parseInt(configured.trim()); + } + } catch (RuntimeException ignored) { + // A malformed override falls back to the default rather than + // disabling the bound, which is the failure that costs a heap. + } + return fallback; + } + + private static boolean boolProperty(String name, boolean fallback) { + try { + String configured = System.getProperty(PREFIX + name); + if (configured != null) { + return Boolean.parseBoolean(configured.trim()); + } + } catch (RuntimeException ignored) { + // As above: an unreadable property must not turn the probe off. + } + return fallback; + } + + private static int positive(int value, int fallback) { + return value > 0 ? value : fallback; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeEvent.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeEvent.java new file mode 100644 index 0000000..55e3818 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeEvent.java @@ -0,0 +1,367 @@ +package com.flanksource.telltale.diag.agent; + +import com.flanksource.telltale.diag.graph.ValueGraph; + +/** + * One thing an instrumented method did: it was entered, it returned a value, it threw, or it + * finished. + * + *

Immutable on purpose: events are published across threads through {@link ProbeRing} with no + * lock, and the only thing that makes that safe is that nothing mutates after publication. The + * value snapshots are trees, but the walk finishes them before the event is built and nothing + * touches them afterwards. + * + *

This class deliberately holds no static state. Its first use is inside an + * instrumented method, so its {@code } runs on an application thread — where a failure + * would be an {@code ExceptionInInitializerError} in application's own call path, permanent for the + * life of the JVM. Rendering lives in {@link ProbeJson}, which is only ever touched by a drain on a + * servlet thread. + */ +final class ProbeEvent { + + static final String ENTER = "enter"; + static final String RETURN = "return"; + static final String THROW = "throw"; + static final String EXIT = "exit"; + + /** + * Position in this ring's sequence. + * + *

The one non-final field, written exactly once by {@link ProbeRing#add} immediately before + * the {@code lazySet} that publishes the reference. That ordering is what makes it safe without a + * volatile write: {@code lazySet} carries a StoreStore barrier, so this store lands before the + * reference does, and the reader's {@code AtomicReferenceArray.get()} is a volatile read. It is + * never written again, and never written by a reader. + */ + long seq; + + /** Epoch nanos, anchored per ring — see {@link ProbeRing#now()}. */ + final long ts; + + final long threadId; + final String thread; + + /** One of {@link #ENTER}, {@link #RETURN}, {@link #THROW}, {@link #EXIT}. */ + final String kind; + + final String key; + + /** Call depth on this thread, 0 for an outermost instrumented call. */ + final int depth; + + /** Identifies the invocation; unique per thread, so the key is (threadId, frame). */ + final long frame; + + /** The enclosing invocation's frame, or 0 when there is none or it is unknown. */ + final long parent; + + /** Set on {@link #ENTER} only; the other kinds are tied to it by {@code frame}. */ + final String className; + + final String method; + + /** + * The raw JVM descriptor of the method entered, e.g. {@code (I)Ljava/lang/String;}. + * + *

Carried per event rather than looked up from whatever armed the probe, because one probe key + * may cover a SET of methods: {@code method} alone cannot say which overload ran, and anything + * keyed on the name alone — a rollup, a call tree — would merge overloads whose costs are the + * reason they were traced separately. + * + *

Raw, not reduced to the simple names a person writes. This is what goes into the {@code + * .jsonl}, which should hold what the JVM said; formatting is a read-side choice and it is not + * reversible once made. + */ + final String descriptor; + + /** + * For a call site, the line in the CALLER's source the call was made from; 0 for a method's own + * entry, whose caller is not in view, and for a class compiled without line numbers. Set on + * {@link #ENTER} only. + */ + final int line; + + /** Snapshotted arguments, or null when the caller did not ask for them. */ + final ValueGraph.Node[] args; + + /** Snapshotted return value, {@link #RETURN} only. */ + final ValueGraph.Node value; + + /** + * What the snapshots above retain, as the walk estimated it. Carried rather than recomputed, + * because {@link #charge()} runs on every add and again on every eviction. + */ + final long valueBytes; + + /** Rendered throwable, {@link #THROW} only. */ + final String error; + + /** Elapsed time of the invocation, on the events that close a frame. */ + final long durationNs; + + /** + * Bytes the thread allocated between the frame's entry and its exit, on the events that close a + * frame of a probe opened with {@code alloc}; -1 when the probe did not ask, or the frame's entry + * reading is unknown (past the depth cap, or the counter could not be read). Never a 0 standing + * in for "unknown". + */ + final long allocBytes; + + /** + * Collections that ran JVM-wide while the frame was open, and their accumulated time in + * milliseconds, on the events that close a frame of a probe opened with {@code gc}; -1 when the + * probe did not ask or a counter is unknown. + */ + final long gcCount; + + final long gcPauseMs; + + /** + * How many frames this event unwound beyond its own — non-zero only when an inner exit was lost + * and the depth self-healed. Reported rather than hidden, because a silent repair reads as a + * depth that was never wrong. + */ + final int unwound; + + /** True when any rendered text in this event hit a length cap. */ + final boolean truncated; + + /** + * On the last event of a filtered call that outgrew its thread's buffer: how many of the call's + * events were left out between its head and its close. See {@link ProbePending}. + */ + final int omitted; + + private ProbeEvent( + long ts, + long threadId, + String thread, + String kind, + String key, + int depth, + long frame, + long parent, + String className, + String method, + String descriptor, + int line, + ValueGraph.Node[] args, + ValueGraph.Node value, + long valueBytes, + String error, + long durationNs, + long allocBytes, + long gcCount, + long gcPauseMs, + int unwound, + boolean truncated, + int omitted) { + this.ts = ts; + this.threadId = threadId; + this.thread = thread; + this.kind = kind; + this.key = key; + this.depth = depth; + this.frame = frame; + this.parent = parent; + this.className = className; + this.method = method; + this.descriptor = descriptor; + this.line = line; + this.args = args; + this.value = value; + this.valueBytes = valueBytes; + this.error = error; + this.durationNs = durationNs; + this.allocBytes = allocBytes; + this.gcCount = gcCount; + this.gcPauseMs = gcPauseMs; + this.unwound = unwound; + this.truncated = truncated; + this.omitted = omitted; + } + + /** This event, carrying how many events of its call were omitted. */ + ProbeEvent omitting(int count) { + return new ProbeEvent( + ts, threadId, thread, kind, key, depth, frame, parent, className, method, descriptor, line, + args, value, valueBytes, error, durationNs, allocBytes, gcCount, gcPauseMs, unwound, + truncated, count); + } + + static ProbeEvent enter( + long ts, + long threadId, + String thread, + String key, + int depth, + long frame, + long parent, + String className, + String method, + String descriptor, + int line, + ValueGraph.Node[] args, + long valueBytes, + boolean truncated) { + return new ProbeEvent( + ts, + threadId, + thread, + ENTER, + key, + depth, + frame, + parent, + className, + method, + descriptor, + line, + args, + null, + valueBytes, + null, + 0L, + -1L, + -1L, + -1L, + 0, + truncated, + 0); + } + + static ProbeEvent returned( + long ts, + long threadId, + String thread, + String key, + int depth, + long frame, + ValueGraph.Node value, + long valueBytes, + boolean truncated) { + return new ProbeEvent( + ts, + threadId, + thread, + RETURN, + key, + depth, + frame, + 0L, + null, + null, + null, + 0, + null, + value, + valueBytes, + null, + 0L, + -1L, + -1L, + -1L, + 0, + truncated, + 0); + } + + static ProbeEvent thrown( + long ts, + long threadId, + String thread, + String key, + int depth, + long frame, + String error, + long durationNs, + long allocBytes, + long gcCount, + long gcPauseMs, + int unwound, + boolean truncated) { + return new ProbeEvent( + ts, + threadId, + thread, + THROW, + key, + depth, + frame, + 0L, + null, + null, + null, + 0, + null, + null, + 0L, + error, + durationNs, + allocBytes, + gcCount, + gcPauseMs, + unwound, + truncated, + 0); + } + + static ProbeEvent exited( + long ts, + long threadId, + String thread, + String key, + int depth, + long frame, + long durationNs, + long allocBytes, + long gcCount, + long gcPauseMs, + int unwound) { + return new ProbeEvent( + ts, + threadId, + thread, + EXIT, + key, + depth, + frame, + 0L, + null, + null, + null, + 0, + null, + null, + 0L, + null, + durationNs, + allocBytes, + gcCount, + gcPauseMs, + unwound, + false, + 0); + } + + /** + * Rough retained size, used to charge this event against the global budget. + * + *

An estimate rather than a measurement: the point is a bound that moves with what is actually + * held, not an exact heap figure. Strings are charged at two bytes per char because they are + * UTF-16 on the Java 8 target, and the value snapshots at what their walk estimated. + */ + long charge() { + long bytes = 128L + valueBytes; + bytes += chars(thread) + chars(key) + chars(className) + chars(method) + chars(descriptor); + bytes += chars(error); + if (args != null) { + bytes += 16L * args.length; + } + return bytes; + } + + private static int chars(String s) { + return s == null ? 0 : 2 * s.length(); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeFilter.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeFilter.java new file mode 100644 index 0000000..34ad3b9 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeFilter.java @@ -0,0 +1,330 @@ +package com.flanksource.telltale.diag.agent; + +import com.flanksource.telltale.diag.graph.ValueGraph; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +/** + * Which ROOT calls a probe publishes: {@code minCost}, {@code exceptionsOnly} and {@code match}. + * + *

A root call is the outermost invocation of the probe on its thread; everything it records + * while it runs — nested targets and call sites — is held on that thread and published or dropped + * with it, so a kept call arrives whole and a dropped one leaves nothing behind but the {@code + * filtered} count. + * + *

Every predicate is resolved and checked when the probe opens, on a servlet thread. A predicate + * the probe could not evaluate — a match on arguments the method is not recording, a regex that + * does not compile, a duration that does not parse — refuses the open: a filter that quietly passed + * everything, or nothing, would read as a quiet method. + * + *

Immutable, and holds no static state: it is consulted on application threads. + */ +final class ProbeFilter { + + static final String ON_ARG = "arg"; + static final String ON_RETURN = "return"; + + final long minCostNanos; + final boolean exceptionsOnly; + private final Predicate[] argPredicates; + private final Predicate[] returnPredicates; + + private ProbeFilter( + long minCostNanos, boolean exceptionsOnly, List args, List returns) { + this.minCostNanos = minCostNanos; + this.exceptionsOnly = exceptionsOnly; + this.argPredicates = args.toArray(new Predicate[0]); + this.returnPredicates = returns.toArray(new Predicate[0]); + } + + /** + * The filter {@code options} ask for, or null when they ask for none. + * + * @throws IllegalArgumentException naming the option that cannot be honoured + */ + static ProbeFilter resolve(Map options) { + if (options == null) { + return null; + } + Object minCost = options.get("minCost"); + Object exceptionsOnly = options.get("exceptionsOnly"); + Object match = options.get("match"); + if (minCost == null && exceptionsOnly == null && match == null) { + return null; + } + long nanos = minCost == null ? 0L : parseDuration(String.valueOf(minCost)); + if (nanos < 0L) { + throw new IllegalArgumentException("minCost must not be negative, got " + minCost); + } + if (exceptionsOnly != null && !(exceptionsOnly instanceof Boolean)) { + throw new IllegalArgumentException("exceptionsOnly must be a boolean, got " + exceptionsOnly); + } + List args = new ArrayList(); + List returns = new ArrayList(); + for (Predicate predicate : predicates(match)) { + requireRecorded(options, predicate); + (ON_ARG.equals(predicate.on) ? args : returns).add(predicate); + } + return new ProbeFilter(nanos, Boolean.TRUE.equals(exceptionsOnly), args, returns); + } + + private static List predicates(Object match) { + List predicates = new ArrayList(); + if (match == null) { + return predicates; + } + if (!(match instanceof List)) { + throw new IllegalArgumentException("match must be a list of predicates, got " + match); + } + for (Object raw : (List) match) { + if (!(raw instanceof Map)) { + throw new IllegalArgumentException("each match predicate must be an object, got " + raw); + } + predicates.add(Predicate.parse((Map) raw)); + } + return predicates; + } + + /** A predicate on a value the probe does not record could never be true. */ + private static void requireRecorded(Map options, Predicate predicate) { + String recorded = ON_ARG.equals(predicate.on) ? "args" : "return"; + if (!Boolean.TRUE.equals(options.get(recorded))) { + throw new IllegalArgumentException( + "match on " + + predicate.on + + " needs " + + recorded + + ":true, or the probe never renders the value it would test"); + } + Object mode = options.get("arg-mode"); + if (mode != null && !ProbeConfig.ARGS_VALUE.equalsIgnoreCase(String.valueOf(mode))) { + throw new IllegalArgumentException( + "match needs arg-mode value; arg-mode " + mode + " renders no value to test"); + } + } + + boolean hasArgPredicates() { + return argPredicates.length > 0; + } + + boolean hasReturnPredicates() { + return returnPredicates.length > 0; + } + + /** Every argument predicate holds on the root call's rendered arguments. */ + boolean matchesArgs(ValueGraph.Node[] args) { + for (Predicate predicate : argPredicates) { + ValueGraph.Node root = + args == null || predicate.index >= args.length ? null : args[predicate.index]; + if (!predicate.test(root)) { + return false; + } + } + return true; + } + + /** Every return predicate holds on the root call's rendered return value. */ + boolean matchesReturn(ValueGraph.Node value) { + for (Predicate predicate : returnPredicates) { + if (!predicate.test(value)) { + return false; + } + } + return true; + } + + /** + * Whether a finished root call is published. + * + * @param returnMatched what {@link #matchesReturn} said when it returned; false when it threw + */ + boolean keeps(long elapsedNanos, boolean threw, boolean returnMatched) { + if (elapsedNanos < minCostNanos || (exceptionsOnly && !threw)) { + return false; + } + return !hasReturnPredicates() || (returnMatched && !threw); + } + + /** A canonical description, so a join that asks for a different filter is refused. */ + String describe() { + StringBuilder out = + new StringBuilder("minCost=").append(minCostNanos).append("ns exceptionsOnly="); + out.append(exceptionsOnly).append(" match=["); + for (Predicate predicate : argPredicates) { + out.append(predicate).append(' '); + } + for (Predicate predicate : returnPredicates) { + out.append(predicate).append(' '); + } + return out.append(']').toString(); + } + + /** + * Parses a Go {@code time.Duration} string — {@code 250ms}, {@code 1.5s}, {@code 1m30s} — into + * nanoseconds. The CLI and the webapp send what a person typed, and Go is what validated it. + */ + static long parseDuration(String text) { + String value = text == null ? "" : text.trim(); + if ("0".equals(value)) { + return 0L; + } + if (value.isEmpty()) { + throw new IllegalArgumentException("minCost must be a duration such as 250ms, got \"\""); + } + double total = 0; + int i = 0; + while (i < value.length()) { + int start = i; + while (i < value.length() && (Character.isDigit(value.charAt(i)) || value.charAt(i) == '.')) { + i++; + } + int unitStart = i; + while (i < value.length() && !Character.isDigit(value.charAt(i)) && value.charAt(i) != '.') { + i++; + } + if (start == unitStart || unitStart == i) { + throw new IllegalArgumentException( + "minCost must be a duration such as 250ms or 1m30s, got \"" + text + "\""); + } + total += + Double.parseDouble(value.substring(start, unitStart)) + * unitNanos(value.substring(unitStart, i), text); + } + return (long) total; + } + + private static long unitNanos(String unit, String text) { + if ("ns".equals(unit)) return 1L; + if ("us".equals(unit) || "µs".equals(unit)) return 1_000L; + if ("ms".equals(unit)) return 1_000_000L; + if ("s".equals(unit)) return 1_000_000_000L; + if ("m".equals(unit)) return 60_000_000_000L; + if ("h".equals(unit)) return 3_600_000_000_000L; + throw new IllegalArgumentException( + "minCost has unknown unit \"" + unit + "\" in \"" + text + "\"; use ns, us, ms, s, m or h"); + } + + /** One {@code {on, index, path, op, value}} test on a rendered value. */ + private static final class Predicate { + final String on; + final int index; + final String[] path; + final String op; + final String value; + final Pattern pattern; + + private Predicate(String on, int index, String[] path, String op, String value) { + this.on = on; + this.index = index; + this.path = path; + this.op = op; + this.value = value; + this.pattern = "regex".equals(op) ? compile(value) : null; + } + + static Predicate parse(Map raw) { + String on = string(raw, "on"); + if (!ON_ARG.equals(on) && !ON_RETURN.equals(on)) { + throw new IllegalArgumentException("match.on must be arg or return, got " + on); + } + String op = string(raw, "op"); + if (!"equals".equals(op) && !"contains".equals(op) && !"regex".equals(op)) { + throw new IllegalArgumentException( + "match.op must be equals, contains or regex, got " + op); + } + Object value = raw.get("value"); + if (!(value instanceof String)) { + throw new IllegalArgumentException("match.value must be a string, got " + value); + } + int index = index(raw.get("index"), on); + Object path = raw.get("path"); + String[] segments = + path == null || String.valueOf(path).isEmpty() + ? new String[0] + : String.valueOf(path).split("\\."); + return new Predicate(on, index, segments, op, (String) value); + } + + private static int index(Object raw, String on) { + if (raw == null) { + if (ON_ARG.equals(on)) { + throw new IllegalArgumentException("match on arg needs index, the argument position"); + } + return 0; + } + if (!(raw instanceof Number) || ((Number) raw).intValue() < 0) { + throw new IllegalArgumentException("match.index must be a non-negative integer, got " + raw); + } + int index = ((Number) raw).intValue(); + if (ON_RETURN.equals(on) && index != 0) { + throw new IllegalArgumentException("match on return takes no index, got " + raw); + } + return index; + } + + private static String string(Map raw, String name) { + Object value = raw.get(name); + return value == null ? null : String.valueOf(value); + } + + private static Pattern compile(String regex) { + try { + return Pattern.compile(regex); + } catch (PatternSyntaxException e) { + throw new IllegalArgumentException( + "match regex " + regex + " does not compile: " + e.getDescription(), e); + } + } + + /** False when the path names nothing the snapshot holds: the value is not there to match. */ + boolean test(ValueGraph.Node root) { + String text = text(walk(root)); + if (text == null) { + return false; + } + if ("equals".equals(op)) { + return text.equals(value); + } + if ("contains".equals(op)) { + return text.contains(value); + } + return pattern.matcher(text).find(); + } + + private ValueGraph.Node walk(ValueGraph.Node root) { + ValueGraph.Node node = root; + for (String segment : path) { + node = node == null ? null : child(node, segment); + } + return node; + } + + private static ValueGraph.Node child(ValueGraph.Node node, String label) { + if (node.children == null) { + return null; + } + for (ValueGraph.Node child : node.children) { + if (label.equals(child.label)) { + return child; + } + } + return null; + } + + /** What a node rendered as: its value text, else its identity ({@code raw}). */ + private static String text(ValueGraph.Node node) { + if (node == null) { + return null; + } + return node.value != null ? node.value : node.raw; + } + + @Override + public String toString() { + return on + "[" + index + "]" + String.join(".", path) + " " + op + " " + value; + } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeGc.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeGc.java new file mode 100644 index 0000000..1e62d08 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeGc.java @@ -0,0 +1,118 @@ +package com.flanksource.telltale.diag.agent; + +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * The JVM's garbage-collection counters, for a probe opened with {@code gc: true}. + * + *

Sums {@code getCollectionCount()} and {@code getCollectionTime()} over every + * GarbageCollectorMXBean. These are JVM-wide, not per thread: a frame's delta is the collections + * that ran anywhere in the JVM while it was open, and so the pause its wall time included. A + * stop-the-world pause stalls every thread; a concurrent collector's time also counts phases that + * ran beside the frame, which makes the figure an upper bound for such collectors. + * + *

One instance per probe opening. It also holds the counters at that opening, so a read can + * report what the whole span collected, frame or no frame. The bean list is resolved once, on the + * servlet thread that opens the probe. + */ +final class ProbeGc { + + private final GarbageCollectorMXBean[] collectors; + private final long openCount; + private final long openMillis; + + /** The counters when recording stopped, so a later read is not still counting; -1 until then. */ + private long endCount = -1L; + + private long endMillis = -1L; + + private ProbeGc(GarbageCollectorMXBean[] collectors) { + this.collectors = collectors; + this.openCount = collections(); + this.openMillis = collectionMillis(); + } + + /** + * @throws IllegalStateException when this JVM reports no collectors, or a collector that cannot + * report its counters — either would make every frame report nothing. + */ + static ProbeGc resolve() { + List beans = ManagementFactory.getGarbageCollectorMXBeans(); + if (beans == null || beans.isEmpty()) { + throw new IllegalStateException( + "this JVM reports no GarbageCollectorMXBean, so there are no" + + " collection counters to read"); + } + ProbeGc gc = new ProbeGc(beans.toArray(new GarbageCollectorMXBean[0])); + if (gc.openCount < 0L || gc.openMillis < 0L) { + throw new IllegalStateException( + "a GarbageCollectorMXBean on this JVM reports its collection count" + + " or time as undefined (-1)"); + } + return gc; + } + + /** Collections so far, summed over every collector, or -1 when any cannot say. */ + long collections() { + long total = 0L; + try { + for (GarbageCollectorMXBean collector : collectors) { + long count = collector.getCollectionCount(); + if (count < 0L) { + return -1L; + } + total += count; + } + } catch (Throwable unreadable) { + return -1L; + } + return total; + } + + /** Accumulated collection time in milliseconds, summed over every collector, or -1. */ + long collectionMillis() { + long total = 0L; + try { + for (GarbageCollectorMXBean collector : collectors) { + long millis = collector.getCollectionTime(); + if (millis < 0L) { + return -1L; + } + total += millis; + } + } catch (Throwable unreadable) { + return -1L; + } + return total; + } + + /** Fixes the span's figure at the moment recording stopped. Idempotent. */ + synchronized void stop() { + if (endCount >= 0L) { + return; + } + endCount = collections(); + endMillis = collectionMillis(); + } + + /** + * What the JVM collected since the probe opened — up to {@link #stop} once recording stopped — as + * {@code {count, pauseMs}}; null when a counter could not be read, never zeros standing in for + * unknown. + */ + synchronized Map sinceOpen() { + long count = endCount >= 0L ? endCount : collections(); + long millis = endCount >= 0L ? endMillis : collectionMillis(); + if (count < 0L || millis < 0L) { + return null; + } + Map span = new LinkedHashMap(); + span.put("count", Long.valueOf(count - openCount)); + span.put("pauseMs", Long.valueOf(millis - openMillis)); + return span; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeJson.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeJson.java new file mode 100644 index 0000000..bf79015 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeJson.java @@ -0,0 +1,165 @@ +package com.flanksource.telltale.diag.agent; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.flanksource.telltale.diag.graph.ValueGraph; +import java.io.IOException; +import java.io.StringWriter; + +/** + * Renders one {@link ProbeEvent} as a line of JSON. + * + *

The JVM renders its own events so that the line travelling out is the line stored: the WAR + * forwards it opaquely and the CLI appends it to a {@code .jsonl} byte for byte. That is what makes + * replaying a saved capture a replay of what the JVM emitted rather than a re-render of a Go + * struct's idea of it, and it removes three format conversions from the path. + * + *

Rendering happens at DRAIN, on a servlet thread — never at probe time. Two consequences + * follow, and both are the point: JSON escaping never runs inside application's call path, and this + * class's {@code } (which builds a {@link JsonFactory}) never runs on an application + * thread, where a failure would be an {@code ExceptionInInitializerError} in code that never asked + * to be observed. + * + *

Jackson is relocated into this jar under {@code com.flanksource.telltale.diag.agent.shaded} — + * see the {@code agentJar} task. The agent sits on the system class path, so an unrelocated copy + * here would shadow application's own Jackson for every webapp in the JVM. + */ +final class ProbeJson { + + private static final JsonFactory FACTORY = new JsonFactory(); + + private ProbeJson() {} + + /** + * One event as one line, with absent fields omitted rather than nulled. + * + *

Never throws and never returns a malformed line. A consumer appends what comes back straight + * into a file every later reader parses line by line, so a half-written object would corrupt the + * archive for everyone; a line that says rendering failed keeps the file readable and the failure + * visible. + */ + static String line(ProbeEvent event) { + StringWriter out = new StringWriter(256); + try { + JsonGenerator json = FACTORY.createGenerator(out); + try { + write(json, event); + } finally { + json.close(); + } + } catch (IOException | RuntimeException e) { + return fallback(event, e); + } + return out.toString(); + } + + private static void write(JsonGenerator json, ProbeEvent event) throws IOException { + json.writeStartObject(); + json.writeNumberField("seq", event.seq); + json.writeNumberField("ts", event.ts); + json.writeStringField("kind", event.kind); + json.writeStringField("key", event.key); + json.writeNumberField("threadId", event.threadId); + if (event.thread != null) { + json.writeStringField("thread", event.thread); + } + json.writeNumberField("depth", event.depth); + json.writeNumberField("frame", event.frame); + if (event.parent != 0L) { + json.writeNumberField("parent", event.parent); + } + if (event.className != null) { + json.writeStringField("class", event.className); + } + if (event.method != null) { + json.writeStringField("method", event.method); + } + if (event.descriptor != null) { + json.writeStringField("descriptor", event.descriptor); + } + if (event.line != 0) { + json.writeNumberField("line", event.line); + } + if (event.args != null) { + json.writeArrayFieldStart("args"); + for (int i = 0; i < event.args.length; i++) { + node(json, event.args[i]); + } + json.writeEndArray(); + } + if (event.value != null) { + json.writeFieldName("value"); + node(json, event.value); + } + if (event.error != null) { + json.writeStringField("error", event.error); + } + if (event.durationNs != 0L) { + json.writeNumberField("durationNs", event.durationNs); + } + if (event.allocBytes >= 0L) { + // Written even when 0: a measured frame that allocated nothing is a + // fact, and an absent field means it was not measured. + json.writeNumberField("allocBytes", event.allocBytes); + } + if (event.gcCount >= 0L && event.gcPauseMs >= 0L) { + json.writeNumberField("gcCount", event.gcCount); + json.writeNumberField("gcPauseMs", event.gcPauseMs); + } + if (event.unwound != 0) { + json.writeNumberField("unwound", event.unwound); + } + if (event.truncated) { + json.writeBooleanField("truncated", true); + } + if (event.omitted > 0) { + json.writeNumberField("omitted", event.omitted); + } + json.writeEndObject(); + } + + /** + * One snapshot node, streamed field by field: the generator does the escaping here, on the drain + * thread, so none of it ever ran inside the method that was observed. A probe's nodes carry no id + * or path — the Go lowering assigns ids when it places the tree, and nothing can re-evaluate a + * probe's object. + */ + private static void node(JsonGenerator json, ValueGraph.Node node) throws IOException { + json.writeStartObject(); + if (node.label != null) { + json.writeStringField("label", node.label); + } + json.writeStringField("kind", node.kind); + if (node.type != null) { + json.writeStringField("type", node.type); + } + if (node.value != null) { + json.writeStringField("value", node.value); + } + if (node.raw != null) { + json.writeStringField("raw", node.raw); + } + if (node.children != null && !node.children.isEmpty()) { + json.writeArrayFieldStart("children"); + for (ValueGraph.Node child : node.children) { + node(json, child); + } + json.writeEndArray(); + } + json.writeEndObject(); + } + + /** + * A valid line standing in for one that could not be rendered, carrying enough to find the gap: + * which sequence it was and what went wrong. + */ + private static String fallback(ProbeEvent event, Throwable cause) { + StringBuilder line = new StringBuilder(128); + line.append("{\"seq\":") + .append(event.seq) + .append(",\"kind\":\"error\",\"error\":\"probe event could not be rendered: ") + .append(cause.getClass().getName()) + .append("\"}"); + return line.toString(); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbePending.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbePending.java new file mode 100644 index 0000000..cf07e89 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbePending.java @@ -0,0 +1,84 @@ +package com.flanksource.telltale.diag.agent; + +import java.util.ArrayList; +import java.util.List; + +/** + * One filtered probe's root call in progress on one thread: the events it has recorded so far, + * held until the call ends and its {@link ProbeFilter} decides whether they are published. + * + *

Bounded by the probe's own budgets — at most {@code events} events (and never more than the + * ring holds) and its share of {@code budget-bytes} — because what sits here is heap the observed + * JVM pays for. A call that outgrows the bound keeps its HEAD and its closing events and counts what + * it {@link #omitted}: the root's entry, its first nested calls, its return and its exit are what a + * reader of a slow or failing call needs, and dropping the whole call for being large would hide + * exactly the calls {@code minCost} exists to find. + * + *

Confined to its thread; nothing here is synchronized. + */ +final class ProbePending { + + final ProbeRing ring; + final long rootFrame; + + /** The thread's depth when the root was entered; a nested event is always deeper. */ + final int rootDepth; + + /** + * The root's arguments already failed a predicate. Nothing more is held for it; it is counted as + * {@code filtered} when it ends. + */ + final boolean rejected; + + boolean returnMatched; + int omitted; + + private final List events = new ArrayList(); + private final int maxEvents; + private final long maxBytes; + private long bytes; + + ProbePending(ProbeRing ring, long rootFrame, int rootDepth, boolean rejected) { + this.ring = ring; + this.rootFrame = rootFrame; + this.rootDepth = rootDepth; + this.rejected = rejected; + this.maxEvents = Math.min(ring.config.maxEvents, ring.capacity); + this.maxBytes = ring.budgetBytes(); + } + + /** Holds an event recorded inside the call, unless the bound is reached. */ + void hold(ProbeEvent event) { + if (rejected) { + return; + } + long charge = event.charge(); + if (events.size() >= maxEvents || bytes + charge > maxBytes) { + omitted++; + return; + } + events.add(event); + bytes += charge; + } + + /** Holds the root's own return or closing event, which is kept whatever the bound. */ + void close(ProbeEvent event) { + if (!rejected) { + events.add(event); + } + } + + /** Publishes the call, marking its last event with what was omitted from it. */ + void publish() { + int last = events.size() - 1; + for (int i = 0; i < last; i++) { + ring.add(events.get(i)); + } + if (last >= 0) { + ring.add(omitted > 0 ? events.get(last).omitting(omitted) : events.get(last)); + } + if (omitted > 0) { + ring.omitted(omitted); + } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeRing.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeRing.java new file mode 100644 index 0000000..ab8bd27 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeRing.java @@ -0,0 +1,367 @@ +package com.flanksource.telltale.diag.agent; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReferenceArray; + +/** + * A bounded, sequenced, drop-oldest buffer of one probe's events. + * + *

Writers are instrumented application methods, so the governing rule is that a write must never + * block, never allocate beyond the event itself, and never throw. The whole write is a {@code + * getAndIncrement} for the ticket and a {@code lazySet} to publish: + * + *

+ *     long s = seq.getAndIncrement();
+ *     event.seq = s;
+ *     slots.lazySet((int) (s & mask), event);
+ * 
+ * + *

Eviction is implicit — overwriting the slot is the eviction — which is why this drops + * the OLDEST rather than the newest. A span that outruns its reader keeps the most recent window + * instead of freezing on the first N events, and the reader is told exactly how much scrolled past. + * + *

Loss is derived, never counted. With {@code writeSeq} events written into {@code capacity} + * slots, everything below {@code writeSeq - capacity} is gone, so a reader holding cursor {@code c} + * missed {@code oldestSeq - c}. There is no drop counter to race, which is what makes the + * conservation law {@code observed + dropped == produced} exact under any interleaving. + * + *

Reads are wait-free and take no lock, so they never interact with a writer. The per-slot + * sequence check is what makes that safe: a slot holding some other sequence is either one that has + * been lapped (count it and move on) or one whose writer holds the ticket but has not published yet + * (stop there and come back). + */ +final class ProbeRing { + + /** + * Live bytes this ring is holding, used only to decide when to degrade. + * + *

Per ring rather than shared across all of them, which is what keeps this free of + * process-global state: the total is already bounded by the key cap times the per-ring budget, so + * a global counter would buy nothing and cost the one failure mode it is easy to miss — a ring + * dropped without being closed would keep its bytes charged against every later probe for the + * life of the JVM. + * + *

Approximate by construction: two writers wrapping onto one slot at the same instant can + * mis-charge each other. That is fine, because it steers a heuristic rather than enforcing a + * limit anything depends on being exact. + */ + private final AtomicLong charge = new AtomicLong(); + + private final AtomicReferenceArray slots; + private final AtomicLong seq = new AtomicLong(); + private final int mask; + + final int capacity; + final String key; + + /** + * Every class this ring records for, as {@code |} — the same key the + * byte stash and the leases use. + * + *

A SET rather than one owner because a span may instrument several classes into one ring. + * Restoring one of them must not close the ring while the others are still instrumented: they + * would keep running rewritten bytecode recording into nothing, and the span would go silent + * without a counter moving. The ring closes when the last owner goes. + * + *

Concurrent because openProbe and the lease reaper touch it from different threads, and + * iteration during a close must not throw. + */ + final Set owners = Collections.newSetFromMap(new ConcurrentHashMap()); + + /** + * What the WAR rewrote into each owner's class for this probe, one entry per arm, oldest first. + * + *

Opaque here: the agent neither reads nor validates it, it only keeps it alive exactly as + * long as the buffer it records into. That is the point. Instrumenting a class replaces all of + * its bytes at once, so a second span arming a class a first span already armed must rewrite BOTH + * — and to do that it has to be able to ask what the first one was. Holding the answer anywhere + * else lets the two disagree: in the WAR it would be lost on a redeploy, which leaves the classes + * patched, and the next arm would then silently disarm every span that survived. + */ + final List> armings = new CopyOnWriteArrayList>(); + + final ProbeConfig config; + + /** + * This ring's share of the JVM-wide budget. Dividing by the key cap is what makes the per-ring + * bound add up to a global one: however many probes are open, together they cannot hold more than + * the budget. + */ + private final long budgetBytes; + + /** + * Identifies this opening of the key. Probe keys carry a per-span random suffix so reuse is + * unlikely, but a close/open cycle would reset the sequence to zero and make a stale cursor look + * like a future one. A reader that passes back a generation it no longer matches is told the + * buffer reset rather than handed a batch that looks continuous. + */ + final long generation; + + /** Wall-clock anchor; see {@link #now()}. */ + private final long anchorEpochNanos; + + private final long anchorNanoTime; + + private volatile boolean active = true; + private volatile boolean complete; + private volatile boolean degraded; + private final AtomicInteger writers = new AtomicInteger(); + private final AtomicLong suppressed = new AtomicLong(); + private final AtomicLong filtered = new AtomicLong(); + private final AtomicLong omitted = new AtomicLong(); + + ProbeRing(String key, String owner, ProbeConfig config) { + this.key = key; + if (owner != null) { + this.owners.add(owner); + } + this.config = config; + this.capacity = config.ringCapacity; + this.budgetBytes = Math.max(1L, config.budgetBytes / Math.max(1, config.maxKeys)); + this.mask = this.capacity - 1; + this.slots = new AtomicReferenceArray(this.capacity); + this.generation = System.currentTimeMillis(); + this.anchorEpochNanos = this.generation * 1_000_000L; + this.anchorNanoTime = System.nanoTime(); + } + + /** + * Records what was rewritten into {@code owner}'s class under this probe. + * + *

Stamped with a JVM-wide sequence rather than left to the order a map happens to iterate in, + * so rebuilding a class from its armings produces the same bytes every time. Two spans arming the + * SAME method nest one inside the other, and which is inner would otherwise vary between runs. + */ + void arm(String owner, Map options) { + if (owner == null) { + return; + } + Map arming = new LinkedHashMap(); + if (options != null) { + arming.putAll(options); + } + arming.put("owner", owner); + arming.put("armedSeq", Long.valueOf(ARMED.incrementAndGet())); + armings.add(arming); + } + + /** Orders armings across every ring; see {@link #arm}. */ + private static final AtomicLong ARMED = new AtomicLong(); + + /** + * Epoch nanoseconds for an event. + * + *

Java 8 has no nanosecond wall clock, and {@code currentTimeMillis() * 1_000_000} is a + * millisecond wearing a nanosecond costume: two events in one millisecond would tie and the JSONL + * would be unorderable. So the wall clock is sampled once, at open, and every event is that + * anchor plus elapsed {@code nanoTime}. The result is monotonic and correctly spaced. It is NOT + * NTP-accurate and is only comparable within this JVM — which is enough, because being + * JVM-monotonic is what lets several probes' events merge into one ordered stream. + */ + long now() { + return anchorEpochNanos + (System.nanoTime() - anchorNanoTime); + } + + /** True while the ring accepts writes. A closed ring records nothing. */ + boolean isActive() { + return active; + } + + /** + * True once the global budget was crossed and events stopped carrying rendered argument and + * return text. Structure and timing survive: losing the values costs a reader detail, losing the + * call graph costs them the answer, and an OOM costs them the JVM they were diagnosing. + */ + boolean isDegraded() { + return degraded; + } + + void suppress() { + suppressed.incrementAndGet(); + } + + long suppressedCount() { + return suppressed.get(); + } + + /** A root call its filter decided against, or that was abandoned unfinished. */ + void filter() { + filtered.incrementAndGet(); + } + + long filteredCount() { + return filtered.get(); + } + + /** Events left out of published calls that outgrew their thread's buffer. */ + void omitted(int count) { + omitted.addAndGet(count); + } + + long omittedCount() { + return omitted.get(); + } + + /** This ring's share of the JVM-wide byte budget, which also bounds a held call. */ + long budgetBytes() { + return budgetBytes; + } + + long writeSeq() { + return seq.get(); + } + + long oldestSeq() { + return Math.max(0L, seq.get() - capacity); + } + + /** Publishes one event. Never blocks, never throws, never waits on a reader. */ + void add(ProbeEvent event) { + long s = seq.getAndIncrement(); + int index = (int) (s & mask); + ProbeEvent evicted = slots.get(index); + event.seq = s; + slots.lazySet(index, event); + charge(event, evicted); + if (s + 1 >= config.maxEvents) { + complete = true; + } + } + + /** + * The event budget is spent: this probe has stopped recording. + * + *

Distinct from {@link #isActive}, and deliberately so. Closed means the buffer is gone and an + * empty read says nothing about what ran; complete means the capture finished on its own terms + * and everything it holds is still there to read. Reporting the second as the first would turn a + * successful sample into a warning about a revoked recording. + * + *

Set from a writer with a plain volatile store rather than a compare-and- set: two calls + * crossing the budget at the same instant both write true, and the count they cross at is a + * budget, not an invariant. + */ + boolean isComplete() { + return complete; + } + + /** + * True once recording stopped and the buffer is still readable. + * + *

What the injected probes check. A frame whose entry was recorded and whose exit was not is + * the ordinary shape of the last invocation before the budget ran out; readers already tolerate + * an unclosed frame, and {@link #isComplete} is what explains it. + */ + boolean isRecording() { + return active && !complete; + } + + boolean beginWrite() { + if (!isRecording()) { + return false; + } + writers.incrementAndGet(); + if (!isRecording()) { + writers.decrementAndGet(); + return false; + } + return true; + } + + void endWrite() { + writers.decrementAndGet(); + } + + private void charge(ProbeEvent added, ProbeEvent evicted) { + long delta = added.charge() - (evicted == null ? 0L : evicted.charge()); + boolean over = charge.addAndGet(delta) > budgetBytes; + if (over != degraded) { + degraded = over; + } + } + + /** Stops new writes and waits for every admitted writer to publish. */ + synchronized boolean freeze() { + if (!active) { + return false; + } + active = false; + while (writers.get() > 0) { + Thread.yield(); + } + return true; + } + + /** Stops the ring recording and lets go of what it held. */ + void close() { + freeze(); + for (int i = 0; i < capacity; i++) { + slots.lazySet(i, null); + } + charge.set(0L); + } + + /** One cursored read: the events after {@code sinceSeq}, and what was missed. */ + static final class Batch { + final List events = new ArrayList(); + long nextSeq; + long oldestSeq; + long writeSeq; + long dropped; + boolean truncated; + } + + /** + * Reads forward from {@code sinceSeq} (inclusive) without removing anything. + * + *

Non-destructive is what lets two readers — a following CLI and a browser — watch one span at + * once, and it removes by construction the race a destructive drain has, where an event written + * between the read and the removal disappears. + */ + Batch read(long sinceSeq, int limit) { + Batch batch = new Batch(); + long write = seq.get(); + long oldest = Math.max(0L, write - capacity); + batch.writeSeq = write; + batch.oldestSeq = oldest; + batch.dropped = Math.max(0L, oldest - sinceSeq); + + long s = Math.max(sinceSeq, oldest); + for (; s < write && batch.events.size() < limit; s++) { + ProbeEvent event = slots.get((int) (s & mask)); + if (event != null && event.seq == s) { + batch.events.add(event); + continue; + } + // The slot no longer holds this sequence. Either it was lapped while + // we read — count it, it is genuinely gone — or a writer claimed the + // ticket and has not published yet, in which case stopping here and + // returning s as the cursor picks it up on the next read. Guessing + // the second case is the first would silently skip a live event. + if (s < Math.max(0L, seq.get() - capacity)) { + batch.dropped++; + continue; + } + break; + } + batch.nextSeq = s; + batch.truncated = s < write; + return batch; + } + + /** Rounds up to a power of two so the index is a mask rather than a modulo. */ + static int roundUpToPowerOfTwo(int requested) { + if (requested <= 1) { + return 1; + } + return Integer.highestOneBit(requested - 1) << 1; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeThreadState.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeThreadState.java new file mode 100644 index 0000000..555d9fa --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/agent/ProbeThreadState.java @@ -0,0 +1,246 @@ +package com.flanksource.telltale.diag.agent; + +/** + * What one thread is in the middle of, so an event can say how deep it is and which invocation it + * belongs to. + * + *

Confined to its owning thread, so nothing here is synchronized and nothing needs to be. That + * is also why frame ids can be a plain field increment rather than an atomic: a parent invocation + * is always on the same thread as its child, so ids only have to be unique per thread and the pair + * {@code (threadId, frame)} is what identifies an invocation downstream. + */ +final class ProbeThreadState { + + final long threadId; + + /** + * Cached, because {@code Thread.getName()} allocates on the Java 8 target — it returns {@code new + * String(char[], true)} on every call — and this would otherwise be an allocation per probe + * inside application code. A thread renamed after first touch keeps its old name here, which is + * why events also carry {@link #threadId}. + */ + final String threadName; + + /** Slots per open invocation in {@link #stack}. */ + private static final int STRIDE = 5; + + /** + * Interleaved {frame, startNanos, startAlloc, startGcCount, startGcMillis}, one group per open + * invocation. startAlloc is -1 unless the probe measures allocation, and the two GC counters -1 + * unless it measures GC. + */ + private long[] stack = new long[32 * STRIDE]; + + private int depth; + private long nextFrame; + private int lastUnwound; + + /** The startAlloc of the frame the most recent {@link #popTo} closed, or -1. */ + private long poppedAlloc = -1L; + + /** The GC counters the frame the most recent {@link #popTo} closed opened at, or -1. */ + private long poppedGcCount = -1L; + + private long poppedGcMillis = -1L; + + /** + * The allocation counter as {@code probeReturn} read it for {@link #returnFrame}, BEFORE it + * snapshotted the value — so the exit that follows charges the method, not the rendering of what + * it returned. -1 when there is none. + */ + private long returnAlloc = -1L; + + private long returnFrame; + + /** The probe generation these open frames belong to; see {@link #resetIfStale}. */ + private long epoch = -1L; + + /** + * Forgets any open frames when the set of open probes has changed since this thread last + * recorded. + * + *

Closing a probe strands whatever frames it had open: the matching exit arrives, finds the + * buffer gone, and returns without popping — so the depth would stay raised for the life of the + * thread. On a pooled request thread that is permanent, and every later trace on it would report + * nesting that never happened. Frames belonging to a probe that is gone can never be closed, so + * dropping them is the repair rather than a loss. + */ + void resetIfStale(long currentEpoch) { + if (epoch == currentEpoch) { + return; + } + epoch = currentEpoch; + depth = 0; + lastUnwound = 0; + returnAlloc = -1L; + // A held call's frames are gone with the rest; it can never end. + pending.clear(); + } + + /** + * The root calls filtered probes hold on this thread, one per probe at most and usually none. A + * list rather than one slot: two filtered spans may nest on one thread. + */ + private final java.util.ArrayList pending = new java.util.ArrayList(2); + + /** The call {@code ring} is holding on this thread, or null. */ + ProbePending pending(ProbeRing ring) { + for (int i = 0; i < pending.size(); i++) { + if (pending.get(i).ring == ring) { + return pending.get(i); + } + } + return null; + } + + void hold(ProbePending call) { + pending.add(call); + } + + void release(ProbePending call) { + pending.remove(call); + } + + /** Set when the last rendered value hit a length cap. */ + boolean truncated; + + /** What the last value snapshot retains, for the event's charge against the budget. */ + long renderedBytes; + + /** + * Set while snapshotting a value, so a rendering that reaches an instrumented method declines + * instead of recursing. The snapshot runs no application code, so what is left to guard is the + * JDK's own {@code toString()} on a value type — reachable only by instrumenting a JDK class — + * and unbounded recursion inside application code is the price of not guarding it. + */ + boolean inProbe; + + ProbeThreadState(Thread thread) { + this.threadId = thread.getId(); + this.threadName = thread.getName(); + } + + int depth() { + return depth; + } + + long nextFrame() { + return ++nextFrame; + } + + /** The enclosing invocation's frame, or 0 when there is none or it went past the cap. */ + long parentFrame(int maxDepth) { + int top = depth - 1; + if (top < 0 || top >= maxDepth) { + return 0L; + } + return stack[top * STRIDE]; + } + + /** + * Opens a frame. Beyond {@code maxDepth} the depth still counts but the frame is not recorded — + * the cap bounds memory, not correctness, and a run that deep is pathological recursion rather + * than something anyone is reading. + */ + void push( + long frame, + long startNanos, + long startAlloc, + long startGcCount, + long startGcMillis, + int maxDepth) { + if (depth < maxDepth) { + ensure(depth); + int base = depth * STRIDE; + stack[base] = frame; + stack[base + 1] = startNanos; + stack[base + 2] = startAlloc; + stack[base + 3] = startGcCount; + stack[base + 4] = startGcMillis; + } + depth++; + } + + /** Holds the allocation counter {@code probeReturn} read for {@code frame}. */ + void markReturn(long frame, long allocated) { + returnFrame = frame; + returnAlloc = allocated; + } + + /** + * The counter {@link #markReturn} held for {@code frame}, consumed, or -1 when it holds none for + * that frame and the caller has to read the counter itself. + */ + long takeReturnAlloc(long frame) { + long allocated = returnFrame == frame ? returnAlloc : -1L; + returnAlloc = -1L; + return allocated; + } + + /** + * Closes {@code frame} and everything opened inside it, returning the nanos it started at, or -1 + * when it is not on the stack. + * + *

Unwinding TO a frame rather than popping one is what keeps depth honest. If an inner exit + * was lost — an exception escaping a method instrumented without a handler, or a probe closed + * mid-flight — popping one would leave the depth permanently high on a pooled Tomcat thread, and + * every later trace on that thread would read as nested when it was not. Unwinding to the named + * frame self-heals on the next outer exit, and {@link #lastUnwound()} reports the repair so it is + * visible rather than silent. + */ + long popTo(long frame) { + int recorded = Math.min(depth, stack.length / STRIDE); + for (int position = recorded - 1; position >= 0; position--) { + if (stack[position * STRIDE] == frame) { + lastUnwound = depth - 1 - position; + long startNanos = stack[position * STRIDE + 1]; + poppedAlloc = stack[position * STRIDE + 2]; + poppedGcCount = stack[position * STRIDE + 3]; + poppedGcMillis = stack[position * STRIDE + 4]; + depth = position; + return startNanos; + } + } + // Not recorded: either past the depth cap or from a frame opened before + // this probe existed. Give back one level rather than none, so depth + // still trends back to zero instead of climbing forever. + lastUnwound = 0; + poppedAlloc = -1L; + poppedGcCount = -1L; + poppedGcMillis = -1L; + if (depth > 0) { + depth--; + } + return -1L; + } + + /** Frames closed beyond the one named, from the most recent {@link #popTo}. */ + int lastUnwound() { + return lastUnwound; + } + + /** The allocation counter the frame closed by the most recent {@link #popTo} opened at, or -1. */ + long poppedAlloc() { + return poppedAlloc; + } + + /** The GC collection count the most recently closed frame opened at, or -1. */ + long poppedGcCount() { + return poppedGcCount; + } + + /** The GC collection milliseconds the most recently closed frame opened at, or -1. */ + long poppedGcMillis() { + return poppedGcMillis; + } + + private void ensure(int index) { + int required = (index + 1) * STRIDE; + if (required <= stack.length) { + return; + } + long[] grown = new long[stack.length * 2]; + System.arraycopy(stack, 0, grown, 0, stack.length); + stack = grown; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/graph/GraphLimits.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/graph/GraphLimits.java new file mode 100644 index 0000000..24ab807 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/graph/GraphLimits.java @@ -0,0 +1,56 @@ +package com.flanksource.telltale.diag.graph; + +/** + * What one {@link ValueGraph} walk may build. Immutable, so a probe resolves it once when it opens + * and every thread shares it. + */ +public final class GraphLimits { + final int depth; + final int children; + final int nodes; + final int chars; + final int eventChars; + + /** A probe snapshot: run no application code, take no monitor, and bound everything. */ + final boolean inert; + + private GraphLimits( + int depth, int children, int nodes, int chars, int eventChars, boolean inert) { + this.depth = depth; + this.children = children; + this.nodes = nodes; + this.chars = chars; + this.eventChars = eventChars; + this.inert = inert; + } + + /** The OGNL surface: {@code depth} levels (0..10), 100 children a container, text uncapped. */ + public static GraphLimits ognl(int depth) { + return new GraphLimits( + Math.max(0, Math.min(depth, 10)), + 100, + Integer.MAX_VALUE, + Integer.MAX_VALUE, + Integer.MAX_VALUE, + false); + } + + /** + * A probe snapshot: {@code depth} levels (1..10) and {@code children} a container; {@code nodes} + * and {@code eventChars} across one whole event, shared by all of its arguments; {@code chars} + * per string. + */ + public static GraphLimits probe(int depth, int children, int nodes, int chars, int eventChars) { + return new GraphLimits( + Math.max(1, Math.min(depth, 10)), + Math.max(1, children), + Math.max(1, nodes), + Math.max(1, chars), + Math.max(1, eventChars), + true); + } + + public int depth() { + return depth; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/graph/InertTypes.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/graph/InertTypes.java new file mode 100644 index 0000000..5d607df --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/graph/InertTypes.java @@ -0,0 +1,134 @@ +package com.flanksource.telltale.diag.graph; + +import java.lang.reflect.Field; + +/** + * What a probe snapshot may render as text, or walk element by element, on application's own + * thread. + * + *

A class under {@code java.} is not enough on its own. Some JDK classes render by running what + * the application handed them — {@code CharBuffer.wrap(sequence)}, a {@code LongAccumulator}'s + * function — and some take their own monitor in every method: another thread can hold a {@code + * Vector}, {@code Hashtable} or {@code StringBuffer} while it runs foreign code ({@code indexOf} + * calls {@code equals}, {@code append} calls {@code toString}) that waits on a lock application's + * thread holds. So both lists name exact classes, and everything else is walked by its fields, + * which reads without calling or locking anything. + */ +public final class InertTypes { + + private InertTypes() {} + + /** + * Whether a probe renders {@code value} as one line of text. An enum renders by {@code name()}, + * which is final; a class by its name; an exception by {@link #throwableText}. + */ + static boolean scalar(Class type, Object value) { + if (value instanceof Enum || value instanceof Class || value instanceof Throwable) { + return true; + } + switch (type.getName()) { + case "java.lang.String": + case "java.lang.StringBuilder": + case "java.lang.Boolean": + case "java.lang.Character": + case "java.lang.Byte": + case "java.lang.Short": + case "java.lang.Integer": + case "java.lang.Long": + case "java.lang.Float": + case "java.lang.Double": + case "java.math.BigDecimal": + case "java.math.BigInteger": + case "java.util.concurrent.atomic.AtomicBoolean": + case "java.util.concurrent.atomic.AtomicInteger": + case "java.util.concurrent.atomic.AtomicLong": + case "java.util.concurrent.atomic.LongAdder": + case "java.util.concurrent.atomic.DoubleAdder": + case "java.util.Date": + case "java.sql.Date": + case "java.sql.Time": + case "java.sql.Timestamp": + case "java.util.UUID": + case "java.util.Currency": + case "java.util.Locale": + case "java.time.Instant": + case "java.time.LocalDate": + case "java.time.LocalDateTime": + case "java.time.LocalTime": + case "java.time.OffsetDateTime": + case "java.time.OffsetTime": + case "java.time.ZonedDateTime": + case "java.time.Duration": + case "java.time.Period": + case "java.time.Year": + case "java.time.YearMonth": + case "java.time.MonthDay": + case "java.time.ZoneOffset": + case "java.time.ZoneRegion": + return true; + default: + return false; + } + } + + /** + * Whether a probe walks a map or iterable of this class element by element: only concrete JDK + * collections whose iteration touches nothing but their own storage, and takes no monitor. + * + *

A wrapper such as {@code Collections$UnmodifiableList} is left out on purpose: it delegates + * to whatever it wraps, which may be a persistent collection that loads on first touch. Walked by + * its fields instead, it still shows what it wraps. {@code WeakHashMap} is left out because + * iterating it expunges entries. + */ + static boolean iterates(Class type) { + switch (type.getName()) { + case "java.util.ArrayList": + case "java.util.LinkedList": + case "java.util.Arrays$ArrayList": + case "java.util.ArrayDeque": + case "java.util.HashMap": + case "java.util.LinkedHashMap": + case "java.util.TreeMap": + case "java.util.IdentityHashMap": + case "java.util.EnumMap": + case "java.util.HashSet": + case "java.util.LinkedHashSet": + case "java.util.TreeSet": + case "java.util.Collections$EmptyList": + case "java.util.Collections$EmptyMap": + case "java.util.Collections$EmptySet": + case "java.util.Collections$SingletonList": + case "java.util.Collections$SingletonMap": + case "java.util.Collections$SingletonSet": + case "java.util.concurrent.ConcurrentHashMap": + case "java.util.concurrent.CopyOnWriteArrayList": + case "java.util.concurrent.ConcurrentLinkedQueue": + return true; + default: + return false; + } + } + + /** + * An exception as its class and message. A JDK exception's own {@code getMessage()} is the JDK's; + * any other class may override it, so its message is read from {@code Throwable}'s field instead. + * That read needs {@code setAccessible}, which Java 8 — what application runs — allows and a JDK + * 9+ module system refuses; refused, the class name alone still says what was thrown. + */ + public static String throwableText(Throwable error) { + String name = error.getClass().getName(); + String message = null; + if (name.startsWith("java.")) { + message = error.getMessage(); + } else { + try { + Field detail = Throwable.class.getDeclaredField("detailMessage"); + detail.setAccessible(true); + message = (String) detail.get(error); + } catch (ReflectiveOperationException | RuntimeException e) { + message = null; + } + } + return message == null ? name : name + ": " + message; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/graph/ValueGraph.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/graph/ValueGraph.java new file mode 100644 index 0000000..05b5b77 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/graph/ValueGraph.java @@ -0,0 +1,593 @@ +package com.flanksource.telltale.diag.graph; + +import java.lang.reflect.Array; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; + +/** + * Walks a live Java value into the object tree the diagnostics UI renders. + * + *

One source, compiled twice. The WAR walks an OGNL expression's result with it; the agent jar + * walks the arguments and return value of an instrumented method with it, relocated there to {@code + * com.flanksource.telltale.diag.agent.graph} by the {@code agentJar} task so the copy on the system + * class path can never shadow the webapp's. The two callers want different things from the same + * walk, and {@link GraphLimits} is the whole difference: + * + *

    + *
  • {@link GraphLimits#ognl} — an operator asked for this value, on a servlet thread, and can + * expand any part of it later. Every node carries its OGNL path, what was not expanded says + * so, and any map or iterable is walked, exactly as the OGNL surface always has. + *
  • {@link GraphLimits#probe} — nobody asked application's thread to render anything, so the + * walk runs no application code and takes no monitor. It reads fields instead of + * calling getters, calls {@code toString()} only on the JDK value types {@link InertTypes} + * names, and iterates only the JDK collections it names. A Hibernate collection is shown by + * its fields — its backing list once initialised, {@code initialized=false} before — and is + * never loaded. Depth, children per container, nodes and characters are all bounded, because + * the tree is held in the agent's ring until someone reads it. + *
+ * + *

No static state, deliberately: in the agent the first walk can run on an application thread, + * where a failing {@code } would be an {@code ExceptionInInitializerError} in application's + * own call path for the life of the JVM. + */ +public final class ValueGraph { + + /** Shown in place of a probe's bean past the depth limit, which nothing can expand later. */ + private static final String NOT_EXPANDED = "not expanded: past the expand depth"; + + private final GraphLimits limits; + + /** + * Objects already walked, with where they were first shown. + * + *

In a probe snapshot this spans the whole event, so an object reached twice — a cycle, or one + * DCL passed as two arguments — is shown once and pointed at after that. The OGNL surface keeps + * only the ancestors of the node being built, as it always has: a shared reference is shown in + * full wherever it appears and only a cycle is cut. + */ + private final IdentityHashMap seen = new IdentityHashMap(); + + private int nodes; + private long chars; + private long bytes; + private boolean truncated; + + public ValueGraph(GraphLimits limits) { + this.limits = limits; + } + + /** + * Walks {@code value} into a tree whose root is labelled {@code label}. + * + * @param path the root's OGNL accessor, or null for a probe snapshot — which then carries no ids + * and no paths, since the object is gone by the time anyone could expand it and the lowering + * assigns ids itself + */ + public Node root(Object value, String label, String path) { + return node(value, limits.depth, path == null ? null : "$", label, path, null); + } + + /** Roughly what this walk retained, in bytes, for the agent's memory budget. */ + public long bytes() { + return bytes; + } + + /** Whether any bound cut something: a string, a container, the depth, or the whole walk. */ + public boolean truncated() { + return truncated; + } + + /** A value recorded as its class and identity only, reading nothing from it. */ + public static Node opaque(String label, Object value) { + return new Node( + null, + label, + "opaque", + null, + false, + value.getClass().getName(), + null, + identity(value), + null); + } + + /** A root that stands for values not recorded, e.g. arguments past the argument cap. */ + public static Node marker(String label, String value) { + return new Node(null, label, "scalar", null, false, null, value, null, null); + } + + private Node node( + Object value, int depth, String id, String label, String path, String parentTrail) { + nodes++; + if (value == null) { + return scalar(id, label, path, "null", null); + } + Class type = value.getClass(); + if (isScalar(type, value)) { + return scalar(id, label, path, scalarText(value), type.getName()); + } + String seenAt = seen.get(value); + if (seenAt != null) { + return limits.inert + ? leaf( + id, label, path, "identity", type.getName(), text("same object as " + seenAt), false) + : leaf(id, label, path, "identity", type.getName(), identity(value), true); + } + // Remembered only where it is expanded: past the depth none of it is shown, so a later, + // shallower occurrence + // pointing back there would lose the object from the capture entirely. + boolean expands = depth > 0; + if (expands) { + seen.put(value, limits.inert ? trail(parentTrail, label) : ""); + } + try { + if (type.isArray()) { + return listNode(value, type, Array.getLength(value), depth, id, label, path, parentTrail); + } + if (value instanceof Map && iterates(type)) { + return mapNode((Map) value, depth, id, label, path, parentTrail); + } + if (value instanceof Iterable && iterates(type)) { + return listNode(value, type, size(value), depth, id, label, path, parentTrail); + } + return beanNode(value, depth, id, label, path, parentTrail); + } finally { + if (expands && !limits.inert) { + seen.remove(value); + } + } + } + + private Node mapNode( + Map map, int depth, String id, String label, String path, String parentTrail) { + String type = map.getClass().getName(); + int size = map.size(); + if (depth == 0) { + cutAtDepth(size > 0); + return leaf(id, label, path, "map", type, "size=" + size, !limits.inert); + } + String trail = trail(parentTrail, label); + List children = new ArrayList(Math.min(size, limits.children)); + int index = 0; + try { + for (Map.Entry entry : map.entrySet()) { + if (index >= limits.children || spent()) { + break; + } + children.add(entry(entry, depth, id, path, trail, index)); + index++; + } + } catch (RuntimeException e) { + if (!limits.inert) { + throw e; + } + children.add(failure(id, path, "iteration", e)); + } + appendTruncation(children, size, index, id, path); + return branch(id, label, path, "map", type, children); + } + + /** + * One map entry. The OGNL surface labels it with the key's text, as it always has; a probe does + * that only for a key it may render, and otherwise shows the key and the value side by side under + * the entry's position. + */ + private Node entry( + Map.Entry entry, int depth, String id, String path, String trail, int index) { + Object key = entry.getKey(); + if (!limits.inert) { + String text = String.valueOf(key); + return node( + entry.getValue(), + depth - 1, + id + "." + index, + text, + path + ".get(\"" + escape(text) + "\")", + trail); + } + if (key == null || isScalar(key.getClass(), key)) { + String text = key == null ? "null" : scalarText(key); + return node(entry.getValue(), depth - 1, null, text, null, trail); + } + String position = "#" + index; + List pair = new ArrayList(2); + pair.add(node(key, depth - 1, null, "key", null, trail(trail, position))); + pair.add(node(entry.getValue(), depth - 1, null, "value", null, trail(trail, position))); + return branch(null, position, null, "bean", "java.util.Map$Entry", pair); + } + + private Node listNode( + Object value, + Class type, + int size, + int depth, + String id, + String label, + String path, + String parentTrail) { + String listLabel = label + " [" + size + "]"; + if (depth == 0) { + cutAtDepth(size > 0); + return leaf(id, listLabel, path, "list", type.getName(), "size=" + size, !limits.inert); + } + String trail = trail(parentTrail, label); + List children = new ArrayList(Math.min(size, limits.children)); + int index = 0; + try { + if (type.isArray()) { + for (; index < size && index < limits.children && !spent(); index++) { + children.add(element(Array.get(value, index), depth, id, path, trail, index)); + } + } else { + for (Object item : (Iterable) value) { + if (index >= limits.children || spent()) { + break; + } + children.add(element(item, depth, id, path, trail, index)); + index++; + } + } + } catch (RuntimeException e) { + if (!limits.inert) { + throw e; + } + children.add(failure(id, path, "iteration", e)); + } + appendTruncation(children, size, index, id, path); + return branch(id, listLabel, path, "list", type.getName(), children); + } + + private Node element(Object item, int depth, String id, String path, String trail, int index) { + String position = String.valueOf(index); + return node( + item, + depth - 1, + id == null ? null : id + "." + position, + position, + path == null ? null : path + "[" + index + "]", + trail); + } + + private Node beanNode( + Object value, int depth, String id, String label, String path, String parentTrail) { + Class type = value.getClass(); + if (depth == 0) { + cutAtDepth(true); + return limits.inert + ? leaf(id, label, path, "opaque", type.getName(), NOT_EXPANDED, false) + : leaf(id, label, path, "opaque", type.getName(), identity(value), true); + } + List fields; + try { + fields = fields(type); + } catch (RuntimeException | LinkageError e) { + // A field whose type cannot be resolved (NoClassDefFoundError), or a SecurityManager refusing + // the read. + if (!limits.inert) { + throw e; + } + truncated = true; + return leaf( + id, + label, + path, + "error", + type.getName(), + "fields unreadable: " + e.getClass().getName(), + false); + } + String trail = trail(parentTrail, label); + List children = new ArrayList(Math.min(fields.size(), limits.children)); + int count = 0; + for (Field field : fields) { + if (count >= limits.children || spent()) { + break; + } + children.add(field(value, field, depth, id, path, trail)); + count++; + } + appendTruncation(children, fields.size(), count, id, path); + return branch(id, label, path, "bean", type.getName(), children); + } + + private Node field(Object owner, Field field, int depth, String id, String path, String trail) { + String name = field.getName(); + String childId = id == null ? null : id + "." + name; + String childPath = path == null ? null : path + "." + name; + try { + if (!field.isAccessible()) { + field.setAccessible(true); + } + return node(field.get(owner), depth - 1, childId, name, childPath, trail); + } catch (RuntimeException | IllegalAccessException e) { + if (!limits.inert) { + return leaf( + childId, name, childPath, "opaque", field.getType().getName(), e.getMessage(), false); + } + truncated = true; + return leaf( + childId, + name, + childPath, + "error", + field.getType().getName(), + "unreadable: " + e.getClass().getName(), + false); + } + } + + private static List fields(Class type) { + List fields = new ArrayList(); + for (Class current = type; + current != null && current != Object.class; + current = current.getSuperclass()) { + for (Field field : current.getDeclaredFields()) { + if (Modifier.isStatic(field.getModifiers()) + || field.isSynthetic() + || field.getName().startsWith("CGLIB$")) { + continue; + } + fields.add(field); + } + } + Collections.sort( + fields, + new Comparator() { + @Override + public int compare(Field a, Field b) { + return a.getName().compareTo(b.getName()); + } + }); + return fields; + } + + /** Whether a value renders as one line of text. For a probe, see {@link InertTypes#scalar}. */ + private boolean isScalar(Class type, Object value) { + if (limits.inert) { + return InertTypes.scalar(type, value); + } + return type.isPrimitive() + || Number.class.isAssignableFrom(type) + || CharSequence.class.isAssignableFrom(type) + || Boolean.class == type + || Character.class == type + || Date.class.isAssignableFrom(type) + || type.isEnum() + || Class.class == type; + } + + private String scalarText(Object value) { + if (!limits.inert) { + return String.valueOf(value); + } + if (value instanceof Enum) { + return ((Enum) value).name(); + } + if (value instanceof Class) { + return ((Class) value).getName(); + } + if (value instanceof Throwable) { + return text(InertTypes.throwableText((Throwable) value)); + } + return text(String.valueOf(value)); + } + + /** + * Whether a map or iterable of this class is walked element by element. For a probe, see {@link + * InertTypes#iterates}. + */ + private boolean iterates(Class type) { + return !limits.inert || InertTypes.iterates(type); + } + + private int size(Object value) { + if (value instanceof Collection) { + return ((Collection) value).size(); + } + int size = 0; + for (Object ignored : (Iterable) value) { + size++; + } + return size; + } + + /** True once a probe's node budget is gone; the OGNL surface has none. */ + private boolean spent() { + return limits.inert && nodes >= limits.nodes; + } + + /** + * Keeps a string within the per-string cap and what is left of the event's text budget, from its + * head AND its tail: a SQL statement cut at the head alone loses its WHERE clause, which is + * usually the part being read. + */ + private String text(String value) { + long remaining = limits.eventChars - chars; + int cap = (int) Math.max(0L, Math.min(limits.chars, remaining)); + if (value.length() <= cap) { + chars += value.length(); + return value; + } + truncated = true; + if (cap == 0) { + return "…(" + value.length() + " chars not kept: this event's text budget is spent)"; + } + chars += cap; + int tail = cap / 2; + int head = cap - tail; + return value.substring(0, head) + + "…(+" + + (value.length() - cap) + + " chars)…" + + value.substring(value.length() - tail); + } + + /** + * A probe's cut at the depth hides what it did not expand as surely as a cap does; the OGNL + * surface can expand it. + */ + private void cutAtDepth(boolean hidAnything) { + if (limits.inert && hidAnything) { + truncated = true; + } + } + + private void appendTruncation( + List children, int total, int included, String id, String path) { + if (total > included) { + if (limits.inert) { + truncated = true; + } + children.add( + scalar( + id == null ? null : id + ".truncated", + "(truncated)", + path, + (total - included) + " more values", + null)); + } + } + + private Node failure(String id, String path, String what, RuntimeException e) { + truncated = true; + return leaf( + id == null ? null : id + ".failed", + "(" + what + " failed)", + path, + "error", + null, + e.getClass().getName(), + false); + } + + private Node scalar(String id, String label, String path, String value, String type) { + return account(new Node(id, label, "scalar", path, false, type, value, null, null)); + } + + private Node leaf( + String id, + String label, + String path, + String kind, + String type, + String raw, + boolean expandable) { + return account(new Node(id, label, kind, path, expandable, type, null, raw, null)); + } + + private Node branch( + String id, String label, String path, String kind, String type, List children) { + return account(new Node(id, label, kind, path, false, type, null, null, children)); + } + + /** + * Charges a node against the walk's byte estimate: a fixed cost for the object and its list slot, + * plus two bytes a character (the Java 8 target's strings are UTF-16). The type is not charged — + * it is the class's own cached name. + */ + private Node account(Node node) { + bytes += 48 + 2L * (length(node.label) + length(node.value) + length(node.raw)); + return node; + } + + private static int length(String value) { + return value == null ? 0 : value.length(); + } + + private static String trail(String parent, String label) { + return parent == null ? label : parent + "." + label; + } + + private static String identity(Object value) { + return value.getClass().getName() + "@" + Integer.toHexString(System.identityHashCode(value)); + } + + private static String escape(String value) { + return value.replace("\\", "\\\\").replace("\"", "\\\""); + } + + /** + * One node, in the JSON shape the Go {@code tracestore.ObjectGraphNode} and clicky-ui's {@code + * ObjectGraph} read. Built once by the walk and never changed after it returns, which is what + * lets the agent publish it across threads. + */ + public static final class Node { + /** Null in a probe snapshot: the Go lowering assigns ids when it places the tree. */ + public final String id; + + public final String label; + + /** scalar, map, list, bean, identity, opaque or error. */ + public final String kind; + + /** The OGNL accessor that re-evaluates this node; null in a probe snapshot. */ + public final String path; + + public final boolean expandable; + public final String type; + public final String value; + public final String raw; + + /** Null for a leaf. */ + public final List children; + + Node( + String id, + String label, + String kind, + String path, + boolean expandable, + String type, + String value, + String raw, + List children) { + this.id = id; + this.label = label; + this.kind = kind; + this.path = path; + this.expandable = expandable; + this.type = type; + this.value = value; + this.raw = raw; + this.children = children; + } + + /** + * The map the OGNL surface has always serialised, with its keys in the order it always wrote + * them. + */ + public Map toMap() { + Map out = new java.util.LinkedHashMap(); + out.put("id", id); + out.put("label", label); + out.put("kind", kind); + out.put("path", path); + out.put("expandable", expandable); + if (value != null) { + out.put("value", value); + } + if (type != null) { + out.put("type", type); + } + if (raw != null) { + out.put("raw", raw); + } + if (children != null) { + List> rendered = new ArrayList>(children.size()); + for (Node child : children) { + rendered.add(child.toMap()); + } + out.put("children", rendered); + } + return out; + } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/AgentBridge.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/AgentBridge.java new file mode 100644 index 0000000..8e87b61 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/AgentBridge.java @@ -0,0 +1,535 @@ +package com.flanksource.telltale.diag.hotpatch; + +import java.lang.instrument.Instrumentation; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Reaches the probe buffer and the patch leases that live in the attached agent, on the system + * class path, from this WAR. + * + *

They live there rather than here for the reason the pre-patch byte stash does: a redeployed + * WAR gets a fresh classloader, while the classes it rewrote stay rewritten. State that can undo a + * patch must outlive the thing that made it. Instrumented bytecode also calls the agent by name, + * and only the system class path is visible from every loader that might define a patched class. + * + *

Everything crossing the boundary is a JDK type — {@code Map}, {@code List}, {@code String}, + * {@code byte[]} — so neither side needs the other's classes. The one exception is the restore + * callback, which is an interface declared in the agent; it is implemented with a {@link Proxy} + * defined by the agent's own loader, since this WAR cannot name a type it cannot see. + */ +public final class AgentBridge { + + private static final Logger LOG = Logger.getLogger(AgentBridge.class.getName()); + + /** Default lease when a caller does not choose one; see {@code /classes/lease}. */ + public static final int DEFAULT_LEASE_SECONDS = 120; + + private static volatile boolean restorerInstalled; + + private AgentBridge() {} + + /** The probe API generation the bytecode this WAR injects depends on. */ + public static final int REQUIRED_PROBE_API = 7; + + /** + * Refuses to instrument when the attached agent predates the probe calls this WAR is about to + * inject into application code. + * + *

This check is the difference between a clear refusal and the worst failure this subsystem + * can produce. The agent jar sits on the system class path and cannot be reloaded without + * restarting the JVM, so a pod that attached an older agent and then received a newer WAR would + * happily redefine a class whose new bytecode calls a method that is not there — and the {@link + * NoSuchMethodError} would be raised inside application's own call path, on a request that never + * asked to be traced. Checking that the class merely loads is not enough; the injected + * signatures have to be there. + */ + public static void requireProbeApi() { + Class agent = agent(); + int version = probeApiVersion(agent); + if (version < REQUIRED_PROBE_API) { + throw new IllegalStateException( + staleAgent( + "it implements probe API v" + + version + + " while this WAR injects calls from v" + + REQUIRED_PROBE_API)); + } + requireMethod( + agent, + "probeEnter", + String.class, + String.class, + String.class, + String.class, + int.class, + Object[].class); + requireMethod(agent, "probeReturn", String.class, long.class, Object.class); + requireMethod(agent, "probeThrow", String.class, long.class, Throwable.class); + requireMethod(agent, "probeExit", String.class, long.class); + // Not injected — the WAR calls these itself, at restore. They are checked + // HERE anyway, because the alternative is discovering the agent cannot + // release a probe at the moment the class is already patched and this is + // the call that would put it back. + requireMethod(agent, "releaseProbe", String.class, String.class); + requireMethod(agent, "probeKeysOwnedBy", String.class); + // Also not injected, and checked here for the mirror-image reason: a WAR + // that could not read back what an earlier span armed would rebuild the + // class from its own selector alone and disarm that span, then report a + // clean capture of nothing. + requireMethod(agent, "probeArmingsOwnedBy", String.class); + } + + /** The probe API generation that measures {@code alloc}; see the agent's PROBE_API_VERSION. */ + public static final int ALLOC_PROBE_API = 8; + + /** The probe API generation that measures {@code gc}. */ + public static final int GC_PROBE_API = 9; + + /** The probe API generation that filters root calls: minCost, exceptionsOnly and match. */ + public static final int FILTER_PROBE_API = 10; + + /** + * Refuses an opt-in probe {@code option} when the attached agent predates {@code version}, the + * generation that implements it. + * + *

Checked only for a span that asks, unlike {@link #requireProbeApi}: no injected call + * changed, so an older agent still serves every other capture. It would ignore the option as + * unknown, though, and report a capture without the measurement and no reason — which is what + * this refuses. + */ + public static void requireProbeApiFor(String option, int version) { + int attached = probeApiVersion(agent()); + if (attached < version) { + throw new IllegalStateException( + option + + " needs probe API v" + + version + + " and the agent" + + " attached to this JVM implements v" + + attached + + ". The agent jar is on the system class" + + " path and cannot be replaced while the JVM runs; restart the JVM to measure " + + option + + ", or trace without it."); + } + } + + private static int probeApiVersion(Class agent) { + try { + return ((Number) agent.getMethod("probeApiVersion").invoke(null)).intValue(); + } catch (ReflectiveOperationException | ClassCastException e) { + throw new IllegalStateException(staleAgent("it has no probeApiVersion()"), e); + } + } + + private static void requireMethod(Class agent, String name, Class... types) { + try { + agent.getMethod(name, types); + } catch (NoSuchMethodException e) { + throw new IllegalStateException( + staleAgent("it has no " + name + " with the signature this WAR uses"), e); + } + } + + private static String staleAgent(String because) { + return "the agent attached to this JVM predates the probe API this WAR uses (" + + because + + "). The agent jar is on the system class path and cannot be replaced while the JVM" + + " runs, so instrumenting now would leave application code calling a method that is" + + " not there. Restart the JVM to pick up the current agent."; + } + + /** + * Opens the buffer an instrumented method records into. + * + *

Called before the class is rewritten, so a refusal — the JVM already has as many probes open + * as it allows — reaches the caller while the JVM is still untouched, rather than after a + * redefinition that would record into nothing. + * + *

The probe is owned by the class it instruments, keyed the way the agent already keys the + * byte stash and the leases, so a restore or a lapsed lease closes it without needing a second + * registry to consult. + */ + @SuppressWarnings("unchecked") + public static Map openProbe( + String key, String loaderHash, String className, Map options) { + InstrumentService.validateProbeKey(key); + return (Map) + invoke( + "openProbe", + new Class[] {String.class, String.class, Map.class}, + key, + PatchService.key(loaderHash, className), + options); + } + + /** Closes one probe and releases its buffer. */ + @SuppressWarnings("unchecked") + public static Map closeProbe(String key) { + InstrumentService.validateProbeKey(key); + return (Map) invoke("closeProbe", new Class[] {String.class}, key); + } + + /** Stops new probe events while retaining the ring for a final cursor read. */ + @SuppressWarnings("unchecked") + public static Map freezeProbe(String key) { + InstrumentService.validateProbeKey(key); + return (Map) invoke("freezeProbe", new Class[] {String.class}, key); + } + + /** + * Closes every probe opened for one class. + * + *

Called from the restore path and from the lease reaper. Without it a killed client leaks its + * buffer for the life of the JVM: probe keys carry a random per-span suffix, so each abandoned + * span strands a distinct key that nothing will ever look up again. + */ + @SuppressWarnings("unchecked") + public static List> closeProbesOwnedBy(String loaderHash, String className) { + return (List>) + invoke( + "closeProbesOwnedBy", + new Class[] {String.class}, + PatchService.key(loaderHash, className)); + } + + /** + * Releases one span's claim on a class, closing its ring if that was the last class the span + * recorded for. + * + *

The keyed counterpart of {@link #closeProbesOwnedBy}. A restore that knows which span is + * finishing must use this: several spans routinely instrument ONE class — the preservation plan + * arms {@code ActivityDal} three ways in a single step — and releasing by class alone closes all + * of their rings at once, so the first span to finish silently disarms the rest. + */ + @SuppressWarnings("unchecked") + public static Map releaseProbe(String key, String loaderHash, String className) { + InstrumentService.validateProbeKey(key); + return (Map) + invoke( + "releaseProbe", + new Class[] {String.class, String.class}, + key, + PatchService.key(loaderHash, className)); + } + + /** + * The probe keys still recording on one class. + * + *

A class's original bytes are shared by every span instrumented into it, so this is the + * question a restore asks before putting them back: an empty list means the restore is this + * span's alone to make. + */ + @SuppressWarnings("unchecked") + public static List probeKeysOwnedBy(String loaderHash, String className) { + return (List) + invoke( + "probeKeysOwnedBy", + new Class[] {String.class}, + PatchService.key(loaderHash, className)); + } + + /** + * Everything still armed into one class, oldest arming first. + * + *

What a NEW arming has to read before it rewrites the class, because a class is redefined + * whole: the bytes it hands the JVM replace every earlier span's rewritten methods along with its + * own, so it must carry all of them. Each row is the options the probe was opened with — the + * selector included — plus the {@code key} it records under. + * + *

Only armings of rings that still claim the class come back. The agent keeps every arming a + * ring was opened with, including one whose class was since released while another class kept the + * ring open — a multi-target span releases its classes one at a time. Rebuilding the class from + * that stale arming would re-wrap a released selector and record it into a ring that no longer + * counts the class as an owner, so nothing would ever restore it on that ring's behalf. + */ + @SuppressWarnings("unchecked") + public static List> probeArmingsOwnedBy(String loaderHash, String className) { + List> armings = + (List>) + invoke( + "probeArmingsOwnedBy", + new Class[] {String.class}, + PatchService.key(loaderHash, className)); + List holding = probeKeysOwnedBy(loaderHash, className); + List> live = new ArrayList<>(armings.size()); + for (Map arming : armings) { + if (holding.contains(String.valueOf(arming.get("key")))) { + live.add(arming); + } + } + return live; + } + + /** + * Every probe this JVM still has open, each with what its owners were armed for. + * + *

The ring cannot say whether it records a watch or a trace — that lives in the options it was + * opened with, which the agent keeps without reading. Interpreting them is this WAR's business + * (the agent can only be replaced by restarting the JVM), so the join happens here: one {@code + * armings} row per owner still holding the ring, carrying the selector and {@code calls}, oldest + * arming first — the owners set has no order of its own. + */ + @SuppressWarnings("unchecked") + public static List> probeKeys() { + List> keys = + (List>) invoke("probeKeys", new Class[0]); + Map>> armingsByOwner = new LinkedHashMap<>(); + for (Map row : keys) { + String key = String.valueOf(row.get("key")); + List> armed = new ArrayList<>(); + for (Object owner : (List) row.get("owners")) { + List> owned = + armingsByOwner.computeIfAbsent( + String.valueOf(owner), + o -> + (List>) + invoke("probeArmingsOwnedBy", new Class[] {String.class}, o)); + for (Map arming : owned) { + if (key.equals(arming.get("key"))) { + armed.add(arming); + } + } + } + armed.sort(Comparator.comparingLong(AgentBridge::armedSeq)); + List> armings = new ArrayList<>(armed.size()); + for (Map arming : armed) { + armings.add(armingRow(key, arming)); + } + row.put("armings", armings); + } + return keys; + } + + private static long armedSeq(Map arming) { + Object seq = arming.get("armedSeq"); + if (!(seq instanceof Number)) { + throw new IllegalStateException("agent reported an arming with no armedSeq: " + arming); + } + return ((Number) seq).longValue(); + } + + private static Map armingRow(String key, Map arming) { + Object owner = arming.get("owner"); + Object method = arming.get("method"); + if (!(method instanceof String) || ((String) method).trim().isEmpty()) { + throw new IllegalStateException( + "probe " + + key + + " is armed on " + + owner + + " without a method selector; every probe the WAR opens records the one it was armed" + + " for"); + } + Map row = new LinkedHashMap<>(); + row.put("owner", owner); + row.put("method", method); + row.put("calls", Boolean.valueOf(Boolean.TRUE.equals(arming.get("calls")))); + return row; + } + + /** + * Reads the events recorded under {@code key} from {@code sinceSeq} forward, without removing + * them. + * + *

A cursor rather than a destructive drain, for two reasons. It lets a following CLI and a + * browser watch one span at once. And it removes by construction the race the destructive version + * had — a window of a few instructions in which a probe that had already fetched the buffer + * appended to the one just removed, and that event went unreported with nothing to show it had + * ever existed. What bounds a long-lived JVM instead is the ring itself, which is fixed-size and + * reports what it evicted. + * + *

The events come back as rendered JSON lines. This WAR forwards them without parsing them: + * they were serialised inside the JVM that produced them and are appended to the caller's archive + * verbatim. + * + *

A read of an open ring also names its {@code owners} — the classes it still records for. A + * span arming several classes into one ring stays open while another surface restores one of + * them, and that restore drops the class from the ring without closing it; the owners are how the + * span learns that one of its targets was revoked while the rest kept recording. + */ + @SuppressWarnings("unchecked") + public static Map drain(String key, long sinceSeq, int limit) { + InstrumentService.validateProbeKey(key); + Map read = + (Map) + invoke( + "drain", + new Class[] {String.class, long.class, int.class}, + key, + Long.valueOf(sinceSeq), + Integer.valueOf(limit)); + for (Map open : + (List>) invoke("probeKeys", new Class[0])) { + if (key.equals(open.get("key"))) { + read.put("owners", open.get("owners")); + } + } + return read; + } + + /** + * When the agent will restore a class on its own, as epoch millis; 0 when no lease is scheduled + * for it. + * + *

Zero is the state worth surfacing rather than hiding: a patched class with no lease is one + * nothing will ever undo, which is exactly what a reader of the patch register needs to be told. + */ + @SuppressWarnings("unchecked") + public static long leaseExpiresAt(String loaderHash, String className) { + Map leases = (Map) field("LEASES"); + Long deadline = leases.get(PatchService.key(loaderHash, className)); + return deadline == null ? 0L : deadline.longValue(); + } + + private static Object field(String name) { + try { + Object value = agent().getField(name).get(null); + if (value == null) { + throw new IllegalStateException( + InstrumentationService.AGENT_CLASS + + "." + + name + + " is null — the attached agent jar predates instrumentation; redeploy" + + " telltale-diag"); + } + return value; + } catch (ReflectiveOperationException e) { + throw new IllegalStateException(staleAgent("it has no " + name + " field"), e); + } + } + + private static Object invoke(String name, Class[] types, Object... args) { + try { + return agent().getMethod(name, types).invoke(null, args); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException(staleAgent("it has no " + name + "()"), e); + } + } + + /** + * Registers or extends the deadline after which the agent puts a class's original bytes back on + * its own. + * + *

This is what makes a killed client safe. Every mutation that a span holds open registers a + * lease and renews it while the span runs; if the process holding it dies — a killed CLI, a + * dropped pod-exec, a Ctrl-C — nothing renews, and the JVM restores itself rather than serving + * rewritten code indefinitely. + */ + public static void lease(String loaderHash, String className, int seconds) { + installRestorer(); + int effective = seconds > 0 ? seconds : DEFAULT_LEASE_SECONDS; + try { + agent() + .getMethod("lease", String.class, long.class) + .invoke(null, PatchService.key(loaderHash, className), (long) effective * 1000L); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException( + "the attached agent jar cannot hold a patch lease, so a patch would outlive the run that" + + " made it. Restart the JVM to pick up the current agent: " + + e, + e); + } + } + + /** + * Teaches the agent how to undo a patch, and starts the daemon that does it. + * + *

The agent owns the deadlines but not the redefinition — it has no {@link ClassIndex} and no + * {@link PatchService}, and giving it either would put the whole hot-patch machinery on the + * system class path where a redeploy could never update it. So the WAR supplies the callback and + * the agent supplies the clock. + */ + public static synchronized void installRestorer() { + Class agent = agent(); + try { + Class restorer = + Class.forName(agent.getName() + "$Restorer", false, agent.getClassLoader()); + Field field = agent.getField("RESTORER"); + // Installed once per WAR instance, and again whenever the agent has + // none. The first half matters after a redeploy: a callback from the + // previous WAR would keep its whole classloader alive. The second is + // the invariant this method exists for — returning while the agent + // has no way back would leave a patch that cannot be undone. + if (restorerInstalled && field.get(null) != null) { + return; + } + field.set( + null, + Proxy.newProxyInstance( + agent.getClassLoader(), + new Class[] {restorer}, + new InvocationHandler() { + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + if ("restore".equals(method.getName()) && args != null && args.length == 1) { + restoreLeased(String.valueOf(args[0])); + return null; + } + return defaultAnswer(proxy, method, args); + } + })); + agent.getMethod("startReaper").invoke(null); + restorerInstalled = true; + } catch (ReflectiveOperationException e) { + throw new IllegalStateException( + "the attached agent jar cannot restore a lapsed patch, so a patch would outlive the run" + + " that made it. Restart the JVM to pick up the current agent: " + + e, + e); + } + } + + /** Puts one leased class back, keyed the way the agent stores it. */ + static void restoreLeased(String key) { + int split = key.indexOf('|'); + if (split < 0) { + throw new IllegalArgumentException("malformed lease key: " + key); + } + String loaderHash = key.substring(0, split); + String className = key.substring(split + 1); + Instrumentation instrumentation = InstrumentationService.instrumentation(); + ClassIndex index = ClassIndex.from(instrumentation); + new PatchService(instrumentation).restore(index, new PatchService.RestoreRequest( + index.resolve(className, loaderHash), PatchService.RestoreScope.ALL)); + LOG.log( + Level.WARNING, + "telltale-diag: restored {0} (loader {1}) — its patch lease lapsed," + + " so the client holding it is gone", + new Object[] {className, loaderHash}); + } + + /** Object's own methods, which a Proxy also routes through the handler. */ + private static Object defaultAnswer(Object proxy, Method method, Object[] args) { + if ("hashCode".equals(method.getName())) { + return System.identityHashCode(proxy); + } + if ("equals".equals(method.getName())) { + return proxy == (args == null ? null : args[0]); + } + if ("toString".equals(method.getName())) { + return "telltale-diag lease restorer"; + } + throw new UnsupportedOperationException( + "the lease restorer does not implement " + method.getName()); + } + + private static Class agent() { + try { + return ClassLoader.getSystemClassLoader().loadClass(InstrumentationService.AGENT_CLASS); + } catch (ClassNotFoundException e) { + throw new IllegalStateException( + "the instrumentation agent is not attached to this JVM: " + e, e); + } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/BytecodeSource.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/BytecodeSource.java new file mode 100644 index 0000000..6526b77 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/BytecodeSource.java @@ -0,0 +1,96 @@ +package com.flanksource.telltale.diag.hotpatch; + +import java.lang.instrument.ClassFileTransformer; +import java.lang.instrument.Instrumentation; +import java.lang.instrument.UnmodifiableClassException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.ProtectionDomain; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Reads the bytecode a class is currently running, not the bytes on disk. A no-op {@link + * ClassFileTransformer} registered with {@code canRetransform=true} is handed the live class file + * buffer during a {@link Instrumentation#retransformClasses} round-trip; it captures the buffer and + * returns {@code null} so nothing is modified. + * + *

Anything that already rewrote the class (an APM agent, or a later hot patch) is therefore + * reflected in what the browser shows — which is the whole point of a live class browser. + */ +public final class BytecodeSource { + + public static final String DIGEST_PREFIX = "sha256:"; + + private final Instrumentation instrumentation; + + public BytecodeSource(Instrumentation instrumentation) { + this.instrumentation = instrumentation; + } + + public byte[] bytecodeOf(Class target) { + com.flanksource.telltale.observer.ObserverRuntime.assertTargetActive(target); + if (!instrumentation.isRetransformClassesSupported()) { + throw new IllegalStateException( + "this JVM does not support class retransformation," + + " so the live bytecode of " + + target.getName() + + " cannot be read"); + } + if (!instrumentation.isModifiableClass(target)) { + throw new IllegalStateException( + target.getName() + + " is not a modifiable class" + + " (primitive, array, or JVM-synthesized), so it has no readable class file"); + } + final AtomicReference captured = new AtomicReference<>(); + final String internalName = target.getName().replace('.', '/'); + ClassFileTransformer capture = + new ClassFileTransformer() { + @Override + public byte[] transform( + ClassLoader loader, + String className, + Class classBeingRedefined, + ProtectionDomain protectionDomain, + byte[] classfileBuffer) { + if (classBeingRedefined == target && internalName.equals(className)) { + captured.set(classfileBuffer.clone()); + } + return null; + } + }; + instrumentation.addTransformer(capture, true); + try { + instrumentation.retransformClasses(target); + } catch (UnmodifiableClassException e) { + throw new IllegalStateException("cannot retransform " + target.getName() + ": " + e, e); + } finally { + instrumentation.removeTransformer(capture); + } + byte[] bytes = captured.get(); + if (bytes == null) { + throw new IllegalStateException( + "retransforming " + + target.getName() + + " produced no class file buffer — another transformer swallowed the retransform"); + } + return bytes; + } + + /** {@code sha256:} over the class bytes; the cache key and patch-drift detector. */ + public static String digest(byte[] bytes) { + MessageDigest sha256; + try { + sha256 = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable in this JVM", e); + } + byte[] hash = sha256.digest(bytes); + StringBuilder out = new StringBuilder(DIGEST_PREFIX.length() + hash.length * 2); + out.append(DIGEST_PREFIX); + for (byte b : hash) { + out.append(Character.forDigit((b >> 4) & 0xF, 16)).append(Character.forDigit(b & 0xF, 16)); + } + return out.toString(); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/ClassArtifact.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/ClassArtifact.java new file mode 100644 index 0000000..a09f7c9 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/ClassArtifact.java @@ -0,0 +1,48 @@ +package com.flanksource.telltale.diag.hotpatch; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.JarURLConnection; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.CodeSource; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +/** Reads defining artifacts without application loader resource calls; generated classes may have no entry. */ +public final class ClassArtifact { + private ClassArtifact() {} + + public static InputStream open(Class type) throws IOException { + String path = type.getName().replace('.', '/') + ".class"; + if (type.getClassLoader() == null) return type.getResourceAsStream("/" + path); + CodeSource source = type.getProtectionDomain().getCodeSource(); + URL location = source == null ? null : source.getLocation(); + if (location == null) return null; + String origin = location.toExternalForm(); + if (origin.startsWith("jrt:/")) return new URL(origin + "/" + path).openStream(); + if (!origin.startsWith("file:") && !origin.startsWith("jar:file:")) { + throw new IOException("code-source location is not a local class artifact: " + location); + } + if (location.getProtocol().equals("file")) { + Path root; + try { root = Paths.get(location.toURI()); } + catch (URISyntaxException error) { throw new IOException("invalid code-source location " + location, error); } + if (Files.isDirectory(root)) { + try { return Files.newInputStream(root.resolve(path)); } + catch (NoSuchFileException generated) { return null; } + } + } + URL jar = location.getProtocol().equals("jar") ? location : new URL("jar:" + origin + "!/"); + JarURLConnection connection = (JarURLConnection) jar.openConnection(); + connection.setUseCaches(true); + JarFile artifact = connection.getJarFile(); + String prefix = connection.getEntryName(); + JarEntry entry = artifact.getJarEntry((prefix == null ? "" : prefix) + path); + return entry == null ? null : artifact.getInputStream(entry); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/ClassIndex.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/ClassIndex.java new file mode 100644 index 0000000..29df4d5 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/ClassIndex.java @@ -0,0 +1,485 @@ +package com.flanksource.telltale.diag.hotpatch; + +import java.lang.instrument.Instrumentation; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Read-only view over the classes currently loaded in this JVM, grouped by classloader. Backs + * contract §1 ({@code /classes/loaders}) and §2 ({@code /classes}). + * + *

Loader ids are {@code Integer.toHexString(System.identityHashCode(cl))} — the same convention + * Arthas uses, so both backends address the same loader with the same id. The bootstrap loader + * ({@code null}) is {@code "0"}. + * + *

A loader id may also be a virtual id (see {@link LoaderProfile}), which stands for a + * whole family of loaders at once. Virtual ids are a listing filter: the classes they return still + * report the real loader that defined them, so decompiling or patching from a merged listing + * addresses a concrete loader. {@link #resolveForWrite} rejects them outright. + */ +public final class ClassIndex { + + public static final String BOOTSTRAP_HASH = "0"; + public static final String BOOTSTRAP_LABEL = "JDK bootstrap"; + public static final int DEFAULT_LIMIT = 200; + public static final int MAX_LIMIT = 1000; + + /** Tomcat's WebappClassLoaderBase.toString() prints the context on its own line. */ + private static final Pattern TOSTRING_CONTEXT = + Pattern.compile("(?m)^\\s*context:\\s*(\\S+)\\s*$"); + + /** + * Package prefixes owned by the Java platform and the EE APIs it ships: everything nobody + * browsing an application JVM is looking for. They are the bulk of {@code getAllLoadedClasses()} + * and drown the application classes, so the listing hides them unless a caller asks for them + * back. + * + *

Prefix-matched on the fully-qualified name, so {@code javax.} covers {@code javax.servlet} + * and {@code javax.persistence} alike. + */ + private static final List PLATFORM_PREFIXES = + Collections.unmodifiableList( + Arrays.asList( + "java.", + "javax.", + "jakarta.", + "jdk.", + "sun.", + "com.sun.", + "org.w3c.dom.", + "org.xml.sax.", + "org.omg.", + "org.ietf.jgss.", + "netscape.javascript.")); + + /** + * @return true when the name belongs to the JDK/EE platform rather than to an application. + */ + public static boolean isPlatform(String className) { + for (String prefix : PLATFORM_PREFIXES) { + if (className.startsWith(prefix)) { + return true; + } + } + return false; + } + + /** One loaded class: its binary name, its defining loader, and (in production) the live Class. */ + public static final class Entry { + private final String name; + private final ClassLoader loader; + private final Class type; + + public Entry(String name, ClassLoader loader) { + this(name, loader, null); + } + + private Entry(String name, ClassLoader loader, Class type) { + if (name == null || name.isEmpty()) { + throw new IllegalArgumentException("class name is required"); + } + this.name = name; + this.loader = loader; + this.type = type; + } + + public static Entry of(Class type) { + return new Entry(type.getName(), type.getClassLoader(), type); + } + + public String getName() { + return name; + } + + public ClassLoader getLoader() { + return loader; + } + + public String getLoaderHash() { + return hash(loader); + } + + /** The live class, or {@code null} for a synthetic entry. */ + public Class getType() { + return type; + } + } + + private final List entries; + private Map groupsByHash; + + public ClassIndex(List entries) { + this.entries = entries; + } + + public static ClassIndex from(Instrumentation instrumentation) { + Class[] loaded = instrumentation.getAllLoadedClasses(); + List entries = new ArrayList<>(loaded.length); + for (Class type : loaded) { + if (isBrowsable(type)) { + entries.add(Entry.of(type)); + } + } + return new ClassIndex(entries); + } + + /** + * Arrays and primitives have no class file, so they can be neither browsed nor decompiled. They + * also report JVM descriptor names — {@code String[].class.getName()} is {@code + * "[Ljava.lang.String;"} — which the package tree splits on {@code '.'} into junk nodes like + * {@code "[[Lcom"}. They have no place in a browse index. + * + *

Package-visible so this is testable without a live {@link Instrumentation}. + */ + static boolean isBrowsable(Class type) { + return !type.isArray() && !type.isPrimitive(); + } + + public static String hash(ClassLoader loader) { + return loader == null ? BOOTSTRAP_HASH : Integer.toHexString(System.identityHashCode(loader)); + } + + /** What this loader is, in words. See {@link LoaderProfile} for the rules. */ + public static String label(ClassLoader loader) { + return LoaderProfile.of(loader).getLabel(); + } + + /** Webapp context path when derivable, else {@code null}. */ + static String contextOf(ClassLoader loader) { + for (String accessor : new String[] {"getContextName", "getContextPath"}) { + String value = invokeAccessor(loader, accessor); + String normalized = normalizeContext(value); + if (normalized != null) { + return normalized; + } + } + Matcher matcher = TOSTRING_CONTEXT.matcher(String.valueOf(loader)); + return matcher.find() ? normalizeContext(matcher.group(1)) : null; + } + + private static String invokeAccessor(ClassLoader loader, String name) { + for (Class type = loader.getClass(); + type != null && type != Object.class; + type = type.getSuperclass()) { + try { + Method method = type.getDeclaredMethod(name); + if (method.getReturnType() != String.class) { + continue; + } + method.setAccessible(true); + return (String) method.invoke(loader); + } catch (ReflectiveOperationException | RuntimeException ignored) { + // Not a Tomcat webapp loader, or the accessor is guarded — fall through. + } + } + return null; + } + + private static String normalizeContext(String value) { + if (value == null) { + return null; + } + String trimmed = value.trim(); + if (trimmed.isEmpty() || "/".equals(trimmed) || "null".equals(trimmed)) { + return null; + } + return trimmed.startsWith("/") ? trimmed : "/" + trimmed; + } + + /** + * Contract §1: one row per loader, application code first. + * + *

Loader families that exist in the thousands — one loader per generated reflection accessor, + * one per runtime-compiled application rule — collapse into a single virtual row each, because a + * picker with 2,171 entries is not a picker. Members stay individually addressable by their real + * hash: the virtual id is a filter, not a rename. + */ + public List> loaders() { + Map> byLoader = new IdentityHashMap<>(); + List bootstrap = new ArrayList<>(); + for (Entry entry : entries) { + if (entry.loader == null) { + bootstrap.add(entry.name); + continue; + } + List names = byLoader.get(entry.loader); + if (names == null) { + names = new ArrayList<>(); + byLoader.put(entry.loader, names); + } + names.add(entry.name); + } + + return LoaderRows.build(byLoader, bootstrap); + } + + public boolean hasLoader(String loaderHash) { + for (Entry entry : entries) { + if (matchesLoader(entry, loaderHash)) { + return true; + } + } + return false; + } + + /** True when {@code loaderHash} names this entry's loader, or a family it belongs to. */ + private boolean matchesLoader(Entry entry, String loaderHash) { + if (entry.getLoaderHash().equals(loaderHash)) { + return true; + } + return LoaderProfile.isGroupId(loaderHash) + && loaderHash.equals(groupsByHash().get(entry.getLoaderHash())); + } + + /** + * Real loader hash to the virtual id it merges into, for the grouped loaders only. Built once per + * index because {@link LoaderProfile#of} can reach for the reflective webapp-context probe, which + * must not run per entry. + */ + private Map groupsByHash() { + if (groupsByHash == null) { + Map seen = new IdentityHashMap<>(); + Map found = new LinkedHashMap<>(); + for (Entry entry : entries) { + if (entry.loader == null || seen.put(entry.loader, Boolean.TRUE) != null) { + continue; + } + String group = LoaderProfile.of(entry.loader).getGroup(); + if (group != null) { + found.put(entry.getLoaderHash(), group); + } + } + groupsByHash = found; + } + return groupsByHash; + } + + /** + * Contract §2: {@code total} counts every match; {@code classes} is the sorted page. {@code + * platformExcluded} reports how many otherwise-matching classes the platform filter hid, so the + * omission is visible rather than silent; it is 0 when {@code includePlatform} is set. + */ + public Map page( + String loaderHash, String query, int limit, int offset, boolean includePlatform) { + Matches matches = matches(loaderHash, query, includePlatform); + matches.entries.sort(order(query)); + List> classes = new ArrayList<>(); + for (int i = offset; i < matches.entries.size() && classes.size() < limit; i++) { + Map row = new LinkedHashMap<>(); + row.put("name", matches.entries.get(i).getName()); + row.put("loader", matches.entries.get(i).getLoaderHash()); + classes.add(row); + } + Map out = new LinkedHashMap<>(); + out.put("total", matches.entries.size()); + out.put("offset", offset); + out.put("limit", limit); + out.put("platformExcluded", matches.platformExcluded); + out.put("classes", classes); + return out; + } + + /** + * Name order without a query. With one, a picker reads only the head of the page, so matches rank + * by how the simple name matches — exactly, as a prefix, anywhere — before a match that is only + * in the package path; within a rank named classes come before inner and anonymous ones. + */ + private static Comparator order(String query) { + Comparator byName = + Comparator.comparing(Entry::getName).thenComparing(Entry::getLoaderHash); + String needle = query == null ? "" : query.trim().toLowerCase(Locale.ROOT); + if (needle.isEmpty()) { + return byName; + } + return Comparator.comparingInt(entry -> matchRank(entry.getName(), needle)) + .thenComparing(entry -> entry.getName().indexOf('$') >= 0) + .thenComparing(byName); + } + + private static int matchRank(String className, String needle) { + String simpleName = + className.substring(className.lastIndexOf('.') + 1).toLowerCase(Locale.ROOT); + if (simpleName.equals(needle)) { + return 0; + } + if (simpleName.startsWith(needle)) { + return 1; + } + return simpleName.contains(needle) ? 2 : 3; + } + + /** Matching entries plus the count the platform filter removed from them. */ + private static final class Matches { + private final List entries = new ArrayList<>(); + private int platformExcluded; + } + + private Matches matches(String loaderHash, String query, boolean includePlatform) { + String needle = query == null ? "" : query.trim().toLowerCase(Locale.ROOT); + Matches matches = new Matches(); + for (Entry entry : entries) { + if (loaderHash != null && !matchesLoader(entry, loaderHash)) { + continue; + } + if (!needle.isEmpty() && !entry.getName().toLowerCase(Locale.ROOT).contains(needle)) { + continue; + } + if (!includePlatform && isPlatform(entry.getName())) { + matches.platformExcluded++; + continue; + } + matches.entries.add(entry); + } + return matches; + } + + /** + * Resolves one loaded class. When {@code loaderHash} is absent and the name is defined by several + * loaders, the loader that has loaded the most classes wins — that is the application loader + * rather than a stray child loader. + */ + public Entry resolve(String className, String loaderHash) { + List candidates = candidates(className, loaderHash); + if (candidates.isEmpty()) { + throw new NoSuchElementException( + loaderHash == null + ? "class not loaded: " + className + : "class not loaded by loader " + loaderHash + ": " + className); + } + if (candidates.size() == 1) { + return candidates.get(0); + } + // "Biggest loader wins" is a reasonable guess for a real hash or for no + // hash at all. It is not reasonable for a virtual id, which stands for + // hundreds of loaders on purpose — picking one of them silently would + // decompile or patch a copy the caller never named. + if (LoaderProfile.isGroupId(loaderHash)) { + throw new IllegalArgumentException( + className + + " is defined by " + + candidates.size() + + " loaders in " + + loaderHash + + "; name one of " + + loaderHashes(candidates)); + } + Map sizes = loaderSizes(); + candidates.sort( + Comparator.comparingInt((Entry e) -> sizes.getOrDefault(e.getLoaderHash(), 0)) + .reversed() + .thenComparing(Entry::getLoaderHash)); + return candidates.get(0); + } + + /** + * Resolves a class that is about to be modified. A virtual id is a listing filter and never a + * patch target: {@code ~compiled} covers hundreds of distinct loaders, so redefining a class "in" + * it would mutate whichever copy happened to sort first and report success either way. + */ + public Entry resolveForWrite(String className, String loaderHash) { + if (LoaderProfile.isGroupId(loaderHash)) { + throw new IllegalArgumentException( + loaderHash + + " is a merged view of several loaders and cannot be modified; name one of " + + loaderHashes(candidates(className, loaderHash))); + } + return resolve(className, loaderHash); + } + + /** Every loaded class with this name, narrowed to {@code loaderHash} when given. */ + List candidates(String className, String loaderHash) { + List candidates = new ArrayList<>(); + for (Entry entry : entries) { + if (!entry.getName().equals(className)) { + continue; + } + if (loaderHash != null && !matchesLoader(entry, loaderHash)) { + continue; + } + candidates.add(entry); + } + return candidates; + } + + private static List loaderHashes(List candidates) { + List hashes = new ArrayList<>(candidates.size()); + for (Entry candidate : candidates) { + hashes.add(candidate.getLoaderHash()); + } + Collections.sort(hashes); + return hashes; + } + + private Map loaderSizes() { + Map sizes = new LinkedHashMap<>(); + for (Entry entry : entries) { + String key = entry.getLoaderHash(); + sizes.put(key, sizes.getOrDefault(key, 0) + 1); + } + return sizes; + } + + /** + * Loaded {@code Outer$Inner} classes CFR needs alongside {@code Outer}. + * + *

Runtime-generated classes share the {@code Outer$} prefix without being nested types — a + * lambda body, or a Spring CGLIB proxy of the target. They are not part of the source CFR is + * reconstructing, so feeding CFR their bytes produces a decompilation of the wrong thing. + */ + public List innerClasses(Entry outer) { + String prefix = outer.getName() + "$"; + List inner = new ArrayList<>(); + for (Entry entry : entries) { + if (entry.getLoader() == outer.getLoader() + && entry.getName().startsWith(prefix) + && !entry.getName().contains("$$Lambda$") + && !entry.getName().contains("$$EnhancerBySpringCGLIB$$")) { + inner.add(entry); + } + } + inner.sort(Comparator.comparing(Entry::getName)); + return Collections.unmodifiableList(inner); + } + + public static int parseLimit(String raw) { + if (raw == null || raw.trim().isEmpty()) { + return DEFAULT_LIMIT; + } + int limit = parseInt(raw, "limit"); + if (limit < 1 || limit > MAX_LIMIT) { + throw new IllegalArgumentException( + "limit must be between 1 and " + MAX_LIMIT + ", got " + raw); + } + return limit; + } + + public static int parseOffset(String raw) { + if (raw == null || raw.trim().isEmpty()) { + return 0; + } + int offset = parseInt(raw, "offset"); + if (offset < 0) { + throw new IllegalArgumentException("offset must not be negative, got " + raw); + } + return offset; + } + + private static int parseInt(String raw, String field) { + try { + return Integer.parseInt(raw.trim()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(field + " must be an integer, got " + raw); + } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/ClassOperations.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/ClassOperations.java new file mode 100644 index 0000000..d235074 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/ClassOperations.java @@ -0,0 +1,399 @@ +package com.flanksource.telltale.diag.hotpatch; + +import java.io.IOException; +import java.lang.instrument.Instrumentation; +import java.util.*; + +public final class ClassOperations { + private final DecompilerService decompiler = new DecompilerService(); + private final CompilerService compiler = new CompilerService(); + private final MethodPatchService methodPatches = new MethodPatchService(); + + public synchronized Map compile(JsonBody body) throws IOException { + String className = body.required("class"); + body.required("source"); + ClassIndex.Entry entry = index().resolveForWrite(className, body.string("loader")); + if (body.string("method") == null) { + return compiler + .compile( + new CompilerService.Request(className, body.required("source"), entry.getType()) + .withClasspath(body.string("classpath"))) + .toMap(); + } + return compileMethod(body, className, entry).toMap(); + } + + public synchronized Map patch(JsonBody body) throws IOException { + String className = body.required("class"); + ClassIndex index = index(); + ClassIndex.Entry entry = index.resolveForWrite(className, body.string("loader")); + return body.string("method") == null + ? patchWholeClass(body, className, index, entry) + : patchMethod(body, className, index, entry); + } + + private Map patchWholeClass( + JsonBody body, String className, ClassIndex index, ClassIndex.Entry entry) { + CompilerService.Result compiled = + compiler.compile( + new CompilerService.Request(className, body.required("source"), entry.getType()) + .withClasspath(body.string("classpath"))); + Map out = new LinkedHashMap<>(compiled.toMap()); + if (!compiled.isSuccess()) { + out.put("applied", false); + return out; + } + registerLease(body, entry, className); + out.putAll(patchService().apply(index, entry, compiled.getClasses())); + out.put("applied", true); + return out; + } + + private Map patchMethod( + JsonBody body, String className, ClassIndex index, ClassIndex.Entry entry) { + MethodPatchService.Result compiled = compileMethod(body, className, entry); + Map out = new LinkedHashMap<>(compiled.toMap()); + out.put("method", compiled.getDescriptor()); + if (!compiled.isSuccess()) { + out.put("applied", false); + return out; + } + byte[] patched = methodPatches.splice(className, compiled, entry.getType().getClassLoader()); + registerLease(body, entry, className); + out.putAll(patchService().apply(index, entry, Collections.singletonMap(className, patched))); + out.put("applied", true); + return out; + } + + private MethodPatchService.Result compileMethod( + JsonBody body, String className, ClassIndex.Entry entry) { + Class target = liveClass(entry, className); + return methodPatches.compile( + className, + target, + PatchService.sourceBytes(InstrumentationService.instrumentation(), target), + MethodSelector.parse(body.required("method")), + body.required("source"), + body.string("classpath")); + } + + private static void registerLease(JsonBody body, ClassIndex.Entry entry, String className) { + int seconds = body.signedInteger("leaseSeconds"); + if (seconds < 0) { + return; + } + AgentBridge.lease(entry.getLoaderHash(), className, seconds); + } + + public synchronized Map instrument(JsonBody body) throws IOException { + String className = body.required("class"); + ClassIndex index = index(); + ClassIndex.Entry entry = index.resolveForWrite(className, body.string("loader")); + Class target = liveClass(entry, className); + JsonBody probe = body.object("probe"); + requireAgentVisibleTo(target); + requireProbeApiFor(probe.fields()); + + String probeKey = probe.required("key"); + String selector = body.required("method"); + MethodSelector.parse(selector); + registerLease(body, entry, className); + Map options = new LinkedHashMap<>(probe.fields()); + options.put("method", selector); + Map opened = + AgentBridge.openProbe(probeKey, entry.getLoaderHash(), className, options); + if (!Boolean.TRUE.equals(opened.get("opened"))) { + throw new IllegalStateException("cannot record " + className + ": " + opened.get("error")); + } + + // Armed by retransformation through ProbeTransformations, not by redefinition: a redefined + // class loses its probes to the next retransformation of it, whoever makes it. The rebuild + // weaves EVERY span still recording on the class — a class is rewritten whole. + Map out; + InstrumentService.Result rewritten; + try { + rewritten = ProbeTransformations.arm(InstrumentationService.instrumentation(), entry, probeKey); + out = new LinkedHashMap<>(patchService().heldStatus(entry)); + } catch (RuntimeException e) { + AgentBridge.releaseProbe(probeKey, entry.getLoaderHash(), className); + rearmSurvivors(entry, target, e); + throw e; + } + out.put("methods", methodRows(rewritten)); + out.put("probeKey", probeKey); + out.put("generation", opened.get("generation")); + out.put("capacity", opened.get("capacity")); + out.put("alloc", Boolean.TRUE.equals(opened.get("alloc"))); + out.put("gc", Boolean.TRUE.equals(opened.get("gc"))); + out.put("instrumented", Boolean.TRUE.equals(out.get("patched"))); + return out; + } + + /** The root-call filter options {@code probe} names, or null when it names none. */ + public static String filterOptions(Map probe) { + StringBuilder named = new StringBuilder(); + for (String option : Arrays.asList("minCost", "exceptionsOnly", "match")) { + if (probe.get(option) != null) { + named.append(named.length() == 0 ? "" : ", ").append(option); + } + } + return named.length() == 0 ? null : named.toString(); + } + + /** Refuses each opt-in probe option the attached agent is too old to honour. */ + private static void requireProbeApiFor(Map probe) { + AgentBridge.requireProbeApi(); + if (Boolean.TRUE.equals(probe.get("alloc"))) { + AgentBridge.requireProbeApiFor("alloc", AgentBridge.ALLOC_PROBE_API); + } + if (Boolean.TRUE.equals(probe.get("gc"))) { + AgentBridge.requireProbeApiFor("gc", AgentBridge.GC_PROBE_API); + } + String filters = filterOptions(probe); + if (filters != null) { + AgentBridge.requireProbeApiFor(filters, AgentBridge.FILTER_PROBE_API); + } + } + + /** + * Rebuilds the class with the spans still recording on it, after this arm failed and its own + * probe was released. A failed rebuild leaves the JVM's starting bytes in place, which carry no + * probes at all — so one span's failure would otherwise silently disarm every other span already + * recording on the class. A rebuild that fails too travels with the arm failure, never instead of + * it. + */ + private static void rearmSurvivors( + ClassIndex.Entry entry, Class target, RuntimeException failure) { + try { + if (!ProbeTransformations.contains(target)) { + return; + } + List remaining = AgentBridge.probeKeysOwnedBy(entry.getLoaderHash(), entry.getName()); + if (remaining.isEmpty()) { + ProbeTransformations.restore(target); + } else { + ProbeTransformations.arm(InstrumentationService.instrumentation(), entry, remaining.get(0)); + } + } catch (RuntimeException | LinkageError rearm) { + failure.addSuppressed(rearm); + } + } + + private static List> methodRows(InstrumentService.Result rewritten) { + List> rows = new ArrayList<>(); + for (MethodSelector.Match match : rewritten.getMatched()) { + Map row = new LinkedHashMap<>(); + row.put("name", match.getMethod().getName()); + row.put("descriptor", match.getMethod().getDescriptor()); + row.put("instrumented", match.isInstrumentable()); + if (!match.isInstrumentable()) { + row.put("skipped", match.getSkipped()); + } + rows.add(row); + } + return rows; + } + + public synchronized Map restore(JsonBody body) throws IOException { + String className = body.required("class"); + ClassIndex index = index(); + ClassIndex.Entry entry = index.resolveForWrite(className, body.string("loader")); + String loaderHash = entry.getLoaderHash(); + String probeKey = body.optionalObject("probe").string("key"); + + if (probeKey != null) { + Map released = AgentBridge.releaseProbe(probeKey, loaderHash, className); + List stillRecording = AgentBridge.probeKeysOwnedBy(loaderHash, className); + if (!stillRecording.isEmpty()) { + ProbeTransformations.arm(InstrumentationService.instrumentation(), entry, stillRecording.get(0)); + Map held = + new LinkedHashMap<>(patchService().status(index, loaderHash, className)); + held.put("restored", Collections.emptyList()); + held.put("released", probeKey); + held.put("heldBy", stillRecording); + held.put("probesClosed", probesClosed(released)); + return held; + } + Map out = new LinkedHashMap<>(patchService().restore(index, + new PatchService.RestoreRequest(entry, PatchService.RestoreScope.PROBES))); + out.put("released", probeKey); + out.put("heldBy", Collections.emptyList()); + out.put("probesClosed", probesClosed(released)); + return out; + } + + boolean sourceOnly = body.string("source") != null; + List> closed = sourceOnly ? Collections.emptyList() : AgentBridge.closeProbesOwnedBy(loaderHash, className); + Map out = new LinkedHashMap<>(patchService().restore(index, + new PatchService.RestoreRequest(entry, sourceOnly ? PatchService.RestoreScope.SOURCE : PatchService.RestoreScope.ALL))); + out.put("probesClosed", closed); + return out; + } + + private static List> probesClosed(Map released) { + if (!Boolean.TRUE.equals(released.get("closed"))) { + return Collections.emptyList(); + } + return Collections.singletonList(released); + } + + public synchronized Map lease(JsonBody body) throws IOException { + String className = body.required("class"); + ClassIndex.Entry entry = index().resolveForWrite(className, body.string("loader")); + Map held = patchService().heldStatus(entry); + String probeKey = body.optionalObject("probe").string("key"); + if (body.string("source") != null) held.put("patched", SourceTransformations.contains(liveClass(entry, className))); + if (probeKey != null) held.put("patched", AgentBridge.probeKeysOwnedBy(entry.getLoaderHash(), className).contains(probeKey)); + if (!Boolean.TRUE.equals(held.get("patched"))) return held; + AgentBridge.lease(entry.getLoaderHash(), className, body.integer("seconds")); + return patchService().heldStatus(entry); + } + + private static final int MAX_PROBE_LIMIT = 10_000; + private static final int DEFAULT_PROBE_LIMIT = 1_000; + + public Map probes(JsonBody body) { + String key = body.string("key"); + if (key == null) { + throw new IllegalArgumentException("key is required"); + } + if (body.string("drain") != null) { + throw new IllegalArgumentException( + "drain= was replaced by since=: a read no longer removes" + + " what it returns, and reports what the ring evicted instead"); + } + return AgentBridge.drain(key, cursor(body), limit(body)); + } + + public Map freezeProbe(JsonBody body) throws IOException { + return AgentBridge.freezeProbe(body.required("key")); + } + + public Map probeKeys() { + Map out = new LinkedHashMap<>(); + out.put("keys", AgentBridge.probeKeys()); + return out; + } + + static long cursor(JsonBody body) { + Object since = body.fields().get("since"); + if (since == null) { + return 0L; + } + try { + if (!(since instanceof Number)) + throw new IllegalArgumentException("since must be an integer"); + long parsed = new java.math.BigDecimal(since.toString()).longValueExact(); + if (parsed < 0L) { + throw new IllegalArgumentException("since must not be negative, got " + since); + } + return parsed; + } catch (ArithmeticException | NumberFormatException e) { + throw new IllegalArgumentException("since must be a sequence number, got " + since, e); + } + } + + private static int limit(JsonBody body) { + if (body.fields().get("limit") == null) { + return DEFAULT_PROBE_LIMIT; + } + int parsed = body.integer("limit"); + if (parsed <= 0) { + throw new IllegalArgumentException("limit must be positive, got " + parsed); + } + return Math.min(parsed, MAX_PROBE_LIMIT); + } + + private static void requireAgentVisibleTo(Class target) { + try { + Class.forName(InstrumentationService.AGENT_CLASS, false, target.getClassLoader()); + } catch (ClassNotFoundException e) { + throw new IllegalStateException( + target.getName() + + " is defined by a class loader that cannot see " + + InstrumentationService.AGENT_CLASS + + " on the system class path, so instrumented code" + + " would fail with NoClassDefFoundError the first time the method ran", + e); + } + } + + private static byte[] liveBytecode(Class target) { + return new BytecodeSource(InstrumentationService.instrumentation()).bytecodeOf(target); + } + + private static Class liveClass(ClassIndex.Entry entry, String className) { + Class type = entry.getType(); + if (type == null) { + throw new IllegalStateException("no live Class handle for " + className); + } + return type; + } + + private static PatchService patchService() { + return new PatchService(InstrumentationService.instrumentation()); + } + + public synchronized Map source(JsonBody body) { + String className = body.string("class"); + if (className == null) { + throw new IllegalArgumentException("class is required"); + } + ClassIndex index = index(); + ClassIndex.Entry entry = index.resolve(className, loader(body, index)); + Class type = liveClass(entry, className); + + BytecodeSource bytecode = new BytecodeSource(InstrumentationService.instrumentation()); + byte[] bytes = bytecode.bytecodeOf(type); + String digest = BytecodeSource.digest(bytes); + DecompilerService.Request request = + new DecompilerService.Request(className, digest, bytes) + .withMethodName(body.string("method")); + for (ClassIndex.Entry inner : index.innerClasses(entry)) { + if (inner.getType() == null) { + continue; + } + try { + request.withInnerClass(inner.getName(), bytecode.bytecodeOf(inner.getType())); + } catch (IllegalStateException e) { + continue; + } + } + + DecompilerService.Result result = decompiler.decompile(request); + Map out = new LinkedHashMap<>(); + out.put("class", className); + out.put("loader", entry.getLoaderHash()); + out.put("source", result.getSource()); + out.put("methods", result.getMethods()); + out.put("sourceLineNumbers", result.getSourceLineNumbers()); + out.put("lineRows", result.getLineRows()); + out.put("digest", digest); + out.put("decompiler", DecompilerService.NAME); + out.put("durationMs", result.getDurationMs()); + out.put("cached", result.isCached()); + return out; + } + + public synchronized byte[] bytecode(JsonBody body) { + String className = body.string("class"); + if (className == null) { + throw new IllegalArgumentException("class is required"); + } + ClassIndex index = index(); + return liveBytecode(liveClass(index.resolve(className, loader(body, index)), className)); + } + + private static ClassIndex index() { + Instrumentation instrumentation = InstrumentationService.instrumentation(); + return ClassIndex.from(instrumentation); + } + + private static String loader(JsonBody body, ClassIndex index) { + String loader = body.string("loader"); + if (loader != null && !index.hasLoader(loader)) { + throw new NoSuchElementException("loader not found: " + loader); + } + return loader; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/CompilerService.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/CompilerService.java new file mode 100644 index 0000000..3a8cf6c --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/CompilerService.java @@ -0,0 +1,469 @@ +package com.flanksource.telltale.diag.hotpatch; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.net.URI; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.CodeSource; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import javax.tools.Diagnostic; +import javax.tools.DiagnosticCollector; +import javax.tools.FileObject; +import javax.tools.ForwardingJavaFileManager; +import javax.tools.JavaCompiler; +import javax.tools.JavaFileManager; +import javax.tools.JavaFileObject; +import javax.tools.SimpleJavaFileObject; +import javax.tools.StandardJavaFileManager; +import javax.tools.ToolProvider; + +/** + * Compiles edited Java source against the classpath of the JVM this WAR runs in, producing the + * class bytes {@link PatchService} redefines. + * + *

javac is loaded the same way the Attach API is: on Java 8 it lives in {@code tools.jar}, + * absent from a JRE image, so it comes from the JDK {@link InstrumentationService#findJdk()} finds + * — including the one application CLI side-loads to {@code /tmp/jdk}. Unlike the Attach API, javac + * is pure Java, so loading it into a child classloader is enough; no native library is involved. + * + *

The classpath is derived from the target class's own {@link CodeSource} rather than from + * {@code URLClassLoader.getURLs()}, which TomEE's webapp loader does not usefully implement. It is + * reported back on every result so a compile failure caused by a missing entry is diagnosable + * instead of mysterious. + */ +public final class CompilerService { + + /** javac's entry point inside tools.jar, used when the JVM has no built-in compiler. */ + private static final String JAVAC_TOOL = "com.sun.tools.javac.api.JavacTool"; + + /** Kept so repeated compiles do not re-open tools.jar. */ + private static volatile JavaCompiler compiler; + + /** What to compile: one top-level class's full source, resolved against one loader. */ + public static final class Request { + private final String className; + private final String source; + private final ClassLoader loader; + private final Class target; + private String classpath; + + public Request(String className, String source, Class target) { + if (className == null || className.trim().isEmpty()) { + throw new IllegalArgumentException("class is required"); + } + if (source == null || source.trim().isEmpty()) { + throw new IllegalArgumentException("source is required for " + className); + } + this.className = className.trim(); + this.source = source; + this.target = target; + this.loader = target == null ? null : target.getClassLoader(); + } + + /** Overrides the derived classpath entirely; for when derivation gets it wrong. */ + public Request withClasspath(String value) { + this.classpath = value == null || value.trim().isEmpty() ? null : value.trim(); + return this; + } + + public String getClassName() { + return className; + } + } + + /** One javac message, positioned in the submitted source. */ + public static final class Message { + private final String kind; + private final long line; + private final long column; + private final String code; + private final String text; + + Message(String kind, long line, long column, String code, String text) { + this.kind = kind; + this.line = line; + this.column = column; + this.code = code; + this.text = text; + } + + public Map toMap() { + Map out = new LinkedHashMap<>(); + out.put("kind", kind); + out.put("line", line); + out.put("column", column); + out.put("code", code == null ? "" : code); + out.put("message", text == null ? "" : text); + return out; + } + + public boolean isError() { + return "ERROR".equals(kind); + } + + @Override + public String toString() { + return kind + " " + line + ":" + column + " " + text; + } + } + + /** + * The outcome of one compile. {@code classes} carries every class file javac produced — the + * top-level class plus its nested and anonymous classes, all of which have to be redefined + * together. + */ + public static final class Result { + private final boolean success; + private final Map classes; + private final List messages; + private final String classpath; + + Result(boolean success, Map classes, List messages, String classpath) { + this.success = success; + this.classes = classes; + this.messages = messages; + this.classpath = classpath; + } + + public boolean isSuccess() { + return success; + } + + public Map getClasses() { + return Collections.unmodifiableMap(classes); + } + + public List getMessages() { + return Collections.unmodifiableList(messages); + } + + public String getClasspath() { + return classpath; + } + + /** Contract shape for the compile/patch responses. */ + public Map toMap() { + List> diagnostics = new ArrayList<>(); + for (Message message : messages) { + diagnostics.add(message.toMap()); + } + Map out = new LinkedHashMap<>(); + out.put("success", success); + out.put("classes", new ArrayList<>(classes.keySet())); + out.put("diagnostics", diagnostics); + out.put( + "classpathEntries", + classpath.isEmpty() ? 0 : classpath.split(File.pathSeparator, -1).length); + return out; + } + } + + public Result compile(Request request) { + String classpath = request.classpath != null ? request.classpath : classpathFor(request.target); + JavaCompiler javac = compiler(); + DiagnosticCollector collected = new DiagnosticCollector<>(); + Map output = new LinkedHashMap<>(); + + StandardJavaFileManager standard = + javac.getStandardFileManager(collected, Locale.ROOT, StandardCharsets.UTF_8); + JavaFileManager capturing = new CapturingFileManager(standard, output); + List options = + new ArrayList<>( + Arrays.asList("-classpath", classpath, "-g", "-proc:none", "-Xlint:-options")); + // This WAR patches Java 8 application classes. A JDK 21 test/diagnostic host + // must not emit JDK 21 bytecode (or call newer JDK APIs) into a class + // that the Java 8 target JVM has to redefine and run. + if (javax.lang.model.SourceVersion.latestSupported() + .compareTo(javax.lang.model.SourceVersion.RELEASE_8) + > 0) { + options.addAll(Arrays.asList("--release", "8")); + } else { + // javac 8 predates --release; the host API is already Java 8. + options.addAll(Arrays.asList("-source", "8", "-target", "8")); + } + + boolean ok; + try { + ok = + Boolean.TRUE.equals( + javac + .getTask( + null, + capturing, + collected, + options, + null, + Collections.singletonList(new SourceFile(request.className, request.source))) + .call()); + } catch (RuntimeException e) { + throw new IllegalStateException( + "compile of " + + request.className + + " failed inside javac: " + + e + + " (classpath had " + + entryCount(classpath) + + " entries)", + e); + } finally { + closeQuietly(capturing); + } + + List messages = new ArrayList<>(); + for (Diagnostic diagnostic : collected.getDiagnostics()) { + messages.add( + new Message( + diagnostic.getKind().name(), + diagnostic.getLineNumber(), + diagnostic.getColumnNumber(), + diagnostic.getCode(), + diagnostic.getMessage(Locale.ROOT))); + } + // javac can report success with no output when the source declares a + // different type than the caller named; that is a failure to us. + if (ok && !output.containsKey(request.className)) { + messages.add( + new Message( + "ERROR", + -1, + -1, + "telltale.diag.wrong-class", + "compiled successfully but produced no class named " + + request.className + + " (produced " + + output.keySet() + + ") — the source must declare that exact type")); + ok = false; + } + return new Result(ok, output, messages, classpath); + } + + /** + * Builds the compile classpath from the target class's code source: for an exploded webapp that + * is {@code WEB-INF/classes}, whose sibling {@code WEB-INF/lib} holds the jars the class was + * compiled against. Any {@link URLClassLoader} ancestors that do answer {@code getURLs()} are + * added too, and {@code java.class.path} last. + */ + static String classpathFor(Class target) { + Set entries = new LinkedHashSet<>(); + Path codeSource = codeSourceOf(target); + if (codeSource != null) { + entries.add(codeSource.toString()); + // /WEB-INF/classes -> /WEB-INF/lib/*.jar + addJars( + entries, codeSource.getParent() == null ? null : codeSource.getParent().resolve("lib")); + // A class loaded from a jar sits beside the rest of them. + if (codeSource.toString().endsWith(".jar")) { + addJars(entries, codeSource.getParent()); + } + } + for (ClassLoader loader = target == null ? null : target.getClassLoader(); + loader != null; + loader = parentOf(loader)) { + addLoaderURLs(entries, loader); + } + for (String entry : System.getProperty("java.class.path", "").split(File.pathSeparator)) { + if (!entry.trim().isEmpty()) { + entries.add(entry.trim()); + } + } + return String.join(File.pathSeparator, entries); + } + + private static Path codeSourceOf(Class target) { + if (target == null) { + return null; + } + try { + CodeSource source = target.getProtectionDomain().getCodeSource(); + if (source == null || source.getLocation() == null) { + return null; + } + return Paths.get(source.getLocation().toURI()); + } catch (RuntimeException | java.net.URISyntaxException e) { + // A non-file: code source (or a SecurityManager) simply contributes nothing. + return null; + } + } + + private static void addJars(Set entries, Path directory) { + if (directory == null || !Files.isDirectory(directory)) { + return; + } + try (java.util.stream.Stream children = Files.list(directory)) { + children + .filter(path -> path.toString().endsWith(".jar")) + .sorted() + .forEach(path -> entries.add(path.toString())); + } catch (IOException e) { + throw new UncheckedIOException("cannot list " + directory + " for the compile classpath", e); + } + } + + /** + * TomEE's webapp loader throws from {@code getURLs()}, so this is best-effort by design: a loader + * that cannot describe itself contributes nothing rather than failing the compile. + */ + private static void addLoaderURLs(Set entries, ClassLoader loader) { + if (!(loader instanceof URLClassLoader)) { + return; + } + URL[] urls; + try { + urls = ((URLClassLoader) loader).getURLs(); + } catch (RuntimeException | LinkageError e) { + return; + } + if (urls == null) { + return; + } + for (URL url : urls) { + try { + entries.add(Paths.get(url.toURI()).toString()); + } catch (RuntimeException | java.net.URISyntaxException ignored) { + // Not a file: URL — nothing javac can read. + } + } + } + + private static ClassLoader parentOf(ClassLoader loader) { + try { + return loader.getParent(); + } catch (RuntimeException e) { + return null; + } + } + + static int entryCount(String classpath) { + return classpath == null || classpath.isEmpty() + ? 0 + : classpath.split(File.pathSeparator, -1).length; + } + + /** + * The JVM's own compiler when it has one (a JDK, or Java 9+), else javac out of the discovered + * JDK's tools.jar. + */ + private static JavaCompiler compiler() { + JavaCompiler existing = compiler; + if (existing != null) { + return existing; + } + synchronized (CompilerService.class) { + if (compiler == null) { + compiler = loadCompiler(); + } + return compiler; + } + } + + private static JavaCompiler loadCompiler() { + JavaCompiler builtin = ToolProvider.getSystemJavaCompiler(); + if (builtin != null) { + return builtin; + } + InstrumentationService.Jdk jdk = InstrumentationService.findJdk(); + if (jdk.toolsJar == null) { + throw new IllegalStateException( + "no java compiler: this JVM has none built in and " + + jdk.home + + " has no lib/tools.jar to load javac from"); + } + try { + URLClassLoader tools = + new URLClassLoader( + new URL[] {jdk.toolsJar.toUri().toURL()}, CompilerService.class.getClassLoader()); + Object tool = Class.forName(JAVAC_TOOL, true, tools).getMethod("create").invoke(null); + return (JavaCompiler) tool; + } catch (ReflectiveOperationException | IOException | RuntimeException e) { + throw new IllegalStateException( + "no java compiler: " + + JAVAC_TOOL + + " could not be loaded from " + + jdk.toolsJar + + ": " + + e, + e); + } + } + + private static void closeQuietly(JavaFileManager manager) { + try { + manager.close(); + } catch (IOException | RuntimeException ignored) { + // The compile result is already in hand; a failed close must not mask it. + } + } + + /** The submitted source, handed to javac without ever touching the filesystem. */ + private static final class SourceFile extends SimpleJavaFileObject { + private final String source; + + SourceFile(String className, String source) { + super( + URI.create("string:///" + className.replace('.', '/') + Kind.SOURCE.extension), + Kind.SOURCE); + this.source = source; + } + + @Override + public CharSequence getCharContent(boolean ignoreEncodingErrors) { + return source; + } + } + + /** Collects javac's class output in memory, keyed by binary name. */ + private static final class CapturingFileManager + extends ForwardingJavaFileManager { + private final Map output; + + CapturingFileManager(StandardJavaFileManager delegate, Map output) { + super(delegate); + this.output = output; + } + + @Override + public JavaFileObject getJavaFileForOutput( + Location location, String className, JavaFileObject.Kind kind, FileObject sibling) { + return new ClassOutput(className, output); + } + } + + private static final class ClassOutput extends SimpleJavaFileObject { + private final String className; + private final Map output; + + ClassOutput(String className, Map output) { + super( + URI.create("bytes:///" + className.replace('.', '/') + Kind.CLASS.extension), Kind.CLASS); + this.className = className; + this.output = output; + } + + @Override + public OutputStream openOutputStream() { + return new ByteArrayOutputStream() { + @Override + public void close() { + output.put(className, toByteArray()); + } + }; + } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/DecompilerService.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/DecompilerService.java new file mode 100644 index 0000000..b490633 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/DecompilerService.java @@ -0,0 +1,399 @@ +package com.flanksource.telltale.diag.hotpatch; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; +import org.benf.cfr.reader.api.CfrDriver; +import org.benf.cfr.reader.api.OutputSinkFactory; +import org.benf.cfr.reader.api.SinkReturns; + +/** + * Decompiles live class bytes with CFR, mirroring Arthas's {@code Decompiler} (CfrDriver + an + * in-memory {@link OutputSinkFactory}). + * + *

CFR 0.152 is the newest release on Maven Central and the version Arthas pins; its own class + * files are major 50, so it loads on the Temurin 8 JVM this WAR is hot-deployed into. + * + *

{@code CfrDriver.analyse} takes file paths, so the bytes are written to a scratch directory + * first — together with any loaded {@code Outer$Inner} classes, which CFR resolves by looking + * beside the analysed file. + */ +public final class DecompilerService { + + public static final String CFR_VERSION = "0.152"; + public static final String NAME = "cfr-" + CFR_VERSION; + + private static final int CACHE_ENTRIES = 64; + + private final Map cache = + Collections.synchronizedMap( + new LinkedHashMap(16, 0.75f, true) { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > CACHE_ENTRIES; + } + }); + + /** What to decompile. Inner classes and a single method name are optional. */ + public static final class Request { + private final String className; + private final String digest; + private final byte[] bytes; + private final Map innerClasses = new LinkedHashMap<>(); + private String methodName; + + public Request(String className, String digest, byte[] bytes) { + if (className == null || className.isEmpty()) { + throw new IllegalArgumentException("class is required"); + } + if (bytes == null || bytes.length == 0) { + throw new IllegalArgumentException("no class bytes supplied for " + className); + } + this.className = className; + this.digest = digest; + this.bytes = bytes; + } + + public Request withInnerClass(String name, byte[] classBytes) { + innerClasses.put(name, classBytes); + return this; + } + + public Request withMethodName(String name) { + this.methodName = name == null || name.trim().isEmpty() ? null : name.trim(); + return this; + } + + String cacheKey() { + return digest + "#" + (methodName == null ? "" : methodName); + } + } + + /** Decompiled source plus the method index and line map built from CFR's line-number mapping. */ + public static final class Result { + private final String source; + private final List> methods; + private final int[] sourceLineNumbers; + private final Map lineRows; + private final long durationMs; + private final boolean cached; + + Result(Cached decompiled, long durationMs, boolean cached) { + this.source = decompiled.source; + this.methods = decompiled.methods; + this.sourceLineNumbers = decompiled.sourceLineNumbers; + this.lineRows = decompiled.lineRows; + this.durationMs = durationMs; + this.cached = cached; + } + + public String getSource() { + return source; + } + + public List> getMethods() { + return methods; + } + + /** See {@link #sourceLineNumbers}; a copy, since the cache holds the original. */ + public int[] getSourceLineNumbers() { + return sourceLineNumbers.clone(); + } + + /** See {@link #lineRows}; unmodifiable, since the cache holds the original. */ + public Map getLineRows() { + return lineRows; + } + + public long getDurationMs() { + return durationMs; + } + + public boolean isCached() { + return cached; + } + } + + /** A decompile that failed; the servlet maps this to 502 with the message. */ + public static final class DecompileException extends RuntimeException { + DecompileException(String message, Throwable cause) { + super(message, cause); + } + } + + private static final class Cached { + final String source; + final List> methods; + final int[] sourceLineNumbers; + final Map lineRows; + + Cached( + String source, + List> methods, + int[] sourceLineNumbers, + Map lineRows) { + this.source = source; + this.methods = methods; + this.sourceLineNumbers = sourceLineNumbers; + this.lineRows = Collections.unmodifiableMap(lineRows); + } + } + + public Result decompile(Request request) { + long started = System.nanoTime(); + Cached hit = cache.get(request.cacheKey()); + if (hit != null) { + return new Result(hit, elapsedMs(started), true); + } + Capture capture; + try { + capture = analyse(request); + } catch (Throwable t) { + // CFR is a third party parsing hostile bytecode; nothing it throws — + // including Errors from a malformed constant pool — may reach the container. + throw new DecompileException("decompile of " + request.className + " failed: " + t, t); + } + String source = capture.source.toString(); + if (source.trim().isEmpty()) { + throw new DecompileException( + "decompile of " + + request.className + + " produced no source" + + (capture.problems.isEmpty() ? "" : ": " + String.join("; ", capture.problems)), + null); + } + Cached decompiled = + new Cached( + source, + methodIndex(capture.mappings, source), + sourceLineNumbers(capture.mappings, source), + lineRows(capture.mappings, source)); + cache.put(request.cacheKey(), decompiled); + return new Result(decompiled, elapsedMs(started), false); + } + + private Capture analyse(Request request) throws IOException { + Path scratch = Files.createTempDirectory("telltale-diag-cfr"); + try { + Path target = scratch.resolve(classFileName(request.className)); + Files.write(target, request.bytes); + for (Map.Entry inner : request.innerClasses.entrySet()) { + Files.write(scratch.resolve(classFileName(inner.getKey())), inner.getValue()); + } + Capture capture = new Capture(); + CfrDriver driver = + new CfrDriver.Builder() + .withOptions(options(request.methodName)) + .withOutputSink(capture) + .build(); + driver.analyse(Collections.singletonList(target.toAbsolutePath().toString())); + return capture; + } finally { + deleteRecursively(scratch); + } + } + + private static Map options(String methodName) { + Map options = new LinkedHashMap<>(); + options.put("showversion", "false"); + // Arthas keeps unicode literals verbatim rather than re-escaping them. + options.put("hideutf", "false"); + // Emits the LINENUMBER sink the method index is built from. + options.put("trackbytecodeloc", "true"); + if (methodName != null) { + options.put("methodname", methodName); + } + return options; + } + + static String classFileName(String binaryName) { + return binaryName.substring(binaryName.lastIndexOf('.') + 1) + ".class"; + } + + /** + * Contract §3 {@code methods[]}. CFR emits one mapping per method, keyed by bytecode + * offset with the emitted output line as the value ( {@code getClassFileMappings} is the + * same keys against the original source lines). The lowest value is therefore the first line of + * that method's body in {@code source}, 1-based, and the highest is the last line of it that + * carries an instruction. + * + *

Both ends are reported because neither can be recovered afterwards. A caller wanting to show + * one method out of a large class has to derive the end from the next method's start, + * and that is wrong here: CFR emits a mapping per method including synthetics, so {@code + * lambda$foo$0}'s lowest line lands inside {@code foo} and this list is not monotonic in + * the way that heuristic assumes. + * + *

{@code endLine} is the last mapped line, not the closing brace — the brace carries no + * instruction — so a reader expanding this to a syntactic block still has to walk out to it. + */ + static List> methodIndex( + List mappings, String source) { + int lines = source.split("\n", -1).length; + List> methods = new ArrayList<>(); + for (SinkReturns.LineNumberMapping mapping : mappings) { + NavigableMap outputLines = mapping.getMappings(); + if (outputLines == null || outputLines.isEmpty()) { + continue; + } + int line = Collections.min(outputLines.values()); + if (line < 1 || line > lines) { + continue; + } + Map row = new LinkedHashMap<>(); + row.put("name", mapping.methodName()); + row.put("descriptor", mapping.methodDescriptor()); + row.put("line", line); + row.put("endLine", Math.min(Collections.max(outputLines.values()), lines)); + methods.add(row); + } + methods.sort(Comparator.comparingInt(row -> (Integer) row.get("line"))); + return methods; + } + + /** + * The original source line behind each decompiled row: entry {@code i} is row {@code i + 1}, and + * 0 where no instruction maps there — a brace, a declaration, a class compiled without a line + * table. + * + *

CFR keys both of its maps by bytecode offset: {@code getMappings} gives the decompiled row + * an offset landed on, {@code getClassFileMappings} the line the class file's line table gives + * that offset. Joining them on the offset is what lets a line from the line table — a call site a + * probe reported — find its row in the source a reader is actually looking at. An offset missing + * from the line table takes the entry at or before it, which is how the line table itself is + * meant to be read. A row several lines compiled into keeps the earliest, the first statement the + * reader sees. + */ + static int[] sourceLineNumbers(List mappings, String source) { + int[] lines = new int[source.split("\n", -1).length]; + for (SinkReturns.LineNumberMapping mapping : mappings) { + NavigableMap rows = mapping.getMappings(); + NavigableMap original = mapping.getClassFileMappings(); + if (rows == null || original == null || original.isEmpty()) { + continue; + } + for (Map.Entry offset : rows.entrySet()) { + int row = offset.getValue(); + Map.Entry line = original.floorEntry(offset.getKey()); + if (line == null || row < 1 || row > lines.length || line.getValue() < 1) { + continue; + } + int index = row - 1; + if (lines[index] == 0 || line.getValue() < lines[index]) { + lines[index] = line.getValue(); + } + } + } + return lines; + } + + /** + * The decompiled row, 1-based, each original source line landed on — the first, when CFR spread + * one line over several rows. + * + *

{@link #sourceLineNumbers} cannot say this. A row several lines were folded into is labelled + * with the earliest alone, so a later line folded into it looks unmapped, and a reader walking up + * to the nearest labelled line lands on whatever row sits between: the wrong statement. + */ + static Map lineRows( + List mappings, String source) { + int lines = source.split("\n", -1).length; + Map rows = new TreeMap<>(); + for (SinkReturns.LineNumberMapping mapping : mappings) { + NavigableMap decompiled = mapping.getMappings(); + NavigableMap original = mapping.getClassFileMappings(); + if (decompiled == null || original == null || original.isEmpty()) { + continue; + } + for (Map.Entry offset : decompiled.entrySet()) { + int row = offset.getValue(); + Map.Entry line = original.floorEntry(offset.getKey()); + if (line == null || row < 1 || row > lines || line.getValue() < 1) { + continue; + } + Integer first = rows.get(line.getValue()); + if (first == null || row < first) { + rows.put(line.getValue(), row); + } + } + } + return rows; + } + + private static long elapsedMs(long startedNanos) { + return (System.nanoTime() - startedNanos) / 1_000_000L; + } + + private static void deleteRecursively(Path root) { + try (java.util.stream.Stream paths = Files.walk(root)) { + List ordered = new ArrayList<>(); + paths.forEach(ordered::add); + Collections.reverse(ordered); + for (Path path : ordered) { + Files.deleteIfExists(path); + } + } catch (IOException ignored) { + // Scratch files live under java.io.tmpdir; a failed cleanup is not worth failing a read. + } + } + + /** Collects CFR's output in memory instead of writing files. */ + private static final class Capture implements OutputSinkFactory { + final StringBuilder source = new StringBuilder(); + final List mappings = new ArrayList<>(); + final List problems = new ArrayList<>(); + + @Override + public List getSupportedSinks(SinkType sinkType, Collection available) { + List preferred; + switch (sinkType) { + case JAVA: + preferred = Arrays.asList(SinkClass.DECOMPILED, SinkClass.STRING); + break; + case LINENUMBER: + preferred = Arrays.asList(SinkClass.LINE_NUMBER_MAPPING, SinkClass.STRING); + break; + default: + preferred = Collections.singletonList(SinkClass.STRING); + break; + } + List supported = new ArrayList<>(); + for (SinkClass candidate : preferred) { + if (available.contains(candidate)) { + supported.add(candidate); + } + } + return supported.isEmpty() ? Collections.singletonList(SinkClass.STRING) : supported; + } + + @Override + @SuppressWarnings("unchecked") + public Sink getSink(SinkType sinkType, SinkClass sinkClass) { + if (sinkType == SinkType.JAVA && sinkClass == SinkClass.DECOMPILED) { + return (Sink) + (Sink) decompiled -> source.append(decompiled.getJava()); + } + if (sinkType == SinkType.JAVA && sinkClass == SinkClass.STRING) { + return (Sink) (Sink) source::append; + } + if (sinkType == SinkType.LINENUMBER && sinkClass == SinkClass.LINE_NUMBER_MAPPING) { + return (Sink) (Sink) mappings::add; + } + if (sinkType == SinkType.EXCEPTION) { + return (Sink) (Sink) value -> problems.add(String.valueOf(value)); + } + return value -> { + // SUMMARY and PROGRESS are noise for an API caller. + }; + } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/InstrumentService.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/InstrumentService.java new file mode 100644 index 0000000..188ea58 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/InstrumentService.java @@ -0,0 +1,654 @@ +package com.flanksource.telltale.diag.hotpatch; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.Label; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; +import org.objectweb.asm.commons.AdviceAdapter; +import org.objectweb.asm.commons.Method; + +/** + * Rewrites one method so it reports what it does, leaving what it does unchanged. + * + *

This is the half of hot patching that needs neither a compiler nor a decompiler. The original + * body is wrapped, not replaced: an entry probe, a timed exit probe, optionally the + * arguments and the returned value, and optionally a catch-all that records a propagating exception + * and rethrows it unchanged. Nothing about the method's behaviour moves. + * + *

Three consequences fall out of transforming the live bytes rather than recompiling + * reconstructed source, and they are the reason this path exists: + * + *

    + *
  • No JDK is needed in the pod — instrumentation never runs javac. + *
  • Weaving another agent already applied to the class survives, because every method except + * the target is copied verbatim (see {@link TargetLoaderClassWriter}) and the target's own + * instructions are kept and merely bracketed. + *
  • No method, field or class is added, so HotSpot's "method bodies only" restriction on + * redefinition is satisfied by construction. A body containing a lambda or an anonymous class + * — which the source path has to refuse — is ordinary here. + *
+ */ +public final class InstrumentService { + + /** Where the injected calls land; see {@code DiagAgent}, on the system class path. */ + static final Type AGENT = Type.getObjectType("com/flanksource/telltale/diag/agent/DiagAgent"); + + private static final Type THROWABLE = Type.getObjectType("java/lang/Throwable"); + + // The injected code names exactly one class — the agent — and four of its + // methods. Every additional name would be another way for a redefinition to + // fail to resolve inside application code, which is the one failure this + // subsystem must not cause. Timing moved into the agent for the same reason: + // one fewer injected call, and one fewer class to resolve. + private static final Method PROBE_ENTER = + new Method( + "probeEnter", + "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;I" + + "[Ljava/lang/Object;)J"); + private static final Method PROBE_EXIT = new Method("probeExit", "(Ljava/lang/String;J)V"); + private static final Method PROBE_RETURN = + new Method("probeReturn", "(Ljava/lang/String;JLjava/lang/Object;)Ljava/lang/Object;"); + private static final Method PROBE_THROW = + new Method("probeThrow", "(Ljava/lang/String;JLjava/lang/Throwable;)V"); + + /** + * Whether a call site belongs in the tree beneath an instrumented method. + * + *

Lifted out of the adapter so it can be asserted directly: reached only through a bytecode + * round-trip, each of these rules could only be tested by finding a class that happens to call + * the thing being excluded. + * + * @param skipJdk leave the platform out, arthas' {@code --skipJDKMethod} + */ + static boolean tracesCallsTo(String owner, String name, boolean skipJdk) { + if (owner == null || owner.equals(AGENT.getInternalName()) || isGenerated(owner, name)) { + return false; + } + return !skipJdk || !isPlatform(owner); + } + + /** + * A class the platform owns. + * + *

A superset of arthas' {@code java/}: the same argument covers the rest of the platform's own + * namespaces, and application's stack reaches all of them. + */ + private static boolean isPlatform(String owner) { + return owner.startsWith("java/") + || owner.startsWith("javax/") + || owner.startsWith("jdk/") + || owner.startsWith("sun/") + || owner.startsWith("com/sun/"); + } + + /** + * A class some framework generated rather than someone wrote. + * + *

The CGLIB markers are arthas' {@code AsmUtils.isEnhancerByCGLIB}; {@code FastClass} is the + * dispatch half of the same mechanism, {@code $$Lambda} is its {@code ClassUtils.isLambdaClass}, + * and {@code com/sun/proxy/} is the JDK dynamic proxy that {@link #isPlatform} would only catch + * while the platform is being skipped. + * + *

application weaves its aspects at COMPILE time rather than proxying at runtime, so the CGLIB + * markers never fire here and the weaver's own artifacts do instead: {@code + * ActivityBll$AjcClosure1}, the {@code ajc$} methods the weaver renames the original body to, and + * {@code aspectOf()}, the generated singleton accessor every advised call site invokes first. + * + *

Measured, not assumed. The first live capture of {@code ActivityBll.process} with calls on + * spent 83 of its 100 events on this plumbing — closure constructors and {@code aspectOf} alone + * were a third of it — leaving 17 for the method anyone was actually looking at. + * + *

The advice bodies themselves are NOT excluded. An aspect is code someone wrote and it does + * real work; what is dropped here is only the machinery that dispatches to it. + * + *

Unconditional, the way arthas keeps its box types unconditional: this is noise whether or + * not the platform is being reported, because it describes the interception rather than the call. + */ + private static boolean isGenerated(String owner, String name) { + return owner.contains("$$EnhancerBySpringCGLIB$$") + || owner.contains("$$EnhancerByCGLIB$$") + || owner.contains("$$FastClassBySpringCGLIB$$") + || owner.contains("$$FastClassByCGLIB$$") + || owner.contains("$$Lambda") + || owner.contains("$AjcClosure") + || owner.startsWith("com/sun/proxy/") + || owner.startsWith("org/aspectj/") + || isWeaverPlumbing(name); + } + + /** A method the AspectJ weaver generated rather than one anyone declared. */ + private static boolean isWeaverPlumbing(String name) { + return name != null + && (name.startsWith("ajc$") || "aspectOf".equals(name) || "hasAspect".equals(name)); + } + + /** What to instrument, and what it should report. */ + public static final class Request { + private final String className; + private final byte[] bytecode; + private final ClassLoader loader; + private final MethodSelector selector; + private final String probeKey; + private boolean captureArgs; + private boolean captureReturn; + private boolean captureException; + private boolean captureCalls; + + /** + * Leave the platform's own call sites out of the call tree. + * + *

True by default, matching arthas' {@code --skipJDKMethod true}. It only reads at all when + * {@link #withCalls} is on. + */ + private boolean skipJdk = true; + + /** + * @param bytecode the bytes the class is currently running (see {@link + * BytecodeSource}), never the bytes on disk. + * @param loader the loader that defined the class, used to compute frames. + */ + public Request( + String className, + byte[] bytecode, + ClassLoader loader, + MethodSelector selector, + String probeKey) { + if (className == null || className.trim().isEmpty()) { + throw new IllegalArgumentException("class is required"); + } + if (bytecode == null || bytecode.length == 0) { + throw new IllegalArgumentException("no bytecode to instrument for " + className); + } + if (selector == null) { + throw new IllegalArgumentException( + "a method selector is required to instrument " + className); + } + validateProbeKey(probeKey); + this.className = className.trim(); + this.bytecode = bytecode; + this.loader = loader; + this.selector = selector; + this.probeKey = probeKey.trim(); + } + + public Request withArgs(boolean value) { + this.captureArgs = value; + return this; + } + + public Request withReturn(boolean value) { + this.captureReturn = value; + return this; + } + + /** + * Report what the instrumented method CALLS, as nested frames. + * + *

Off by default, and deliberately: a method's call sites outnumber the method itself by an + * order of magnitude, and turning this on for a hot one is how a ring fills with events nobody + * asked for. It is what `arthas trace` gives and `watch` does not, and the flag is the line + * between the two. + */ + public Request withCalls(boolean value) { + this.captureCalls = value; + return this; + } + + public Request withException(boolean value) { + this.captureException = value; + return this; + } + + /** + * Report the platform's own calls in the call tree too. + * + *

The inverse of arthas' {@code --skipJDKMethod}, phrased so the default is the quiet one. + * Worth turning on to see what a method spends inside a JDK collection or a JDBC driver; expect + * the tree to grow by an order of magnitude when you do. + */ + public Request withPlatformCalls(boolean value) { + this.skipJdk = !value; + return this; + } + } + + /** The rewritten class, and every method the selector named. */ + public static final class Result { + private final List matched; + private final byte[] bytecode; + + Result(List matched, byte[] bytecode) { + this.matched = Collections.unmodifiableList(matched); + this.bytecode = bytecode; + } + + /** + * Every method the selector named, armed or not, in declaration order. + * + *

The whole SET including the ones that could not be armed, because the resolution is the + * answer worth checking: a pattern that named four and armed three is indistinguishable from + * one that named three if only the successes come back, and that is precisely the "captured + * less than you asked for" reading everything here exists to prevent. + */ + public List getMatched() { + return matched; + } + + /** The armed methods' JVM descriptors, in declaration order. */ + public List getDescriptors() { + List descriptors = new ArrayList(matched.size()); + for (MethodSelector.Match match : matched) { + if (match.isInstrumentable()) { + descriptors.add(match.getMethod().getDescriptor()); + } + } + return descriptors; + } + + /** + * The one armed method's descriptor. + * + * @throws IllegalStateException when a pattern armed several — a caller that can only hold one + * must not silently receive the first of four. + */ + public String getDescriptor() { + List descriptors = getDescriptors(); + if (descriptors.size() != 1) { + throw new IllegalStateException( + "this selector armed " + descriptors.size() + " methods; ask for getDescriptors()"); + } + return descriptors.get(0); + } + + public byte[] getBytecode() { + return bytecode; + } + } + + /** + * Probe keys are embedded in the injected code and name a buffer the client reads back, so they + * are constrained to the shape the client sends: {@code Class.method.suffix}. The client checks + * this too; the client is not the gate. + */ + static void validateProbeKey(String key) { + if (key == null || key.trim().isEmpty()) { + throw new IllegalArgumentException("a probe key is required"); + } + String trimmed = key.trim(); + for (int i = 0; i < trimmed.length(); i++) { + char c = trimmed.charAt(i); + boolean allowed = + c == '.' + || c == '-' + || c == '_' + || c == '$' + || (c >= 'a' && c <= 'z') + || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9'); + if (!allowed) { + throw new IllegalArgumentException( + "probe key \"" + + key + + "\" contains '" + + c + + "'; keys are letters, digits, '.', '-', '_' and '$' (shaped" + + " Class.method.suffix)"); + } + } + } + + /** + * Rewrites every method the selector names, in ONE pass over the class. + * + *

One pass because a class is redefined whole. Arming a pattern's methods one at a time would + * mean re-reading the already-rewritten bytes for the second, and its probe calls would end up + * wrapping the first's — nested frames on one invocation, and durations that contain each other. + */ + public Result instrument(Request request) { + ClassReader reader = new ClassReader(request.bytecode); + final List matched = + request.selector.resolveAll(request.className, reader); + // Keyed on name+descriptor because a name alone is not an identity here: + // a pattern may arm two overloads and the visitor sees them separately. + final Set armed = new HashSet(); + for (MethodSelector.Match match : matched) { + if (match.isInstrumentable()) { + armed.add(match.getMethod().getName() + match.getMethod().getDescriptor()); + } + } + ClassWriter writer = new TargetLoaderClassWriter(reader, request.loader); + + try { + // Frames are computed as the instructions are visited, so the + // resolution this can fail on happens here, not in toByteArray. + reader.accept( + new ClassVisitor(Opcodes.ASM9, writer) { + @Override + public MethodVisitor visitMethod( + int access, String name, String descriptor, String signature, String[] exceptions) { + MethodVisitor delegate = + super.visitMethod(access, name, descriptor, signature, exceptions); + if (!armed.contains(name + descriptor)) { + // Returned unwrapped, so ASM copies this method's bytes + // verbatim rather than re-deriving them. + return delegate; + } + if ((access & (Opcodes.ACC_ABSTRACT | Opcodes.ACC_NATIVE)) != 0) { + throw new IllegalArgumentException( + request.className + + "." + + name + + descriptor + + " is " + + ((access & Opcodes.ACC_NATIVE) != 0 ? "native" : "abstract") + + " and has no body to instrument"); + } + return new ProbeAdapter(delegate, access, name, descriptor, request); + } + }, + ClassReader.SKIP_FRAMES); + return new Result(matched, writer.toByteArray()); + } catch (TypeNotPresentException e) { + // COMPUTE_FRAMES resolves types at merge points through the target's + // loader. One it cannot see is a real answer — "this method's frames + // cannot be computed here" — not something to paper over with + // java/lang/Object, which would produce bytes the verifier rejects. + throw new IllegalStateException( + "computing stack map frames for " + + request.className + + "." + + describe(armed) + + " needs " + + e.typeName() + + ", which its own class loader cannot resolve", + e); + } + } + + private static String describe(Set armed) { + StringBuilder out = new StringBuilder(); + for (String method : armed) { + if (out.length() > 0) { + out.append(", "); + } + out.append(method); + } + return out.toString(); + } + + /** + * Brackets one method's existing instructions with probe calls. + * + *

{@link AdviceAdapter} is what makes the constructor case correct: its {@code onMethodEnter} + * fires after the {@code super()} call, so the entry probe and the catch-all's protected + * range never see an uninitialised {@code this} — which the verifier would reject. + */ + private static final class ProbeAdapter extends AdviceAdapter { + + private final String probeKey; + // Pushed as constants, so the injected code needs no getClass() and no + // `this` — which is why one emission works for instance, static and + // constructor alike. + private final String className; + private final String methodName; + + /** + * The method's own JVM descriptor, pushed alongside its name. + * + *

A constant here rather than a lookup at the far end: a name is not an identity, and once + * one probe key covers a set of methods the events are the only place the overload survives. + * The instrument response carries a descriptor too, but it answers for the one call that armed + * this method and cannot say which of a set a given entry belongs to. + */ + private final String methodDescriptor; + + private final boolean captureArgs; + private final boolean captureReturn; + private final boolean captureException; + private final boolean captureCalls; + private final boolean skipJdk; + private final Type returnType; + private final Label bodyStart = new Label(); + private final Label bodyEnd = new Label(); + + private int frameSlot = -1; + private int returnSlot = -1; + private int throwableSlot = -1; + + /** + * Holds the token of the call currently being made. + * + *

ONE slot is enough for a whole method because call sites cannot overlap. In {@code a(b())} + * the JVM has already invoked and closed {@code b} before it invokes {@code a} — the probe for + * {@code a} is emitted immediately before its own invocation instruction, which is after {@code + * b}'s closing probe. Nesting between them comes from the agent's per-thread stack, not from a + * second local. + */ + private int callSlot = -1; + + /** + * True while this adapter is emitting its own probe calls. + * + *

{@code invokeStatic} routes through {@link #visitMethodInsn}, so without this the wrapper + * would wrap itself: the guard is what stops the emitter recursing into the code it is + * emitting. + */ + private boolean emitting; + + /** + * The source line of the instructions being visited, from the method's line table; 0 until the + * first entry, and throughout a class compiled without one. + * + *

Pushed with each call site, because it is the one thing that lets a trace show the code + * that made a call: the line table is gone once the class is running, and a decompiled class + * has lines of its own. + */ + private int line; + + ProbeAdapter( + MethodVisitor delegate, int access, String name, String descriptor, Request request) { + super(Opcodes.ASM9, delegate, access, name, descriptor); + this.probeKey = request.probeKey; + this.className = request.className; + this.methodName = name; + this.methodDescriptor = descriptor; + this.captureArgs = request.captureArgs; + this.captureReturn = request.captureReturn; + this.captureException = request.captureException; + this.captureCalls = request.captureCalls; + this.skipJdk = request.skipJdk; + this.returnType = Type.getReturnType(descriptor); + } + + /** + * Brackets one invocation the instrumented method makes. + * + *

The stack is why this is safe rather than delicate. At an invocation instruction the + * receiver and arguments are already pushed; pushing six more values and calling a static that + * pops exactly those six leaves everything beneath untouched, because the operand stack is a + * stack. The returned token is stored to a local, so nothing of ours is left on it when the + * real invocation runs. + * + *

The callee is reported by its DECLARED owner at the call site, not its runtime class — an + * interface call reports the interface. That is the same thing arthas trace reports and it is + * what the bytecode knows; the alternative is a receiver type lookup on every call, inside the + * method being measured. + * + *

Nothing is emitted around a throw path: if the callee throws, its closing probe is skipped + * and the agent's own {@code popTo} unwinds the dangling frame when the enclosing method's + * handler reports. Wrapping every call site in its own try/catch would multiply the method's + * handler table by its call count to report what the enclosing frame already makes visible. + */ + @Override + public void visitLineNumber(int number, Label start) { + line = number; + super.visitLineNumber(number, start); + } + + @Override + public void visitMethodInsn( + int opcode, String owner, String name, String descriptor, boolean isInterface) { + if (!captureCalls || emitting || !worthTracing(owner, name)) { + super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); + return; + } + emitting = true; + push(probeKey); + push(owner.replace('/', '.')); + push(name); + push(descriptor); + push(line); + visitInsn(Opcodes.ACONST_NULL); + invokeStatic(AGENT, PROBE_ENTER); + storeLocal(callSlot); + emitting = false; + + super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); + + emitting = true; + push(probeKey); + loadLocal(callSlot); + invokeStatic(AGENT, PROBE_EXIT); + emitting = false; + } + + /** + * Whether a call site is one a reader asked about. + * + *

Ported from arthas, which draws these lines in {@code Enhancer.enhance} and defaults them + * the same way. Three separate reasons to leave a call site alone: + * + *

    + *
  • The platform, when {@code skipJdk} (the default, and arthas' {@code + * --skipJDKMethod true}). {@code "a" + b()} is a StringBuilder construction and three + * appends, and a hot method's platform calls outnumber its own by an order of magnitude — + * reporting them buries the one call that matters and evicts it from the ring to make + * room. Set {@code platform} to keep them. + *
  • Generated proxy and aspect frames, always. A Spring AOP or CGLIB proxy sits + * between the caller and the method that does the work, so tracing it reports the + * interception rather than the call; the same goes for a lambda's synthetic class, whose + * name is a counter nobody wrote and which changes between runs. Unconditional because + * they are noise whether or not the platform is being reported, which is how arthas + * treats the box types. + *
  • The agent, always, because instrumenting our own probes is unbounded recursion + * inside application code. {@link #emitting} already covers the calls this adapter emits; + * this covers a method that happens to call the agent itself. + *
+ */ + private boolean worthTracing(String owner, String name) { + return tracesCallsTo(owner, name, skipJdk); + } + + /** + * Opens the frame. + * + *

The frame token goes into a local rather than staying on the operand stack because + * entering an exception handler wipes the stack down to the throwable — anything parked there + * would be unreachable from the handler that has to close the frame. {@code newLocal} is called + * here and not in the constructor, and the token is only ever touched through {@code + * loadLocal}/{@code storeLocal}, because both are what {@link + * org.objectweb.asm.commons.LocalVariablesSorter} requires to renumber the method's own locals + * correctly. + * + *

The store must land before {@code bodyStart}: the handler's stack map frame has to prove + * the local is definitely assigned, and a range that opens first cannot. + */ + @Override + protected void onMethodEnter() { + frameSlot = newLocal(Type.LONG_TYPE); + if (captureReturn && returnType.getSort() != Type.VOID) { + returnSlot = newLocal(returnType); + } + throwableSlot = newLocal(THROWABLE); + if (captureCalls) { + callSlot = newLocal(Type.LONG_TYPE); + } + + push(probeKey); + push(className); + push(methodName); + push(methodDescriptor); + // A method's own entry has no call site in view: its caller was not + // rewritten, so there is no line to report. + push(0); + if (captureArgs) { + loadArgArray(); + } else { + visitInsn(Opcodes.ACONST_NULL); + } + invokeStatic(AGENT, PROBE_ENTER); + storeLocal(frameSlot); + mark(bodyStart); + } + + /** + * Emits the exit frame at every normal return. + * + *

An {@code ATHROW} is deliberately left alone. When exceptions are being captured the + * catch-all below reports it once, with the elapsed time; when they are not, a method that + * threw produces an entry frame and no exit frame, which reads as what happened — entered, + * never returned — rather than as a normal exit that never occurred. + */ + @Override + protected void onMethodExit(int opcode) { + if (opcode == ATHROW) { + return; + } + if (returnSlot >= 0) { + // Through a local rather than stack acrobatics: dup/swap depends + // on whether the returned value occupies one slot or two, and the + // boxing probeReturn needs would have to differ with it. + storeLocal(returnSlot); + push(probeKey); + loadLocal(frameSlot); + loadLocal(returnSlot); + box(returnType); + invokeStatic(AGENT, PROBE_RETURN); + pop(); + loadLocal(returnSlot); + } + push(probeKey); + loadLocal(frameSlot); + invokeStatic(AGENT, PROBE_EXIT); + } + + /** + * Closes the frame on the way out of a propagating exception, and rethrows it unchanged. + * + *

The handler is emitted ALWAYS, not only when the caller asked for exception detail. With + * depth tracked per thread, an entry with no matching close leaves the depth permanently raised + * on a pooled Tomcat thread, so every later trace on that thread would report nesting that + * never happened — off by however many exceptions it had swallowed. {@code captureException} + * therefore controls how much is said about the throwable, never whether the frame is closed. + * The elapsed time is reported either way, which is the case worth having: a method that took + * eight seconds and then threw. + * + *

{@code catchException} runs here, after the method's own handlers were visited, so ASM + * writes this entry last and every handler the method declared for itself still wins. It must + * stay before {@code super.visitMaxs}. + */ + @Override + public void visitMaxs(int maxStack, int maxLocals) { + mark(bodyEnd); + catchException(bodyStart, bodyEnd, null); + storeLocal(throwableSlot); + push(probeKey); + loadLocal(frameSlot); + loadLocal(throwableSlot); + invokeStatic(AGENT, PROBE_THROW); + loadLocal(throwableSlot); + visitInsn(Opcodes.ATHROW); + // COMPUTE_FRAMES recomputes both, but the visitor contract still + // requires the call that ends the method. + super.visitMaxs(maxStack, maxLocals); + } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/InstrumentationService.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/InstrumentationService.java new file mode 100644 index 0000000..122b47e --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/InstrumentationService.java @@ -0,0 +1,412 @@ +package com.flanksource.telltale.diag.hotpatch; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.lang.instrument.Instrumentation; +import java.lang.management.ManagementFactory; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Obtains {@link Instrumentation} for the JVM this WAR is running inside, by launching a + * short-lived JDK process that attaches back to us and loads {@link + * com.flanksource.telltale.diag.agent.DiagAgent}. + * + *

The attach happens out-of-process because the Attach API has two halves that must come from + * the same JDK: {@code tools.jar} (classes) and {@code libattach.so} (native transport, + * resolved through {@code java.library.path}). application images are frequently JRE-only, which + * supplies neither; loading a side-loaded {@code tools.jar} into an in-process classloader supplies + * only the first half and dies with {@code UnsatisfiedLinkError: no attach in java.library.path}. + * Launching the attach from a complete JDK's own {@code bin/java} makes both halves line up, and as + * a bonus is not self-attach, so Java 9+ needs no {@code jdk.attach.allowAttachSelf}. + * + *

The result is cached. So is a failure: every failure mode here is structural (no usable JDK, + * missing agent jar, a SecurityManager, a JVM that refuses attach), so retrying per request would + * only cost an attach round-trip to produce the same message. A webapp redeploy builds a new + * classloader and therefore retries from scratch. + */ +public final class InstrumentationService { + + /** Holder class parked on the system class path by the attached agent. */ + public static final String AGENT_CLASS = "com.flanksource.telltale.diag.agent.DiagAgent"; + + /** + * The {@code diagnosticAgentJar} build, nested whole into telltale-agent.jar by the {@code + * agentJar} Gradle task (a plain Jar, so it is not unpacked the way the shadow plugin would). + */ + public static final String AGENT_RESOURCE = "/telltale-diag-agent.jar"; + + /** Main class of {@link #AGENT_RESOURCE}, run by the attaching JDK process. */ + private static final String ATTACH_MAIN = "com.flanksource.telltale.diag.agent.AttachMain"; + + /** Operator override naming the JDK to attach with; probed before everything else. */ + private static final String JDK_HOME_PROPERTY = "telltale.diag.jdk-home"; + + /** How long the attaching process gets before we give up on it. */ + private static final long ATTACH_TIMEOUT_SECONDS = 60; + + /** Grace period for the output drain once the attacher has exited or been killed. */ + private static final long DRAIN_TIMEOUT_MILLIS = 2000; + + private static Instrumentation cached; + private static String failure; + + private InstrumentationService() {} + + public static boolean installed() { + return fromHolder() != null; + } + + /** + * @return the live {@link Instrumentation} for this JVM. + * @throws IllegalStateException with a mode-specific message when it cannot be obtained. + */ + public static synchronized Instrumentation instrumentation() { + if (cached != null) { + return cached; + } + if (failure != null) { + throw new IllegalStateException(failure); + } + try { + cached = attach(); + return cached; + } catch (IllegalStateException e) { + failure = e.getMessage(); + throw e; + } + } + + /** Contract §4 {@code instrumentation} block for the selftest response. */ + public static Map status() { + Map out = new LinkedHashMap<>(); + try { + Instrumentation inst = instrumentation(); + out.put("available", true); + out.put("retransformSupported", inst.isRetransformClassesSupported()); + out.put("redefineSupported", inst.isRedefineClassesSupported()); + out.put("error", ""); + } catch (RuntimeException e) { + out.put("available", false); + out.put("retransformSupported", false); + out.put("redefineSupported", false); + out.put("error", e.getMessage() == null ? String.valueOf(e) : e.getMessage()); + } + return out; + } + + private static Instrumentation attach() { + Instrumentation existing = fromHolder(); + if (existing != null) { + return existing; + } + String pid = pid(); + Path agentJar = stageAgentJar(); + Jdk jdk = findJdk(); + + List command = jdk.attachCommand(pid, agentJar); + String output = run(command); + + Instrumentation loaded = fromHolder(); + if (loaded == null) { + throw new IllegalStateException( + "agent " + + agentJar.toAbsolutePath() + + " was loaded into pid " + + pid + + " by " + + jdk.javaBin + + " but " + + AGENT_CLASS + + ".INSTRUMENTATION is still null" + + " (a stale agent jar without an agentmain hook is on the system class path)." + + " Attacher said: " + + output); + } + return loaded; + } + + /** + * Environment variables through which the container injects JVM flags. The application image sets + * JAVA_TOOL_OPTIONS with its own {@code -javaagent} pair (Prometheus JMX exporter, + * OpenTelemetry). A child JVM inherits them, tries to bind the exporter port this JVM already + * owns, and aborts with "processing of -javaagent failed" before it ever attaches — so the + * attacher must run as a bare JVM. + */ + private static final List INHERITED_JVM_OPTION_VARS = + Arrays.asList("JAVA_TOOL_OPTIONS", "_JAVA_OPTIONS", "JDK_JAVA_OPTIONS"); + + /** Package-visible for tests; see {@link #INHERITED_JVM_OPTION_VARS}. */ + static void stripInheritedJvmOptions(Map environment) { + environment.keySet().removeAll(INHERITED_JVM_OPTION_VARS); + } + + /** Runs the attaching JDK process, failing loud with its own output. */ + private static String run(List command) { + Process process; + try { + ProcessBuilder builder = new ProcessBuilder(command).redirectErrorStream(true); + stripInheritedJvmOptions(builder.environment()); + process = builder.start(); + } catch (IOException e) { + throw new IllegalStateException( + "attach refused: could not run " + command + ": " + describe(e), e); + } catch (SecurityException e) { + throw securityManagerFailure("launch the attaching JDK process", e); + } + // Drain on another thread: reading to EOF on this one would block past + // ATTACH_TIMEOUT_SECONDS if the child hangs without closing stdout, + // which is exactly the case the timeout exists for. + StringBuilder sink = new StringBuilder(); + Thread drain = + new Thread( + () -> sink.append(readFully(process.getInputStream())), "telltale-diag-attach-output"); + drain.setDaemon(true); + drain.start(); + + boolean exited; + try { + exited = process.waitFor(ATTACH_TIMEOUT_SECONDS, TimeUnit.SECONDS); + if (!exited) { + process.destroyForcibly(); + } + drain.join(DRAIN_TIMEOUT_MILLIS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + process.destroyForcibly(); + throw new IllegalStateException("attach interrupted while waiting for " + command, e); + } + String output = sink.toString(); + if (!exited) { + throw new IllegalStateException( + "attach timed out: " + + command + + " did not finish within " + + ATTACH_TIMEOUT_SECONDS + + "s. Output so far: " + + output); + } + if (process.exitValue() != 0) { + throw new IllegalStateException( + "attach refused: " + + command + + " exited " + + process.exitValue() + + ". The JVM may have been started with -XX:+DisableAttachMechanism," + + " or the attaching process may not share this process' user. Output: " + + output); + } + return output; + } + + private static String readFully(InputStream in) { + StringBuilder out = new StringBuilder(); + byte[] buffer = new byte[4096]; + try { + for (int read; (read = in.read(buffer)) != -1; ) { + out.append(new String(buffer, 0, read, StandardCharsets.UTF_8)); + } + } catch (IOException e) { + out.append("'); + } + return out.toString().trim(); + } + + /** + * A JDK complete enough to attach with: a {@code bin/java} launcher, plus that same JDK's {@code + * tools.jar} on Java 8. Pairing the two is the whole point — a JRE launcher cannot find {@code + * libattach.so} for someone else's tools.jar. + */ + static final class Jdk { + final Path home; + final Path javaBin; + + /** null on Java 9+, where the attach API is in the base modules. */ + final Path toolsJar; + + Jdk(Path home, Path javaBin, Path toolsJar) { + this.home = home; + this.javaBin = javaBin; + this.toolsJar = toolsJar; + } + + List attachCommand(String pid, Path agentJar) { + String classpath = agentJar.toAbsolutePath().toString(); + if (toolsJar != null) { + classpath = toolsJar + File.pathSeparator + classpath; + } + return Arrays.asList( + javaBin.toString(), + "-cp", + classpath, + ATTACH_MAIN, + pid, + agentJar.toAbsolutePath().toString()); + } + } + + /** + * @return a JDK complete enough to attach with — and, for {@link CompilerService}, to compile + * with: the same tools.jar carries javac on Java 8. + */ + static Jdk findJdk() { + boolean needsToolsJar = needsToolsJar(); + for (Path home : jdkHomeCandidates()) { + Path javaBin = home.resolve("bin").resolve("java"); + if (!Files.isExecutable(javaBin)) { + continue; + } + if (!needsToolsJar) { + return new Jdk(home, javaBin, null); + } + Path toolsJar = home.resolve("lib").resolve("tools.jar"); + if (Files.isReadable(toolsJar)) { + return new Jdk(home, javaBin, toolsJar); + } + } + throw new IllegalStateException( + "no JDK available to attach with" + + " (java.version=" + + System.getProperty("java.version") + + ", java.home=" + + System.getProperty("java.home") + + ")." + + " Needed a directory with bin/java" + + (needsToolsJar ? " and lib/tools.jar" : "") + + "; probed " + + jdkHomeCandidates() + + ". Run the application on a full JDK image or point -D" + + JDK_HOME_PROPERTY + + " at a matching JDK."); + } + + /** Java 8 keeps the attach API in tools.jar; 9+ ships it in the base modules. */ + private static boolean needsToolsJar() { + String spec = System.getProperty("java.specification.version", ""); + return spec.startsWith("1."); + } + + /** + * @return every JDK home probed, in priority order. Package-visible so tests can assert the + * ordering, which is load-bearing: a JDK in the image must win over the side-loaded one. + */ + static List jdkHomeCandidates() { + List candidates = new ArrayList<>(); + String override = System.getProperty(JDK_HOME_PROPERTY); + if (override != null && !override.trim().isEmpty()) { + candidates.add(Paths.get(override.trim())); + } + // A JDK 8 process reports java.home as /jre, so the JDK sits one level up. + String javaHome = System.getProperty("java.home"); + if (javaHome != null && !javaHome.isEmpty()) { + candidates.add(Paths.get(javaHome)); + candidates.add(Paths.get(javaHome, "..").normalize()); + } + String env = System.getenv("JAVA_HOME"); + if (env != null && !env.isEmpty()) { + candidates.add(Paths.get(env)); + } + return candidates; + } + + private static Path stageAgentJar() { + Path target = + Paths.get(System.getProperty("java.io.tmpdir"), "telltale-diag", "telltale-diag-agent.jar"); + try (InputStream in = InstrumentationService.class.getResourceAsStream(AGENT_RESOURCE)) { + if (in == null) { + throw new IllegalStateException( + "agent jar missing: " + + AGENT_RESOURCE + + " is not bundled in this WAR — rebuild the WAR so the Gradle agentJar task runs"); + } + Files.createDirectories(target.getParent()); + Files.copy(in, target, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + throw new IllegalStateException( + "agent jar missing: could not stage " + + AGENT_RESOURCE + + " to " + + target + + ": " + + describe(e), + e); + } catch (SecurityException e) { + throw securityManagerFailure("stage the agent jar to " + target, e); + } + return target; + } + + private static Instrumentation fromHolder() { + try { + Class holder = ClassLoader.getSystemClassLoader().loadClass(AGENT_CLASS); + Field field = holder.getField("INSTRUMENTATION"); + return (Instrumentation) field.get(null); + } catch (ClassNotFoundException e) { + return null; + } catch (SecurityException e) { + throw securityManagerFailure("read " + AGENT_CLASS + ".INSTRUMENTATION", e); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException( + "agent holder " + + AGENT_CLASS + + " is on the system class path but unreadable: " + + describe(e) + + " (a different telltale-diag agent jar was attached earlier)", + e); + } + } + + private static String pid() { + String name = ManagementFactory.getRuntimeMXBean().getName(); + String pid = name.split("@")[0]; + if (pid.isEmpty() || !pid.matches("\\d+")) { + throw new IllegalStateException( + "attach refused: RuntimeMXBean name \"" + name + "\" does not start with a numeric pid"); + } + return pid; + } + + private static IllegalStateException securityManagerFailure( + String what, SecurityException cause) { + SecurityManager securityManager = System.getSecurityManager(); + if (securityManager == null) { + return new IllegalStateException( + "access denied trying to " + + what + + ": " + + describe(cause) + + " (no SecurityManager is installed, so this is a platform-level restriction)", + cause); + } + return new IllegalStateException( + "SecurityManager present: " + + securityManager.getClass().getName() + + " denied " + + what + + " — " + + describe(cause) + + ". Grant the telltale-diag webapp AllPermission or start the JVM without" + + " -Djava.security.manager", + cause); + } + + private static String describe(Throwable t) { + if (t == null) { + return "unknown cause"; + } + return t.getMessage() == null + ? t.getClass().getName() + : t.getClass().getName() + ": " + t.getMessage(); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/JsonBody.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/JsonBody.java new file mode 100644 index 0000000..c38afde --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/JsonBody.java @@ -0,0 +1,142 @@ +package com.flanksource.telltale.diag.hotpatch; + +import com.flanksource.telltale.diag.Json; +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.Map; + +/** + * One JSON request body, read strictly. + * + *

Every accessor here refuses rather than guesses. These bodies name a class to rewrite in a + * running JVM, so a field of the wrong type is a caller bug, and coercing it — a missing boolean + * read as false, a string "30" read as thirty seconds — would turn that bug into a patch nobody + * asked for. + */ +public final class JsonBody { + + private final Map fields; + + public JsonBody(Map fields) { + this.fields = fields; + } + + @SuppressWarnings("unchecked") + static JsonBody read(InputStream in) throws IOException { + Map parsed = Json.read(in, Map.class); + if (parsed == null) { + throw new IllegalArgumentException("a JSON object body is required"); + } + return new JsonBody(parsed); + } + + /** + * The raw fields, for a callee that takes an open-ended option bag rather than a fixed set — the + * probe object's per-probe tunables are the case. The agent ignores entries it does not + * recognise, so an older CLI talking to a newer JVM still works. + */ + Map fields() { + return fields; + } + + String required(String field) { + String value = string(field); + if (value == null) { + throw new IllegalArgumentException(field + " is required"); + } + return value; + } + + /** A present, non-blank string, or null. Absent and blank mean the same thing. */ + String string(String field) { + Object value = fields.get(field); + if (value == null) return null; + if (!(value instanceof String)) throw new IllegalArgumentException(field + " must be a string"); + if (((String) value).trim().isEmpty()) { + return null; + } + // Source and a classpath are content, not identifiers: trimming them + // would silently move the line numbers javac reports against. + return "source".equals(field) || "classpath".equals(field) + ? (String) value + : ((String) value).trim(); + } + + boolean flag(String field) { + Object value = fields.get(field); + if (value == null) { + return false; + } + if (!(value instanceof Boolean)) { + throw new IllegalArgumentException(field + " must be true or false, got " + value); + } + return (Boolean) value; + } + + /** Zero means "leave the default in place"; a negative is a caller bug. */ + int integer(String field) { + Object value = fields.get(field); + if (value == null) { + return 0; + } + if (!(value instanceof Number)) { + throw new IllegalArgumentException(field + " must be a number, got " + value); + } + int parsed = signedInteger(field); + if (parsed < 0) { + throw new IllegalArgumentException(field + " must not be negative, got " + parsed); + } + return parsed; + } + + /** + * Like {@link #integer}, but a negative is meaningful rather than a caller bug — {@code + * leaseSeconds: -1} is how a caller says "leave this in place until I restore it", and refusing + * it would make that unsayable. + */ + int signedInteger(String field) { + Object value = fields.get(field); + if (value == null) { + return 0; + } + if (!(value instanceof Number)) { + throw new IllegalArgumentException(field + " must be a number, got " + value); + } + double parsed = ((Number) value).doubleValue(); + if (!Double.isFinite(parsed) + || parsed != Math.rint(parsed) + || parsed < Integer.MIN_VALUE + || parsed > Integer.MAX_VALUE) { + throw new IllegalArgumentException(field + " must be a 32-bit integer, got " + value); + } + return (int) parsed; + } + + /** A required nested object, e.g. {@code probe}. */ + @SuppressWarnings("unchecked") + JsonBody object(String field) { + Object value = fields.get(field); + if (!(value instanceof Map)) { + throw new IllegalArgumentException(field + " is required, as an object"); + } + return new JsonBody((Map) value); + } + + /** + * An optional nested object, absent reading as an empty one so a caller can ask through it + * without branching first. + * + *

Still strict where it counts: a field of the wrong type is absent here rather than coerced, + * and every accessor reached through the result refuses on its own terms. + */ + @SuppressWarnings("unchecked") + JsonBody optionalObject(String field) { + Object value = fields.get(field); + if (value != null && !(value instanceof Map)) + throw new IllegalArgumentException(field + " must be an object"); + return value instanceof Map + ? new JsonBody((Map) value) + : new JsonBody(Collections.emptyMap()); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/LoaderProfile.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/LoaderProfile.java new file mode 100644 index 0000000..e87a390 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/LoaderProfile.java @@ -0,0 +1,387 @@ +package com.flanksource.telltale.diag.hotpatch; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * What a classloader is, as opposed to what class implements it. + * + *

A TomEE JVM running application reports thousands of loaders, and the raw class name tells an + * operator nothing: {@code URLClassLoader} and {@code AgentClassLoader} are not answers to "which + * one holds the code I want to decompile". Worse, two families are pure noise — one loader per + * JIT-generated reflection accessor, and one per business rule application compiles at runtime — + * and between them they account for over 2,000 of the ~2,170 rows. + * + *

So each loader gets a {@link #getLabel() label} (what it is), a {@link #getKind() kind} (which + * drives ordering, so application code sorts above platform noise), and optionally a {@link + * #getGroup() group} — a virtual loader id that collapses a whole family into one row. + * + *

The rules are matched on the loader's class name first and its webapp context second. Class + * name first is also what keeps this cheap: the two noisy families are recognised without the + * reflective context probe that {@link ClassIndex#contextOf} performs. + */ +public final class LoaderProfile { + + /** Application code — what someone opening the class browser came for. */ + public static final String KIND_APP = "app"; + + /** This diagnostic tool's own WAR. */ + public static final String KIND_TOOL = "tool"; + + /** The servlet container and the shared libraries it publishes. */ + public static final String KIND_CONTAINER = "container"; + + /** Some other deployed web application. */ + public static final String KIND_WEBAPP = "webapp"; + + /** Bytecode generated at runtime rather than shipped in a JAR. */ + public static final String KIND_GENERATED = "generated"; + + /** A java agent, loaded in isolation from the application. */ + public static final String KIND_AGENT = "agent"; + + /** The JDK itself. */ + public static final String KIND_JDK = "jdk"; + + /** No rule matched. */ + public static final String KIND_UNKNOWN = "unknown"; + + /** + * Virtual loader ids are {@code ~}-prefixed. {@code ~} is unreserved in RFC 3986 so it survives a + * query string untouched, it cannot collide with an identity hash ({@code [0-9a-f]+}, bootstrap + * {@code "0"}), and unlike {@code :} it cannot collide with the {@code :} keys the + * web UI builds. {@code @} is unusable here: the CLI layer in front of this WAR reads a leading + * {@code @} as "load the value from this file". + */ + static final String GROUP_PREFIX = "~"; + + public static final String GROUP_COMPILED = GROUP_PREFIX + "compiled"; + public static final String GROUP_JDK_REFLECTION = GROUP_PREFIX + "jdk-reflection"; + public static final String GROUP_JMX_RMI = GROUP_PREFIX + "jmx-rmi"; + + /** How many library names a description lists before it stops. */ + private static final int DESCRIPTION_FACETS = 4; + + private static final LoaderProfile BOOTSTRAP = new LoaderProfile("JDK bootstrap", KIND_JDK, null); + + /** + * Loader class name to profile. Matched before the webapp context, because these are the loaders + * that have no context and probing for one is the expensive part. + */ + private static final Map BY_CLASS_NAME = byClassName(); + + private static Map byClassName() { + Map rules = new LinkedHashMap<>(); + rules.put( + "org.codehaus.janino.ByteArrayClassLoader", + new LoaderProfile("Runtime compiled classes", KIND_GENERATED, GROUP_COMPILED)); + rules.put( + "sun.reflect.DelegatingClassLoader", + new LoaderProfile("JDK reflection accessors", KIND_JDK, GROUP_JDK_REFLECTION)); + rules.put( + "javax.management.remote.rmi.NoCallStackClassLoader", + new LoaderProfile("JMX RMI connector", KIND_JDK, GROUP_JMX_RMI)); + rules.put( + "io.opentelemetry.javaagent.bootstrap.AgentClassLoader", + new LoaderProfile("OpenTelemetry agent", KIND_AGENT, null)); + rules.put( + "net.bytebuddy.utility.dispatcher.JavaDispatcher$DynamicClassLoader", + new LoaderProfile("ByteBuddy dispatcher", KIND_AGENT, null)); + rules.put( + "sun.misc.Launcher$AppClassLoader", + new LoaderProfile("JVM classpath + java agents", KIND_CONTAINER, null)); + rules.put( + "jdk.internal.loader.ClassLoaders$AppClassLoader", + new LoaderProfile("JVM classpath + java agents", KIND_CONTAINER, null)); + rules.put( + "sun.misc.Launcher$ExtClassLoader", + new LoaderProfile("JDK extensions (lib/ext)", KIND_JDK, null)); + rules.put( + "jdk.internal.loader.ClassLoaders$PlatformClassLoader", + new LoaderProfile("JDK platform classes", KIND_JDK, null)); + rules.put( + "sun.reflect.misc.MethodUtil", + new LoaderProfile("JDK introspection trampoline", KIND_JDK, null)); + return Collections.unmodifiableMap(rules); + } + + /** + * Deliberately absent from {@link #BY_CLASS_NAME}: {@code java.net.URLClassLoader}. On the lab + * application JVM it happens to be TomEE's shared {@code lib/} loader, but that is a fact about + * one deployment rather than about the class, and naming it "TomEE common" would be a guess + * presented as a label. It takes the fallback and is described by its contents instead, which + * says EclipseLink/Coherence/OpenEJB/ActiveMQ/CXF — the part that is actually true and actually + * useful. + */ + private static final String CONTEXT_TOOL = "/telltale-diag"; + + /** + * Ordering: application code first, platform noise last. + * + *

Unknown outranks the JDK deliberately. Every JDK loader worth naming is classified + * explicitly, so a loader that matched nothing is by construction not one of them — it + * is far more likely to be application or container infrastructure. Sorting it below the + * bootstrap loader would bury the largest unclassified loader in a TomEE JVM (the shared {@code + * lib/} one, 8,000+ classes) at the bottom of the list. + */ + private static final List KIND_ORDER = + Collections.unmodifiableList( + Arrays.asList( + KIND_APP, + KIND_TOOL, + KIND_CONTAINER, + KIND_WEBAPP, + KIND_GENERATED, + KIND_AGENT, + KIND_UNKNOWN, + KIND_JDK)); + + /** + * Package prefix to the name a human would use for it. Most specific first — lookup takes the + * first match, so {@code io.prometheus.jmx} must precede a hypothetical {@code io.prometheus}. + * + *

This is a package table rather than a loader table on purpose: it is the reusable + * half. An unmapped package degrades to its first two segments rather than to nothing. + */ + private static final Map LIBRARY_NAMES = libraryNames(); + + private static Map libraryNames() { + Map names = new LinkedHashMap<>(); + names.put("com.flanksource.telltale.diag", "telltale-diag"); + names.put("org.benf.cfr", "CFR decompiler"); + names.put("org.eclipse.persistence", "EclipseLink"); + names.put("com.tangosol", "Coherence"); + names.put("org.apache.openejb", "OpenEJB"); + names.put("org.apache.tomee", "TomEE"); + names.put("org.apache.catalina", "Tomcat"); + names.put("org.apache.tomcat", "Tomcat"); + names.put("org.apache.juli", "Tomcat logging"); + names.put("org.apache.activemq", "ActiveMQ"); + names.put("org.apache.cxf", "CXF"); + names.put("org.apache.bval", "BVal"); + names.put("org.apache.xbean", "XBean"); + names.put("org.apache.commons", "Commons"); + names.put("org.glassfish.jersey", "Jersey"); + names.put("org.glassfish.hk2", "HK2"); + names.put("com.fasterxml.jackson", "Jackson"); + names.put("net.sf.saxon", "Saxon"); + names.put("org.codehaus.janino", "Janino"); + names.put("org.springframework", "Spring"); + names.put("reactor.core", "Reactor"); + names.put("org.aspectj", "AspectJ"); + names.put("com.microsoft.sqlserver", "MSSQL JDBC"); + names.put("io.opentelemetry", "OpenTelemetry"); + names.put("net.bytebuddy", "ByteBuddy"); + names.put("org.objectweb.asm", "ASM"); + names.put("okhttp3", "OkHttp"); + names.put("io.prometheus.jmx", "Prometheus JMX exporter"); + names.put("co.elastic.logging", "Elastic logging"); + names.put("org.slf4j", "SLF4J"); + names.put("org.apache.logging", "Log4j"); + return Collections.unmodifiableMap(names); + } + + private final String label; + private final String kind; + private final String group; + private final String context; + + LoaderProfile(String label, String kind, String group) { + this(label, kind, group, null); + } + + private LoaderProfile(String label, String kind, String group, String context) { + this.label = label; + this.kind = kind; + this.group = group; + this.context = context; + } + + public String getLabel() { + return label; + } + + public String getKind() { + return kind; + } + + /** The virtual loader id this loader merges into, or {@code null} to stand alone. */ + public String getGroup() { + return group; + } + + /** + * @return true when {@code id} is a virtual loader id rather than an identity hash. + */ + public static boolean isGroupId(String id) { + return id != null && id.startsWith(GROUP_PREFIX); + } + + /** + * Classifies one loader. The webapp context is probed only when no class-name rule matched, which + * keeps the reflective {@link ClassIndex#contextOf} probe off the thousands of generated and + * reflection loaders that will never have one. + */ + public static LoaderProfile of(ClassLoader loader) { + if (loader == null) { + return BOOTSTRAP; + } + Class type = loader.getClass(); + LoaderProfile byName = BY_CLASS_NAME.get(type.getName()); + if (byName != null) { + return byName; + } + return of(type.getName(), type.getSimpleName(), ClassIndex.contextOf(loader)); + } + + /** + * The rule table itself, addressable by name — so it can be exercised without conjuring a loader + * instance per rule, several of which live in packages a test has no business defining classes + * in. + */ + static LoaderProfile of(String className, String simpleName, String context) { + LoaderProfile byName = BY_CLASS_NAME.get(className); + if (byName != null) { + return byName; + } + if (context != null) { + return forContext(context); + } + return new LoaderProfile( + simpleName == null || simpleName.isEmpty() ? className : simpleName, KIND_UNKNOWN, null); + } + + private static LoaderProfile forContext(String context) { + if (CONTEXT_TOOL.equals(context)) { + return new LoaderProfile("telltale-diag WAR (this tool)", KIND_TOOL, null); + } + return new LoaderProfile("Application (" + context + ")", KIND_APP, null, context); + } + + private static LoaderProfile webapp(String context) { + return new LoaderProfile("Web application (" + context + ")", KIND_WEBAPP, null); + } + + /** + * This profile once the loader's classes are known. The context alone does not make a loader the + * application: the lab cycle JVM carries a second {@code /worker} loader beside the real one + * holding a single {@code com.sun} class. A loader in an application context with nothing but + * platform classes is listed as a plain web application loader, so the application stays the one + * loader a picker can default to. + */ + LoaderProfile holding(Collection classNames) { + if (!KIND_APP.equals(kind)) { + return this; + } + for (String className : classNames) { + if (!ClassIndex.isPlatform(className)) { + return this; + } + } + return webapp(context); + } + + /** Sort position for {@link #getKind()}; an unrecognised kind sorts as unknown. */ + public static int rank(String kind) { + int index = KIND_ORDER.indexOf(kind); + return index < 0 ? KIND_ORDER.indexOf(KIND_UNKNOWN) : index; + } + + /** + * A one-line summary of what a loader holds, derived from the class names it actually loaded + * rather than hardcoded per loader — so it stays true as the deployment changes, and works on a + * tier this table has never seen. + * + *

{@link #KIND_GENERATED} loaders are summarised by the family of the generated class ({@code + * OnChangeProcessor}, {@code MathEngine}) because their package is a single constant and would + * say nothing. Everything else is summarised by the libraries it holds. + */ + public static String describe(String kind, Collection classNames) { + return join(KIND_GENERATED.equals(kind) ? topFamilies(classNames) : topLibraries(classNames)); + } + + private static List topLibraries(Collection classNames) { + Map counts = new LinkedHashMap<>(); + for (String name : classNames) { + tally(counts, libraryOf(name)); + } + return top(counts); + } + + /** Known library name, else the first two package segments, else the name itself. */ + private static String libraryOf(String className) { + for (Map.Entry library : LIBRARY_NAMES.entrySet()) { + String prefix = library.getKey(); + if (className.startsWith(prefix + ".")) { + return library.getValue(); + } + } + int first = className.indexOf('.'); + if (first < 0) { + return className; + } + int second = className.indexOf('.', first + 1); + return second < 0 ? className.substring(0, first) : className.substring(0, second); + } + + private static List topFamilies(Collection classNames) { + Map counts = new LinkedHashMap<>(); + for (String name : classNames) { + tally(counts, familyOf(name)); + } + return top(counts); + } + + /** + * The generated-class family: application names a compiled rule {@code + * OnChangeProcessor_PolicyScreen_MandateStartDate_}, so the part before the first underscore + * is the kind of rule it is. + */ + private static String familyOf(String className) { + int lastDot = className.lastIndexOf('.'); + String simple = lastDot < 0 ? className : className.substring(lastDot + 1); + int underscore = simple.indexOf('_'); + return underscore < 0 ? simple : simple.substring(0, underscore); + } + + private static void tally(Map counts, String key) { + Integer previous = counts.get(key); + counts.put(key, previous == null ? 1 : previous + 1); + } + + /** The most common keys, ties broken alphabetically so the output is stable. */ + private static List top(Map counts) { + List> ordered = new ArrayList<>(counts.entrySet()); + ordered.sort( + Comparator.comparingInt((Map.Entry entry) -> entry.getValue()) + .reversed() + .thenComparing(Map.Entry::getKey)); + Set top = new LinkedHashSet<>(); + for (Map.Entry entry : ordered) { + if (top.size() == DESCRIPTION_FACETS) { + break; + } + top.add(entry.getKey()); + } + return new ArrayList<>(top); + } + + private static String join(List parts) { + StringBuilder out = new StringBuilder(); + for (String part : parts) { + if (out.length() > 0) { + out.append(", "); + } + out.append(part); + } + return out.toString(); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/LoaderRows.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/LoaderRows.java new file mode 100644 index 0000000..aaf4888 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/LoaderRows.java @@ -0,0 +1,104 @@ +package com.flanksource.telltale.diag.hotpatch; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Builds the {@code /classes/loaders} payload from the classes each loader holds. + * + *

Separate from {@link ClassIndex} because it answers a different question: the index knows + * which loader defined what, this knows how to present that as a list somebody has to choose from. + * The presentation is where the merging happens — families that exist in the thousands collapse to + * one virtual row each (see {@link LoaderProfile#getGroup()}), and the rows are ordered so + * application code comes before platform noise. + */ +final class LoaderRows { + + private LoaderRows() {} + + /** + * @param byLoader class names held by each non-bootstrap loader + * @param bootstrapNames class names held by the bootstrap loader + */ + static List> build( + Map> byLoader, List bootstrapNames) { + List> rows = new ArrayList<>(); + if (!bootstrapNames.isEmpty()) { + rows.add(row(ClassIndex.BOOTSTRAP_HASH, "", LoaderProfile.of(null), bootstrapNames)); + } + Map groups = new LinkedHashMap<>(); + for (Map.Entry> loader : byLoader.entrySet()) { + LoaderProfile profile = LoaderProfile.of(loader.getKey()).holding(loader.getValue()); + String className = loader.getKey().getClass().getName(); + if (profile.getGroup() == null) { + rows.add(row(ClassIndex.hash(loader.getKey()), className, profile, loader.getValue())); + continue; + } + Group group = groups.get(profile.getGroup()); + if (group == null) { + group = new Group(profile, className); + groups.put(profile.getGroup(), group); + } + group.add(loader.getValue()); + } + for (Map.Entry group : groups.entrySet()) { + rows.add(group.getValue().row(group.getKey())); + } + sort(rows); + return rows; + } + + private static void sort(List> rows) { + Comparator> byKind = + Comparator.comparingInt(row -> LoaderProfile.rank(String.valueOf(row.get("kind")))); + Comparator> byCount = + Comparator.comparingInt(row -> (Integer) row.get("classCount")); + Comparator> byHash = + Comparator.comparing(row -> String.valueOf(row.get("hash"))); + rows.sort(byKind.thenComparing(byCount.reversed()).thenComparing(byHash)); + } + + private static Map row( + String hash, String className, LoaderProfile profile, List classNames) { + Map row = new LinkedHashMap<>(); + row.put("hash", hash); + row.put("className", className); + row.put("label", profile.getLabel()); + row.put("description", LoaderProfile.describe(profile.getKind(), classNames)); + row.put("kind", profile.getKind()); + row.put("classCount", classNames.size()); + return row; + } + + /** One virtual loader under construction: the members of a merged family. */ + private static final class Group { + private final LoaderProfile profile; + private final String className; + private final List classNames = new ArrayList<>(); + private int loaderCount; + + private Group(LoaderProfile profile, String className) { + this.profile = profile; + this.className = className; + } + + private void add(List names) { + classNames.addAll(names); + loaderCount++; + } + + private Map row(String id) { + Map row = LoaderRows.row(id, className, profile, classNames); + // How many loaders were folded together is the whole point of the + // row, so it belongs in the label rather than only in a field a + // picker may not render. + row.put("label", profile.getLabel() + " (" + loaderCount + " loaders)"); + row.put("virtual", Boolean.TRUE); + row.put("loaderCount", loaderCount); + return row; + } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/MethodPatchService.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/MethodPatchService.java new file mode 100644 index 0000000..022214d --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/MethodPatchService.java @@ -0,0 +1,132 @@ +package com.flanksource.telltale.diag.hotpatch; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.commons.Method; + +/** + * Replaces one method of a live class with a freshly compiled body: generate a stand-in for the + * class from its own shape, compile the author's method against it, and splice only that method's + * bytecode into the bytes the class is running. + * + *

This exists so that replacing a method needs no decompiler. Decompiling the whole class and + * recompiling it is unusable in the environment this ships into for two independent reasons: a + * reconstruction of a class an agent has already woven frequently does not compile at all, and even + * when it does, recompiling it discards that agent's instrumentation from every other method — on + * the one class someone is actively debugging. + */ +public final class MethodPatchService { + + private final SkeletonService skeletons = new SkeletonService(); + private final CompilerService compiler = new CompilerService(); + private final MethodSplicer splicer = new MethodSplicer(); + + /** A compile of one replacement method, positioned in the source the author sent. */ + public static final class Result { + private final CompilerService.Result compiled; + private final List> diagnostics; + private final Method target; + private final byte[] liveBytecode; + + Result( + CompilerService.Result compiled, + List> diagnostics, + Method target, + byte[] liveBytecode) { + this.compiled = compiled; + this.diagnostics = diagnostics; + this.target = target; + this.liveBytecode = liveBytecode; + } + + public boolean isSuccess() { + return compiled.isSuccess(); + } + + /** The resolved JVM descriptor of the method being replaced. */ + public String getDescriptor() { + return target.getDescriptor(); + } + + /** + * The compile response, with every diagnostic repositioned onto the method the caller actually + * submitted. + */ + public Map toMap() { + Map out = new LinkedHashMap(compiled.toMap()); + out.put("diagnostics", diagnostics); + return out; + } + } + + /** + * Compiles {@code methodSource} against a stand-in for {@code target}'s live shape, without + * touching the running class. + * + * @param liveBytecode what the class is currently running (see {@link BytecodeSource}). + */ + public Result compile( + String className, + Class target, + byte[] liveBytecode, + MethodSelector selector, + String methodSource, + String classpath) { + Method resolved = selector.resolve(className, new ClassReader(liveBytecode)); + SkeletonService.Skeleton skeleton = skeletons.skeleton(target, methodSource, resolved); + CompilerService.Result compiled = + compiler.compile( + new CompilerService.Request(className, skeleton.getSource(), target) + .withClasspath(classpath)); + return new Result( + compiled, + rebase(compiled.getMessages(), skeleton.getMethodLine(), className), + resolved, + liveBytecode); + } + + /** + * Splices the compiled method into the live bytes. + * + * @return the class bytes to redefine with. + */ + public byte[] splice(String className, Result result, ClassLoader loader) { + if (!result.isSuccess()) { + throw new IllegalStateException( + className + " did not compile, so there is nothing to splice"); + } + return splicer.splice( + className, result.liveBytecode, result.compiled.getClasses(), result.target, loader); + } + + /** + * Moves javac's line numbers from the generated stand-in onto the method the caller sent, since + * that is the only text the caller has to look at. + * + *

A diagnostic that lands in the generated part is not renumbered into something that looks + * like the author's code — it is stripped of its position and labelled, because a wrong line + * number is read as a fact. + */ + private static List> rebase( + List messages, int methodLine, String className) { + List> out = new ArrayList>(); + for (CompilerService.Message message : messages) { + Map row = message.toMap(); + Object reported = row.get("line"); + long line = reported instanceof Number ? ((Number) reported).longValue() : -1L; + if (line >= methodLine) { + row.put("line", line - methodLine + 1); + } else { + row.put("line", -1L); + row.put("column", -1L); + row.put( + "message", "in the generated stand-in for " + className + ": " + row.get("message")); + } + out.add(row); + } + return out; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/MethodSelector.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/MethodSelector.java new file mode 100644 index 0000000..91f13a8 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/MethodSelector.java @@ -0,0 +1,631 @@ +package com.flanksource.telltale.diag.hotpatch; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.NoSuchElementException; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; +import org.objectweb.asm.commons.Method; + +/** + * A reference to one method of a class, in whichever form the author wrote it. + * + *

A JVM addresses a method by an exact descriptor — {@code + * (Lexample/application/pas/dcl/ActivityFilterDcl;Ljava/util/List;Ljava/lang/String;)Ljava/lang/String;} + * — while a person writes a name, a name with parameter types, or a signature pasted out of a stack + * trace. Four forms are accepted: + * + *

+ *   buildQuery                                        bare name, any arity
+ *   buildQuery(ActivityFilterDcl, List, String)       simple parameter types
+ *   private static String buildQuery(Foo f, Bar b)    a pasted Java signature
+ *   buildQuery(Ljava/util/List;)Ljava/lang/String;    a JVM descriptor
+ * 
+ * + *

{@link #resolve} is what makes this the gate rather than a convenience. The client parses the + * same forms before making the request, but only the WAR holds the loaded class, so only the WAR + * can enumerate the overloads and refuse an ambiguous one. Guessing an overload here would + * instrument or patch a method the author did not name and report success either way — the + * wrong-overload analogue of patching a class under the wrong loader. + */ +public final class MethodSelector { + + /** + * The most methods one pattern may arm. + * + *

Every instrumented method costs probe calls on a live JVM's hot path and a share of one + * bounded ring, so a pattern that matched two hundred of them would be a mistake discovered by + * watching the JVM slow down. The number is named in the refusal so it can be argued with rather + * than guessed at. + */ + public static final int MAX_PATTERN_METHODS = 16; + + /** One method a class declares, with the access flags a pattern has to read. */ + public static final class Declared { + final int access; + final Method method; + + public Declared(int access, Method method) { + this.access = access; + this.method = method; + } + } + + /** + * One method a selector named, and whether it can actually be armed. + * + *

A skipped match is reported rather than dropped. A pattern that named four methods and armed + * three is otherwise indistinguishable from one that named three — the caller sees a list of + * three either way — and "it captured less than you asked for, silently" is the failure this + * whole subsystem is built to make impossible. + */ + public static final class Match { + private final Method method; + private final String skipped; + + Match(Method method, String skipped) { + this.method = method; + this.skipped = skipped; + } + + public Method getMethod() { + return method; + } + + /** False when the method matched by name but has no body to instrument. */ + public boolean isInstrumentable() { + return skipped == null; + } + + /** Why this match was not armed, or null when it was. */ + public String getSkipped() { + return skipped; + } + } + + private final String name; + + /** Simple parameter type names, or null when no parameter list was written. */ + private final List params; + + /** An exact JVM descriptor, or null when the author wrote a readable form. */ + private final String descriptor; + + /** + * The parameter list was written {@code (*)}: this exact name, every arity. + * + *

A third thing from the two either side of it. A bare name is a request for ONE method and + * refuses when it is ambiguous; a name glob is a request for everything matching a prefix, which + * reaches names nobody wrote. Neither says "the overloads of this method", which is what a caller + * measuring a cost spread across them actually wants. + */ + private final boolean anyParams; + + /** + * The name carries a glob, or the parameter list does — either way this selector asks for a SET + * rather than one method. Derived rather than passed in, so there is no way to build a selector + * whose flag disagrees with it. + */ + private final boolean pattern; + + private MethodSelector(String name, List params, String descriptor) { + this.name = name; + this.params = params; + this.descriptor = descriptor; + this.anyParams = params != null && params.size() == 1 && "*".equals(params.get(0)); + this.pattern = this.anyParams || name.indexOf('*') >= 0 || name.indexOf('?') >= 0; + } + + public static MethodSelector parse(String spec) { + String trimmed = spec == null ? "" : spec.trim(); + if (trimmed.isEmpty()) { + throw new IllegalArgumentException("method selector is empty"); + } + int open = trimmed.indexOf('('); + if (open < 0) { + String bare = lastJavaIdentifier(trimmed); + if (bare.isEmpty()) { + throw new IllegalArgumentException( + "method selector \"" + spec + "\" does not name a method"); + } + return new MethodSelector(bare, null, null); + } + + String name = lastJavaIdentifier(trimmed.substring(0, open).trim()); + if (name.isEmpty()) { + throw new IllegalArgumentException( + "method selector \"" + spec + "\" does not name a method before \"(\""); + } + int closing = matchParen(trimmed, open); + if (closing < 0) { + throw new IllegalArgumentException( + "method selector \"" + spec + "\" has an unclosed parameter list"); + } + + // A JVM descriptor is the only form carrying a return type AFTER the ')'. + // A lone ';' is a pasted signature someone terminated, not a return + // descriptor — no descriptor return type is a bare semicolon. + String rest = trimmed.substring(closing + 1).trim(); + if (!rest.isEmpty() && !";".equals(rest)) { + return new MethodSelector(name, null, trimmed.substring(open, closing + 1) + rest); + } + + String inner = trimmed.substring(open + 1, closing).trim(); + if (inner.isEmpty()) { + return new MethodSelector(name, Collections.emptyList(), null); + } + List params = new ArrayList(); + for (String part : splitTopLevel(inner)) { + String typeName = parameterTypeName(part); + if (typeName.isEmpty()) { + throw new IllegalArgumentException( + "method selector \"" + spec + "\" has an unreadable parameter \"" + part.trim() + "\""); + } + params.add(typeName); + } + return new MethodSelector(name, params, null); + } + + public String getName() { + return name; + } + + /** + * Reports whether a JVM descriptor satisfies this selector. A selector that named no parameter + * list matches any arity. + */ + public boolean matches(String candidate) { + if (descriptor != null) { + return descriptor.equals(candidate); + } + if (params == null || anyParams) { + return true; + } + Type[] actual; + try { + actual = Type.getArgumentTypes(candidate); + } catch (RuntimeException e) { + return false; + } + if (actual.length != params.size()) { + return false; + } + for (int i = 0; i < actual.length; i++) { + if (!simpleName(actual[i]).equals(params.get(i))) { + return false; + } + } + return true; + } + + /** + * Picks the one method of {@code reader}'s class that this selector names. + * + * @throws NoSuchElementException when nothing matches, listing what the class does declare. + * @throws IllegalArgumentException when several overloads match, listing their descriptors — + * never a silent pick. + */ + public Method resolve(String className, ClassReader reader) { + return resolve(className, declaredMethods(declaredIn(reader))); + } + + /** + * Every method of {@code reader}'s class this selector names. + * + *

A pattern resolves to the whole set; anything else resolves to exactly one, with {@link + * #resolve}'s refusals unchanged. That asymmetry is the point rather than an inconsistency: an + * author writing a name is naming a method, and quietly instrumenting five overloads they did not + * ask for is the same defect as quietly instrumenting one of four — doing other than what was + * written and reporting success either way. + * + * @throws NoSuchElementException when nothing matches, listing what the class declares. + * @throws IllegalArgumentException when a bare name is ambiguous, or a pattern exceeds {@link + * #MAX_PATTERN_METHODS}. + */ + public List resolveAll(String className, ClassReader reader) { + return resolveAll(className, declaredIn(reader)); + } + + List resolveAll(String className, List declared) { + if (!pattern) { + return Collections.singletonList( + new Match(resolve(className, declaredMethods(declared)), null)); + } + List matched = new ArrayList(); + int armed = 0; + for (Declared candidate : declared) { + if (!reachable(candidate) + || !globMatches(name, candidate.method.getName()) + || !matches(candidate.method.getDescriptor())) { + continue; + } + String skipped = noBodyReason(candidate.access); + matched.add(new Match(candidate.method, skipped)); + if (skipped == null) { + armed++; + } + } + if (matched.isEmpty()) { + throw new NoSuchElementException( + className + + " has no method matching " + + this + + "; it declares " + + names(declaredMethods(declared))); + } + if (armed == 0) { + throw new NoSuchElementException( + className + + " has nothing to instrument matching " + + this + + "; every method it names is " + + matched.get(0).getSkipped() + + " (" + + descriptors(matchedMethods(matched)) + + ")"); + } + if (armed > MAX_PATTERN_METHODS) { + throw new IllegalArgumentException( + this + + " matches " + + armed + + " methods of " + + className + + ", and instrumenting them all would put probe calls on that many" + + " hot paths at once; at most " + + MAX_PATTERN_METHODS + + " may be armed together." + + " Narrow the pattern: " + + descriptors(matchedMethods(matched))); + } + return matched; + } + + /** + * Whether a pattern may reach this method at all. + * + *

{@code } is the one whose failure is an {@code ExceptionInInitializerError} that + * breaks its class for the life of the JVM, and {@code } is not something anyone writes a + * pattern to reach. Synthetic and bridge methods — {@code lambda$...} bodies, covariant bridges — + * were emitted by the compiler rather than written by anyone, so a trace full of them is noise + * nobody would recognise. + * + *

These are excluded from the match rather than reported as skipped: a pattern was never + * asking for them, so listing them as things it could not arm would be noise of a different kind. + */ + private static boolean reachable(Declared candidate) { + if (candidate.method.getName().startsWith("<")) { + return false; + } + return (candidate.access & (Opcodes.ACC_SYNTHETIC | Opcodes.ACC_BRIDGE)) == 0; + } + + /** + * Why a matched method has no body to wrap, or null when it has one. + * + *

Reported rather than dropped, because a pattern over an interface's implementations reaches + * abstract declarations routinely, and the caller has to be able to tell "you named four, three + * were armed, the fourth is abstract" from "you named three". + */ + private static String noBodyReason(int access) { + if ((access & Opcodes.ACC_NATIVE) != 0) { + return "native"; + } + if ((access & Opcodes.ACC_ABSTRACT) != 0) { + return "abstract"; + } + return null; + } + + private static List matchedMethods(List matched) { + List out = new ArrayList(matched.size()); + for (Match match : matched) { + out.add(match.getMethod()); + } + return out; + } + + /** + * Glob matching over a method name: {@code *} spans any run of characters, {@code ?} exactly one. + * + *

Iterative with a backtrack point rather than recursive, so a pattern of several stars cannot + * cost exponential time on a name that nearly matches. + */ + static boolean globMatches(String pattern, String candidate) { + int p = 0; + int c = 0; + int star = -1; + int afterStar = 0; + while (c < candidate.length()) { + if (p < pattern.length() + && (pattern.charAt(p) == '?' || pattern.charAt(p) == candidate.charAt(c))) { + p++; + c++; + } else if (p < pattern.length() && pattern.charAt(p) == '*') { + star = p++; + afterStar = c; + } else if (star >= 0) { + // Give the last star one more character and resume after it. + p = star + 1; + c = ++afterStar; + } else { + return false; + } + } + while (p < pattern.length() && pattern.charAt(p) == '*') { + p++; + } + return p == pattern.length(); + } + + private static List declaredIn(ClassReader reader) { + final List declared = new ArrayList(); + reader.accept( + new ClassVisitor(Opcodes.ASM9) { + @Override + public MethodVisitor visitMethod( + int access, + String methodName, + String methodDescriptor, + String signature, + String[] exceptions) { + declared.add(new Declared(access, new Method(methodName, methodDescriptor))); + return null; + } + }, + ClassReader.SKIP_CODE | ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES); + return declared; + } + + private static List declaredMethods(List declared) { + List methods = new ArrayList(declared.size()); + for (Declared candidate : declared) { + methods.add(candidate.method); + } + return methods; + } + + Method resolve(String className, List declared) { + List sameName = new ArrayList(); + List matched = new ArrayList(); + for (Method candidate : declared) { + if (!candidate.getName().equals(name)) { + continue; + } + sameName.add(candidate); + if (matches(candidate.getDescriptor())) { + matched.add(candidate); + } + } + if (matched.size() == 1) { + return matched.get(0); + } + if (matched.isEmpty()) { + if (sameName.isEmpty()) { + throw new NoSuchElementException( + className + " declares no method named " + name + "; it declares " + names(declared)); + } + throw new NoSuchElementException( + className + + " has no " + + name + + " matching " + + this + + "; its overloads are " + + descriptors(sameName)); + } + throw new IllegalArgumentException( + this + + " matches " + + matched.size() + + " overloads of " + + className + + ", and instrumenting the wrong one would capture nothing without saying so." + + " Name one exactly, as a descriptor: " + + descriptors(matched)); + } + + /** Renders the selector the way it would be written, for messages and labels. */ + @Override + public String toString() { + if (descriptor != null) { + return name + descriptor; + } + if (params == null) { + return name; + } + StringBuilder out = new StringBuilder(name).append('('); + for (int i = 0; i < params.size(); i++) { + if (i > 0) { + out.append(", "); + } + out.append(params.get(i)); + } + return out.append(')').toString(); + } + + /** + * The simple type name a descriptor argument reduces to: {@code List}, not {@code + * java.util.List}; {@code Entry}, not {@code Map$Entry}; {@code String[]} for both arrays and + * varargs. That is what a reader writes, and it is unambiguous enough to pick an overload without + * either side having to translate into the other's dialect. + */ + private static String simpleName(Type type) { + String rendered = type.getClassName(); + int bracket = rendered.indexOf('['); + String base = bracket < 0 ? rendered : rendered.substring(0, bracket); + String suffix = bracket < 0 ? "" : rendered.substring(bracket); + int cut = Math.max(base.lastIndexOf('.'), base.lastIndexOf('$')); + return base.substring(cut + 1) + suffix; + } + + /** + * Reduces one declared parameter to its simple type name, discarding annotations, a {@code final} + * modifier, generic arguments and the parameter's own name, and folding varargs into an array. + */ + private static String parameterTypeName(String part) { + String text = part.trim(); + while (text.startsWith("@")) { + int end = 1; + while (end < text.length() && isJavaIdentifier(text.charAt(end))) { + end++; + } + if (end < text.length() && text.charAt(end) == '(') { + int closing = matchParen(text, end); + if (closing >= 0) { + end = closing + 1; + } + } + text = text.substring(end).trim(); + } + if (text.startsWith("final ")) { + text = text.substring("final ".length()).trim(); + } + if (text.isEmpty()) { + return ""; + } + + // Split off a trailing parameter name: the last top-level whitespace run + // that is not inside <>. "List ids" -> "List". + int space = lastTopLevelSpace(text); + if (space >= 0) { + text = text.substring(0, space).trim(); + } + + StringBuilder array = new StringBuilder(); + if (text.endsWith("...")) { + text = text.substring(0, text.length() - 3).trim(); + array.append("[]"); + } + while (text.endsWith("]")) { + int open = text.lastIndexOf('['); + if (open < 0) { + break; + } + text = text.substring(0, open).trim(); + array.append("[]"); + } + int generic = text.indexOf('<'); + if (generic >= 0) { + text = text.substring(0, generic).trim(); + } + int dot = text.lastIndexOf('.'); + if (dot >= 0) { + text = text.substring(dot + 1); + } + return text.isEmpty() ? "" : text + array; + } + + /** + * The trailing method name of {@code s}, which for a pasted signature is the method name — + * everything before it being modifiers, type parameters and the return type. {@code } and + * {@code } keep their brackets so a constructor can be named at all. + * + *

Glob characters count as part of the name, and that is load-bearing rather than incidental. + * Without it the leading walk-back over non-name characters swallows the trailing {@code *} and + * {@code describe*} parses cleanly to {@code describe} — one method armed while the caller + * believes it named a set, which nothing downstream can detect. Neither character appears next to + * a method name in any signature someone would paste: a generic wildcard's {@code ?} lives inside + * the parameter list or the return type, never adjacent to the name. + */ + private static String lastJavaIdentifier(String s) { + int end = s.length(); + while (end > 0 && !isMethodNameChar(s.charAt(end - 1))) { + end--; + } + int start = end; + while (start > 0 && isMethodNameChar(s.charAt(start - 1))) { + start--; + } + String identifier = s.substring(start, end); + if (start > 0 && s.charAt(start - 1) == '<' && end < s.length() && s.charAt(end) == '>') { + return "<" + identifier + ">"; + } + return identifier; + } + + private static boolean isMethodNameChar(char c) { + return c == '*' || c == '?' || isJavaIdentifier(c); + } + + private static boolean isJavaIdentifier(char c) { + return c == '_' + || c == '$' + || (c >= 'a' && c <= 'z') + || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9'); + } + + /** The index of the ')' closing the '(' at {@code open}, or -1 when unbalanced. */ + private static int matchParen(String s, int open) { + int depth = 0; + for (int i = open; i < s.length(); i++) { + if (s.charAt(i) == '(') { + depth++; + } else if (s.charAt(i) == ')' && --depth == 0) { + return i; + } + } + return -1; + } + + /** Splits on ',' outside of {@code <>}, {@code ()} and {@code []}. */ + private static List splitTopLevel(String s) { + List parts = new ArrayList(); + int depth = 0; + int start = 0; + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c == '<' || c == '(' || c == '[') { + depth++; + } else if (c == '>' || c == ')' || c == ']') { + depth--; + } else if (c == ',' && depth == 0) { + parts.add(s.substring(start, i)); + start = i + 1; + } + } + parts.add(s.substring(start)); + return parts; + } + + /** The whitespace separating a declared type from the parameter name, or -1. */ + private static int lastTopLevelSpace(String s) { + int depth = 0; + for (int i = s.length() - 1; i >= 0; i--) { + char c = s.charAt(i); + if (c == '>' || c == ')' || c == ']') { + depth++; + } else if (c == '<' || c == '(' || c == '[') { + depth--; + } else if (depth == 0 && (c == ' ' || c == '\t' || c == '\n' || c == '\r')) { + return i; + } + } + return -1; + } + + private static List names(List methods) { + List out = new ArrayList(); + for (Method method : methods) { + if (!out.contains(method.getName())) { + out.add(method.getName()); + } + } + Collections.sort(out); + return out; + } + + private static List descriptors(List methods) { + List out = new ArrayList(); + for (Method method : methods) { + out.add(method.getName() + method.getDescriptor()); + } + Collections.sort(out); + return out; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/MethodSplicer.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/MethodSplicer.java new file mode 100644 index 0000000..2814f7f --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/MethodSplicer.java @@ -0,0 +1,217 @@ +package com.flanksource.telltale.diag.hotpatch; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.commons.Method; +import org.objectweb.asm.tree.ClassNode; +import org.objectweb.asm.tree.MethodNode; + +/** + * Puts one freshly compiled method into a class's live bytes, leaving every other method exactly as + * it was running. + * + *

This is what replaces "decompile the class, edit the text, recompile the whole thing". A + * whole-class recompile discards anything an APM or tracing agent had woven into the other methods + * — on the very class someone is debugging — and depends on a decompilation being faithful enough + * to compile at all. Here the author's method is compiled against a generated skeleton (see {@link + * SkeletonService}), and only its bytecode crosses over. + */ +public final class MethodSplicer { + + /** + * The class-file method access flags. ASM adds {@code ACC_DEPRECATED} (0x20000) of its own when + * the attribute is present, which is not a modifier and must not be compared as one. + */ + private static final int JVM_METHOD_FLAGS = 0xFFFF; + + /** + * @param liveBytecode what the class is currently running. + * @param compiled every class javac produced from the skeleton, keyed by binary name. + * @param loader the loader that defined the class, used to compute frames. + * @return the live bytes with {@code target}'s body replaced. + */ + public byte[] splice( + String className, + byte[] liveBytecode, + Map compiled, + Method target, + ClassLoader loader) { + final MethodNode replacement = methodFrom(compiled, className, target); + ClassReader reader = new ClassReader(liveBytecode); + ClassWriter writer = new TargetLoaderClassWriter(reader, loader); + final boolean[] spliced = {false}; + + byte[] patched; + try { + // Frames are computed as the instructions are visited, so the + // resolution this can fail on happens here, not in toByteArray. + reader.accept( + new ClassVisitor(Opcodes.ASM9, writer) { + @Override + public MethodVisitor visitMethod( + int access, String name, String descriptor, String signature, String[] exceptions) { + if (!target.getName().equals(name) || !target.getDescriptor().equals(descriptor)) { + // Unwrapped, so ASM copies this method's bytes verbatim. + return super.visitMethod(access, name, descriptor, signature, exceptions); + } + refuseModifierChange(className, target, access, replacement.access); + spliced[0] = true; + // accept() writes the whole method — declaration and body — so + // the original is skipped entirely by returning null below. + replacement.accept(cv); + return null; + } + }, + ClassReader.SKIP_FRAMES); + patched = writer.toByteArray(); + } catch (TypeNotPresentException e) { + throw new IllegalStateException( + "computing stack map frames for " + + className + + "." + + target.getName() + + target.getDescriptor() + + " needs " + + e.typeName() + + ", which its own class loader cannot resolve", + e); + } + + if (!spliced[0]) { + throw new NoSuchElementException( + className + + " no longer declares " + + target.getName() + + target.getDescriptor() + + ", so there is nothing to splice into"); + } + return patched; + } + + /** + * HotSpot rejects a redefinition that changes a modifier, and does so with a bare {@code + * UnsupportedOperationException}. Naming the two modifier sets is the difference between "rewrite + * the declaration" and a dead end. + */ + private static void refuseModifierChange( + String className, Method target, int live, int replacement) { + if ((live & JVM_METHOD_FLAGS) == (replacement & JVM_METHOD_FLAGS)) { + return; + } + throw new IllegalArgumentException( + "the replacement declares " + + className + + "." + + target.getName() + + " with different modifiers than the running method (" + + modifiers(replacement) + + " vs " + + modifiers(live) + + "). A redefinition may change a method's body only," + + " so the declaration must keep the modifiers it has"); + } + + /** + * Reads the one method out of the compiled skeleton, refusing anything the JVM could not then + * accept. + * + *

A redefinition may change method bodies and nothing else, so a compile that produced a + * second class — an anonymous class in the author's body — or a synthetic method — a lambda, or + * access to another class's private member — cannot be applied. Saying so here names the + * construct; letting the JVM say it produces an {@code UnsupportedOperationException} with no + * hint of which one did it. + */ + private static MethodNode methodFrom( + Map compiled, String className, Method target) { + byte[] compiledBytes = compiled.get(className); + if (compiledBytes == null) { + throw new IllegalArgumentException( + "the compile produced no class named " + + className + + " (produced " + + compiled.keySet() + + ")"); + } + List newClasses = new ArrayList(compiled.keySet()); + newClasses.remove(className); + if (!newClasses.isEmpty()) { + Collections.sort(newClasses); + throw new IllegalStateException( + "the replacement body for " + + className + + "." + + target.getName() + + " compiled to extra classes the JVM cannot add to a running system: " + + newClasses + + ". A redefinition may change method bodies only — an anonymous inner class in the" + + " replacement is the usual cause; a named nested class that already exists is" + + " fine"); + } + + ClassNode parsed = new ClassNode(Opcodes.ASM9); + new ClassReader(compiledBytes).accept(parsed, ClassReader.SKIP_FRAMES); + + MethodNode found = null; + List synthetic = new ArrayList(); + for (MethodNode method : parsed.methods) { + if (target.getName().equals(method.name) && target.getDescriptor().equals(method.desc)) { + found = method; + } else if ((method.access & Opcodes.ACC_SYNTHETIC) != 0) { + synthetic.add(method.name + method.desc); + } + } + if (found == null) { + throw new IllegalArgumentException( + "the compiled source does not declare " + + target.getName() + + target.getDescriptor() + + " of " + + className + + ", so there is nothing to splice — the replacement must declare the method being" + + " replaced exactly, including its parameter types"); + } + if (!synthetic.isEmpty()) { + Collections.sort(synthetic); + throw new IllegalStateException( + "the replacement body for " + + className + + "." + + target.getName() + + " needs synthetic methods the JVM cannot add to a loaded class: " + + synthetic + + ". A redefinition may change method bodies only — rewrite the body without the" + + " construct that needed them (a lambda, or access to a private member of another" + + " class)"); + } + return found; + } + + /** Renders method access flags the way they would be written in Java. */ + private static String modifiers(int access) { + StringBuilder out = new StringBuilder(); + append(out, access, Opcodes.ACC_PUBLIC, "public"); + append(out, access, Opcodes.ACC_PROTECTED, "protected"); + append(out, access, Opcodes.ACC_PRIVATE, "private"); + append(out, access, Opcodes.ACC_STATIC, "static"); + append(out, access, Opcodes.ACC_ABSTRACT, "abstract"); + append(out, access, Opcodes.ACC_FINAL, "final"); + append(out, access, Opcodes.ACC_SYNCHRONIZED, "synchronized"); + append(out, access, Opcodes.ACC_NATIVE, "native"); + append(out, access, Opcodes.ACC_STRICT, "strictfp"); + return out.length() == 0 ? "package-private" : out.toString(); + } + + private static void append(StringBuilder out, int access, int flag, String keyword) { + if ((access & flag) != 0) { + out.append(out.length() == 0 ? "" : " ").append(keyword); + } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/PatchService.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/PatchService.java new file mode 100644 index 0000000..820b0ec --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/PatchService.java @@ -0,0 +1,375 @@ +package com.flanksource.telltale.diag.hotpatch; + +import java.lang.instrument.ClassDefinition; +import java.lang.instrument.Instrumentation; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import com.flanksource.telltale.observer.ObserverRuntime; + +/** + * Applies and reverses authored method edits through the JVM transformation chain. + * + *

The source edit coordinator and original-byte registry live on the system class path, so a + * diagnostics WAR redeploy can still inspect and restore edits installed by its previous instance. + * + *

What the JVM permits is narrow, and deliberately not papered over: HotSpot redefinition may + * change method bodies only. Adding or removing a method or field, changing a signature, or + * changing the class hierarchy is rejected by the JVM, and this reports that as the schema-change + * limitation it is rather than as a mystery. + */ +public final class PatchService { + enum RestoreScope { ALL, SOURCE, PROBES } + static final class RestoreRequest { + final ClassIndex.Entry entry; + final RestoreScope scope; + RestoreRequest(ClassIndex.Entry entry, RestoreScope scope) { this.entry = entry; this.scope = scope; } + } + + private static final String ORIGINALS_FIELD = "ORIGINAL_BYTES"; + + private final Instrumentation instrumentation; + private final BytecodeSource bytecode; + + public PatchService(Instrumentation instrumentation) { + this.instrumentation = instrumentation; + this.bytecode = new BytecodeSource(instrumentation); + } + + /** {@code |} — a class is only unique per loader. */ + static String key(String loaderHash, String className) { + return loaderHash + "|" + className; + } + + /** + * The pre-patch bytes for one class: the stashed ones when this is a re-patch, read from the JVM + * only the first time. + * + *

Not just an optimisation. Reading is a retransformation, and on a JVM whose transformer + * chain rebuilds the class that reverts the patch already installed — so re-reading on every + * patch would undo the previous span on the way to arming the next one, and would then record the + * bytes it just reverted TO as this patch's "original". + */ + private byte[] preservedOriginal(ClassIndex.Entry entry, String className, Class type) { + byte[] stashed = originals().get(key(entry.getLoaderHash(), className)); + return stashed != null ? stashed : sourceBytes(instrumentation, type); + } + + static byte[] sourceBytes(Instrumentation instrumentation, Class type) { + if (!TransformOwner.owns(SourceTransformations.class)) return (byte[]) TransformOwner.invoke( + SourceTransformations.class, "source", new Object[] {instrumentation, type}); + if (!ObserverRuntime.installed()) return ProbeTransformations.contains(type) + ? ProbeTransformations.sourceBytes(type) : new BytecodeSource(instrumentation).bytecodeOf(type); + try { return ObserverRuntime.sourceBytes(type); } + catch (Exception error) { throw new IllegalStateException("read source bytecode for " + type.getName(), error); } + } + + /** + * Installs the changed methods from {@code compiled}, remembering pre-patch bytes for reporting. + * + * @param compiled every class javac produced — the top-level class plus its nested and anonymous + * classes, which must be redefined in the same call to stay consistent with each other. + */ + public Map apply( + ClassIndex index, ClassIndex.Entry entry, Map compiled) { + SourceTransformations.install(instrumentation); + // Resolve the stash before anything is redefined. It is the only route + // back, and a JVM still running an older agent jar does not have one — + // reaching it afterwards would leave the class patched with no undo. + Map registry = originals(); + Class target = liveClass(entry); + byte[] replacement = compiled.get(entry.getName()); + if (replacement == null) { + throw new IllegalArgumentException( + "compiled output has no class named " + + entry.getName() + + " (has " + + compiled.keySet() + + ")"); + } + + List definitions = new ArrayList<>(); + Map originals = new LinkedHashMap<>(); + originals.put(entry.getName(), preservedOriginal(entry, entry.getName(), target)); + definitions.add(new ClassDefinition(target, replacement)); + + // Nested classes only come along when they are already loaded; a newly + // introduced one cannot be defined by a redefinition and is reported. + for (Map.Entry produced : compiled.entrySet()) { + if (produced.getKey().equals(entry.getName())) { + continue; + } + ClassIndex.Entry nested = findLoaded(index, entry, produced.getKey()); + if (nested == null) { + throw new IllegalStateException( + "compiling " + + entry.getName() + + " produced a new class " + + produced.getKey() + + " that is not loaded in the JVM." + + " A redefinition cannot introduce classes — remove the new nested or anonymous" + + " class (a lambda or anonymous inner class in added code is the usual cause)"); + } + Class nestedType = liveClass(nested); + originals.put(produced.getKey(), preservedOriginal(entry, produced.getKey(), nestedType)); + definitions.add(new ClassDefinition(nestedType, produced.getValue())); + } + + SourceTransformations.apply(instrumentation, definitions); + + for (Map.Entry original : originals.entrySet()) { + // putIfAbsent: the first patch holds the true original. Re-patching a + // patched class must not overwrite it with already-modified bytes. + registry.putIfAbsent(key(entry.getLoaderHash(), original.getKey()), original.getValue()); + } + return appliedStatus(entry, originals.get(entry.getName()), replacement); + } + + /** + * What a patch installed, reported WITHOUT reading the class back. + * + *

Reading back is not free here: {@link BytecodeSource#bytecodeOf} is implemented as + * retransformClasses, which re-runs every retransform-capable transformer in the JVM and installs + * the result. On a JVM where another transformer contributes bytes derived from its own cached + * original, that re-read UNDOES the redefinition it was meant to verify — measured on the dev + * cycle JVM, where the bytes handed over digested fdba8856 and the class read back as its + * untouched ce23c9b1, with zero probe references left in it and no error anywhere. The same call + * against the web JVM patched cleanly, which is why this looked JVM-specific rather than + * self-inflicted. + * + *

So the report is built from the bytes handed to the JVM. That is a real trade: {@link + * #status} derives from live bytes precisely so it cannot drift from what is running, and this + * cannot make that claim. It states what was installed, and leaves the class alone — which is the + * only way the patch survives being described. + */ + private Map appliedStatus( + ClassIndex.Entry entry, byte[] original, byte[] handed) { + String originalDigest = BytecodeSource.digest(original); + String handedDigest = BytecodeSource.digest(handed); + Map out = new LinkedHashMap<>(); + out.put("class", entry.getName()); + out.put("loader", entry.getLoaderHash()); + out.put("digest", handedDigest); + out.put("originalDigest", originalDigest); + out.put("handedDigest", handedDigest); + out.put("patched", !originalDigest.equals(handedDigest)); + out.put("leaseExpiresAt", AgentBridge.leaseExpiresAt(entry.getLoaderHash(), entry.getName())); + // A rewrite that changed nothing is the other silent failure: the JVM + // takes the definition, the class is identical, and every layer above + // reports success over a method that was never instrumented. + if (originalDigest.equals(handedDigest)) { + out.put( + "error", + "the rewrite of " + + entry.getName() + + " (loader " + + entry.getLoaderHash() + + ") produced bytes identical to the original, so nothing was instrumented" + + " and nothing will be recorded"); + } + return out; + } + + /** Removes authored edits and rebuilds through the current foreign transformer chain. */ + public Map restore(ClassIndex index, RestoreRequest request) { + ClassIndex.Entry entry = request.entry; + Map registry = originals(); + if (request.scope != RestoreScope.SOURCE && ProbeTransformations.contains(liveClass(entry))) { + ProbeTransformations.restore(liveClass(entry)); + if (request.scope == RestoreScope.PROBES || !registry.containsKey(key(entry.getLoaderHash(), entry.getName()))) { + Map result = new LinkedHashMap<>(heldStatus(entry)); + result.put("class", entry.getName()); result.put("loader", entry.getLoaderHash()); + result.put("restored", java.util.Collections.singletonList(entry.getName())); + return result; + } + } + if (request.scope == RestoreScope.PROBES) throw new NoSuchElementException("no method probes are armed on " + entry.getName()); + String prefix = key(entry.getLoaderHash(), entry.getName()); + List> restoredTypes = new ArrayList<>(); + List restored = new ArrayList<>(); + for (Map.Entry stored : registry.entrySet()) { + String storedKey = stored.getKey(); + if (!storedKey.equals(prefix) && !storedKey.startsWith(prefix + "$")) { + continue; + } + String className = storedKey.substring(storedKey.indexOf('|') + 1); + ClassIndex.Entry loaded = findLoaded(index, entry, className); + if (loaded == null) { + continue; + } + restoredTypes.add(liveClass(loaded)); + restored.add(className); + } + if (restoredTypes.isEmpty()) { + throw new NoSuchElementException( + entry.getName() + + " is not patched" + + " (loader " + + entry.getLoaderHash() + + "), so there is nothing to restore"); + } + String handedDigest = BytecodeSource.digest(registry.get(prefix)); + SourceTransformations.restore(restoredTypes); + for (String className : restored) { + registry.remove(key(entry.getLoaderHash(), className)); + } + Map out = new LinkedHashMap<>(); + out.put("class", entry.getName()); + out.put("loader", entry.getLoaderHash()); + out.put("patched", ProbeTransformations.contains(liveClass(entry))); + out.put("restored", restored); + out.put("digest", handedDigest); + out.put("handedDigest", handedDigest); + return out; + } + + /** + * Every class whose bytes currently differ from the originals held for it. + * + *

Derived by comparing live bytes against the stash rather than tracked as metadata, so it + * stays true across a WAR redeploy and cannot drift from what the JVM is actually running. + */ + public List> patches(ClassIndex index) { + List> out = new ArrayList<>(); + for (Class type : ProbeTransformations.types()) { + ClassIndex.Entry entry = ClassIndex.Entry.of(type); + if (!originals().containsKey(key(entry.getLoaderHash(), entry.getName()))) + out.add(status(index, entry.getLoaderHash(), entry.getName())); + } + for (Map.Entry stored : originals().entrySet()) { + int split = stored.getKey().indexOf('|'); + if (split < 0) { + continue; + } + String loaderHash = stored.getKey().substring(0, split); + String className = stored.getKey().substring(split + 1); + try { + out.add(status(index, loaderHash, className)); + } catch (NoSuchElementException | IllegalStateException e) { + // The class went away with its classloader (a webapp redeploy); + // the stash entry is stale but harmless, and saying so beats + // failing the whole listing. + Map row = new LinkedHashMap<>(); + row.put("class", className); + row.put("loader", loaderHash); + row.put("patched", false); + row.put("error", "no longer loaded: " + e.getMessage()); + out.add(row); + } + } + out.sort(Comparator.comparing(row -> String.valueOf(row.get("class")))); + return out; + } + + /** + * Live digest vs the stashed original for one class, and when the agent will put it back on its + * own. + * + *

{@code leaseExpiresAt} is what separates a patch a live span is holding open from one that + * nothing will ever undo. Without it a reader can see that a class is patched but not whether + * walking away is safe, and those are the two states a patch register exists to tell apart. + */ + public Map status(ClassIndex index, String loaderHash, String className) { + ClassIndex.Entry entry = index.resolve(className, loaderHash); + byte[] current = bytecode.bytecodeOf(liveClass(entry)); + byte[] original = originals().get(key(entry.getLoaderHash(), className)); + String currentDigest = BytecodeSource.digest(current); + Map out = new LinkedHashMap<>(); + out.put("class", className); + out.put("loader", entry.getLoaderHash()); + out.put("digest", currentDigest); + out.put("originalDigest", original == null ? "" : BytecodeSource.digest(original)); + out.put("patched", ProbeTransformations.contains(liveClass(entry)) + || original != null && !BytecodeSource.digest(original).equals(currentDigest)); + out.put("leaseExpiresAt", AgentBridge.leaseExpiresAt(entry.getLoaderHash(), className)); + return out; + } + + /** + * Whether a span still holds its patch on one class, and until when — answered from the stash and + * the lease table, WITHOUT reading the class back. + * + *

This is the lease renewal's answer, and the renewal fires every 40s for as long as a span + * holds the patch. {@link #status} is the wrong answer there: its read-back is a + * retransformation, and on the cycle JVM that silently reverts the very patch being renewed (see + * {@link #appliedStatus}). A class is held while its original is stashed; {@link #restore} is + * what clears it. + */ + public Map heldStatus(ClassIndex.Entry entry) { + Map result = heldStatus( + entry.getLoaderHash(), + entry.getName(), + originals(), + AgentBridge.leaseExpiresAt(entry.getLoaderHash(), entry.getName())); + if (ProbeTransformations.contains(liveClass(entry))) result.put("patched", true); + return result; + } + + static Map heldStatus( + String loaderHash, String className, Map stash, long leaseExpiresAt) { + Map out = new LinkedHashMap<>(); + out.put("class", className); + out.put("loader", loaderHash); + out.put("patched", stash.containsKey(key(loaderHash, className))); + out.put("leaseExpiresAt", leaseExpiresAt); + return out; + } + + private static Class liveClass(ClassIndex.Entry entry) { + Class type = entry.getType(); + if (type == null) { + throw new IllegalStateException("no live Class handle for " + entry.getName()); + } + return type; + } + + /** The same class name under the same loader as the class being patched. */ + private static ClassIndex.Entry findLoaded( + ClassIndex index, ClassIndex.Entry sibling, String className) { + try { + return index.resolve(className, sibling.getLoaderHash()); + } catch (NoSuchElementException e) { + return null; + } + } + + /** + * The stash on the system class path, reached reflectively because the agent jar and this WAR are + * loaded by different classloaders. Both the map and its key/value types are JDK types, so it + * crosses that boundary safely. + */ + @SuppressWarnings("unchecked") + static Map originals() { + try { + Class holder = + ClassLoader.getSystemClassLoader().loadClass(InstrumentationService.AGENT_CLASS); + Field field = holder.getField(ORIGINALS_FIELD); + Object value = field.get(null); + if (value == null) { + throw new IllegalStateException( + InstrumentationService.AGENT_CLASS + + "." + + ORIGINALS_FIELD + + " is null — the attached agent jar predates hot patching; redeploy" + + " telltale-diag"); + } + return (Map) value; + } catch (ClassNotFoundException | NoSuchFieldException e) { + throw new IllegalStateException( + "the attached agent jar has no " + + ORIGINALS_FIELD + + " stash, so patches could not be reversed after a redeploy." + + " Restart the JVM to pick up the current agent: " + + e, + e); + } catch (IllegalAccessException e) { + throw new IllegalStateException( + "cannot read " + InstrumentationService.AGENT_CLASS + "." + ORIGINALS_FIELD + ": " + e, + e); + } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/ProbeTransformations.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/ProbeTransformations.java new file mode 100644 index 0000000..bfe1d65 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/ProbeTransformations.java @@ -0,0 +1,192 @@ +package com.flanksource.telltale.diag.hotpatch; + +import java.lang.instrument.ClassFileTransformer; +import java.lang.instrument.Instrumentation; +import java.lang.instrument.UnmodifiableClassException; +import java.security.ProtectionDomain; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Level; +import java.util.logging.Logger; +import com.flanksource.telltale.observer.ObserverRuntime; + +/** Rebuilds live method probes on the current transformer chain's input. */ +public final class ProbeTransformations implements ClassFileTransformer { + private static final Logger LOG = Logger.getLogger(ProbeTransformations.class.getName()); + private static final Map, Target> TARGETS = Collections.synchronizedMap(new WeakHashMap<>()); + private static Instrumentation instrumentation; + private static ProbeTransformations transformer; + private static final InstrumentService INSTRUMENTER = new InstrumentService(); + private static final Map, AtomicReference> READS = new ConcurrentHashMap<>(); + + private static synchronized void install(Instrumentation installed) { + if (instrumentation != null) { + return; + } + if (!installed.isRetransformClassesSupported()) throw new IllegalStateException("method probes require class retransformation"); + SourceTransformations.install(installed); + transformer = new ProbeTransformations(); + installed.addTransformer(transformer, true); + instrumentation = installed; + } + + public static synchronized void shutdown() { + if (!TARGETS.isEmpty()) throw new IllegalStateException("method probes are still active"); + if (transformer != null && !instrumentation.removeTransformer(transformer)) + throw new IllegalStateException("probe transformer was not registered"); + transformer = null; + instrumentation = null; + READS.clear(); + } + + static synchronized InstrumentService.Result arm(Instrumentation installed, ClassIndex.Entry entry, String probeKey) { + if (!TransformOwner.owns(ProbeTransformations.class)) { + @SuppressWarnings("unchecked") Map result = (Map) TransformOwner.invoke( + ProbeTransformations.class, "arm", new Object[] {installed, entry.getType(), probeKey}); + return importResult(result); + } + ObserverRuntime.assertTargetActive(entry.getType()); + if (!installed.isModifiableClass(entry.getType())) throw new IllegalStateException("method probe target is not modifiable: " + entry.getName()); + install(installed); + Target target = new Target(AgentBridge.probeArmingsOwnedBy(entry.getLoaderHash(), entry.getName()), probeKey); + Target previous = TARGETS.put(entry.getType(), target); + try { + retransform(entry.getType()); + if (target.failure != null) throw new IllegalStateException("cannot rebuild method probes on " + entry.getName(), target.failure); + if (target.result == null) throw new IllegalStateException("the agent has no arming for probe " + probeKey + " on " + + entry.getName() + " immediately after opening it, so the rewrite could not be built." + + " The probe was closed underneath this request — a lapsed lease or a restore."); + target.armed = true; + return target.result; + } catch (RuntimeException | LinkageError error) { + // Put back what every other span armed: the failed rebuild left the class without any probe. + if (previous == null) TARGETS.remove(entry.getType()); else TARGETS.put(entry.getType(), previous); + try { + retransform(entry.getType()); + } catch (RuntimeException | LinkageError rearm) { + error.addSuppressed(rearm); + } + throw error; + } + } + + static boolean contains(Class type) { + if (!TransformOwner.owns(ProbeTransformations.class)) + return Boolean.TRUE.equals(TransformOwner.invoke(ProbeTransformations.class, "contains", new Object[] {type})); + return TARGETS.containsKey(type); + } + + static List> types() { + if (!TransformOwner.owns(ProbeTransformations.class)) { + @SuppressWarnings("unchecked") List> result = (List>) TransformOwner.invoke(ProbeTransformations.class, "types", new Object[0]); + return result; + } + synchronized (TARGETS) { return new ArrayList<>(TARGETS.keySet()); } + } + + static byte[] sourceBytes(Class type) { + if (!TransformOwner.owns(ProbeTransformations.class)) + return (byte[]) TransformOwner.invoke(ProbeTransformations.class, "source", new Object[] {type}); + AtomicReference read = new AtomicReference<>(); + if (READS.putIfAbsent(type, read) != null) throw new IllegalStateException("concurrent source read for " + type.getName()); + try { + retransform(type); + if (read.get() == null) throw new IllegalStateException("no probe source bytes for " + type.getName()); + return read.get(); + } finally { READS.remove(type); } + } + + static synchronized void restore(Class type) { + if (!TransformOwner.owns(ProbeTransformations.class)) { + TransformOwner.invoke(ProbeTransformations.class, "restore", new Object[] {type}); return; + } + ObserverRuntime.assertTargetActive(type); + TARGETS.remove(type); + retransform(type); + } + + public static Object dispatch(String operation, Object[] args) { + switch (operation) { + case "arm": return exportResult(arm((Instrumentation) args[0], ClassIndex.Entry.of((Class) args[1]), (String) args[2])); + case "contains": return contains((Class) args[0]); + case "types": return types(); + case "source": return sourceBytes((Class) args[0]); + case "restore": restore((Class) args[0]); return null; + default: throw new IllegalArgumentException("unknown probe transformation operation " + operation); + } + } + + private static Map exportResult(InstrumentService.Result result) { + List> matches = new ArrayList<>(); + for (MethodSelector.Match match : result.getMatched()) { + Map row = new LinkedHashMap<>(); + row.put("name", match.getMethod().getName()); row.put("descriptor", match.getMethod().getDescriptor()); + row.put("skipped", match.getSkipped()); matches.add(row); + } + Map exported = new LinkedHashMap<>(); + exported.put("bytecode", result.getBytecode()); exported.put("matches", matches); + return exported; + } + + @SuppressWarnings("unchecked") + private static InstrumentService.Result importResult(Map result) { + List matches = new ArrayList<>(); + for (Map row : (List>) result.get("matches")) + matches.add(new MethodSelector.Match(new org.objectweb.asm.commons.Method((String) row.get("name"), (String) row.get("descriptor")), (String) row.get("skipped"))); + return new InstrumentService.Result(matches, (byte[]) result.get("bytecode")); + } + + private static void retransform(Class type) { + try { instrumentation.retransformClasses(type); } + catch (UnmodifiableClassException error) { throw new IllegalStateException("cannot retransform method probes on " + type.getName(), error); } + } + + public byte[] transform(ClassLoader loader, String name, Class type, ProtectionDomain domain, byte[] bytes) { + if (type != null) { + AtomicReference read = READS.get(type); + if (read != null) read.compareAndSet(null, bytes.clone()); + ObserverRuntime.captureSource(type, bytes); + } + Target target = type == null ? null : TARGETS.get(type); + if (target == null) return null; + try { + for (Map armed : target.armings) { + InstrumentService.Result result = INSTRUMENTER.instrument(request(type.getName(), loader, bytes, armed)); + bytes = result.getBytecode(); + if (target.probeKey.equals(armed.get("key"))) target.result = result; + } + return bytes; + } catch (Throwable error) { + target.failure = error; + // The JVM discards what a transformer throws. During an arm the failure reaches the caller + // through the target; a later rebuild by anyone else has no caller, so it is logged rather + // than leaving the spans silently recording nothing. + if (target.armed) LOG.log(Level.WARNING, "telltale-diag: could not re-apply the probes armed on " + type.getName() + + " while it was retransformed; they record nothing until the class is armed again", error); + return null; + } + } + + private static InstrumentService.Request request(String name, ClassLoader loader, byte[] bytes, Map armed) { + Object selector = armed.get("method"); + if (!(selector instanceof String)) throw new IllegalStateException("method probe " + armed.get("key") + " has no selector"); + return new InstrumentService.Request(name, bytes, loader, MethodSelector.parse((String) selector), String.valueOf(armed.get("key"))) + .withArgs(Boolean.TRUE.equals(armed.get("args"))) + .withReturn(Boolean.TRUE.equals(armed.get("return"))) + .withException(Boolean.TRUE.equals(armed.get("exception"))) + .withCalls(Boolean.TRUE.equals(armed.get("calls"))) + .withPlatformCalls(Boolean.TRUE.equals(armed.get("platform"))); + } + + private static final class Target { + final List> armings; + final String probeKey; + volatile InstrumentService.Result result; + volatile Throwable failure; + volatile boolean armed; + Target(List> armings, String probeKey) { + this.armings = new ArrayList<>(armings); this.probeKey = probeKey; + } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/SkeletonService.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/SkeletonService.java new file mode 100644 index 0000000..1c82490 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/SkeletonService.java @@ -0,0 +1,390 @@ +package com.flanksource.telltale.diag.hotpatch; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.GenericArrayType; +import java.lang.reflect.Member; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.lang.reflect.TypeVariable; +import java.lang.reflect.WildcardType; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +/** + * Generates a compilable stand-in for a live class, so one replacement method can be compiled + * against the class's own fields and methods without the class's source — and without a decompiler. + * + *

The author's method has to type-check against everything it touches: the fields it reads, the + * helpers it calls, the superclass it inherits from. It does not need any of their + * bodies. This runs inside the JVM holding the loaded {@link Class}, so reflection gives + * that shape exactly — generics included — which a decompilation only approximates and, on a class + * an agent has already instrumented, frequently fails to reproduce at all. + * + *

Only the target method's bytecode is kept ({@link MethodSplicer} splices it into the live + * bytes and discards the rest), so the skeleton is free to be dishonest about everything else: + * every other body throws, and {@code final} is dropped from fields because definite-assignment + * rules would otherwise demand initialisers that serve no purpose here. + */ +public final class SkeletonService { + + /** + * A generated stand-in, and where in it the author's own source begins. + * + *

The line matters as much as the source: javac reports diagnostics against the file it + * compiled, and the author never sees that file. Without the offset, "cannot find symbol at line + * 61" would point into generated boilerplate for a method the author wrote 12 lines of. + */ + public static final class Skeleton { + private final String source; + private final int methodLine; + + Skeleton(String source, int methodLine) { + this.source = source; + this.methodLine = methodLine; + } + + public String getSource() { + return source; + } + + /** 1-based line of {@code methodSource}'s first line within {@link #getSource()}. */ + public int getMethodLine() { + return methodLine; + } + } + + /** + * @param target the live class the replacement belongs to. + * @param methodSource the author's method declaration, verbatim, braces and all. + * @param replaced the method it replaces, so the generated stand-in is left out. + */ + public Skeleton skeleton( + Class target, String methodSource, org.objectweb.asm.commons.Method replaced) { + refuseUnsupported(target); + StringBuilder out = new StringBuilder(); + // Derived from the binary name rather than getPackage(), which answers + // only for a loader that has defined the package object. + int dot = target.getName().lastIndexOf('.'); + if (dot > 0) { + out.append("package ").append(target.getName().substring(0, dot)).append(";\n\n"); + } + // The stand-in bodies are unreachable and the dropped `final`s are + // deliberate; javac's warnings about either would be noise in a report + // the author reads to find a real compile error. + out.append("@SuppressWarnings(\"all\")\n"); + appendDeclaration(out, target); + out.append(" {\n"); + + for (Field field : sorted(target.getDeclaredFields())) { + if (field.isSynthetic()) { + continue; + } + appendField(out, field); + } + for (Constructor constructor : sorted(target.getDeclaredConstructors())) { + if (constructor.isSynthetic() || matches(constructor, replaced)) { + continue; + } + appendConstructor(out, target, constructor); + } + for (Method method : sorted(target.getDeclaredMethods())) { + if (method.isSynthetic() || method.isBridge() || matches(method, replaced)) { + continue; + } + appendMethod(out, method); + } + + out.append('\n'); + int methodLine = 1; + for (int i = 0; i < out.length(); i++) { + if (out.charAt(i) == '\n') { + methodLine++; + } + } + out.append(methodSource.trim()).append("\n}\n"); + return new Skeleton(out.toString(), methodLine); + } + + /** + * The skeleton is a plain class body; an enum's constants, an interface's lack of one and an + * annotation's element syntax are each a different generator. Refusing names the limit instead of + * emitting source that fails to compile for a reason the author cannot act on. + */ + private static void refuseUnsupported(Class target) { + String kind = null; + if (target.isEnum()) { + kind = "an enum"; + } else if (target.isAnnotation()) { + kind = "an annotation"; + } else if (target.isInterface()) { + kind = "an interface"; + } else if (target.isArray() || target.isPrimitive()) { + kind = "not a source-level class"; + } + if (kind != null) { + throw new IllegalArgumentException( + target.getName() + + " is " + + kind + + ", which cannot be given a compilable stand-in." + + " Instrument the method instead (jvm.trace), which needs no compiler"); + } + } + + /** {@code public abstract class Name extends Super implements I} */ + private static void appendDeclaration(StringBuilder out, Class target) { + int access = target.getModifiers() & (Modifier.PUBLIC | Modifier.ABSTRACT | Modifier.FINAL); + if (access != 0) { + out.append(Modifier.toString(access)).append(' '); + } + // A nested class is generated as a top-level class literally named + // Outer$Inner: '$' is a legal identifier character, the binary name is + // what a redefinition matches on, and the alternative — reproducing the + // whole enclosing class — would defeat the point of a skeleton. + out.append("class ").append(simpleBinaryName(target)); + appendTypeParameters(out, target.getTypeParameters()); + if (target.getSuperclass() != null && target.getSuperclass() != Object.class) { + out.append(" extends ").append(typeName(target.getGenericSuperclass())); + } + Type[] interfaces = target.getGenericInterfaces(); + for (int i = 0; i < interfaces.length; i++) { + out.append(i == 0 ? " implements " : ", ").append(typeName(interfaces[i])); + } + } + + private static void appendField(StringBuilder out, Field field) { + // `final` is dropped on purpose: a final field would demand an + // initialiser or a constructor assignment, and no field of this skeleton + // is ever read at runtime. + int access = field.getModifiers() & Modifier.fieldModifiers() & ~Modifier.FINAL; + out.append(" ") + .append(modifiers(access)) + .append(typeName(field.getGenericType())) + .append(' ') + .append(field.getName()) + .append(";\n"); + } + + private static void appendMethod(StringBuilder out, Method method) { + int access = method.getModifiers() & Modifier.methodModifiers(); + out.append(" ").append(modifiers(access)); + if (method.getTypeParameters().length > 0) { + appendTypeParameters(out, method.getTypeParameters()); + out.append(' '); + } + out.append(typeName(method.getGenericReturnType())).append(' ').append(method.getName()); + appendParameters(out, method.getGenericParameterTypes(), method.isVarArgs()); + appendThrows(out, method.getGenericExceptionTypes()); + if (Modifier.isAbstract(access) || Modifier.isNative(access)) { + out.append(";\n"); + return; + } + out.append(" { throw new UnsupportedOperationException(); }\n"); + } + + /** + * A constructor whose body is an explicit {@code super(...)}. + * + *

Explicit because the superclass may have no no-argument constructor — {@code ActivityDal + * extends BaseDal} is the case this exists for. The arguments are there to select an overload, + * never to run: nothing ever constructs this class. + */ + private static void appendConstructor( + StringBuilder out, Class target, Constructor constructor) { + out.append(" ") + .append(modifiers(constructor.getModifiers() & Modifier.constructorModifiers())) + .append(simpleBinaryName(target)); + appendParameters(out, constructor.getGenericParameterTypes(), constructor.isVarArgs()); + appendThrows(out, constructor.getGenericExceptionTypes()); + out.append(" {"); + String superCall = superCall(target.getSuperclass()); + if (!superCall.isEmpty()) { + out.append(' ').append(superCall); + } + out.append(" }\n"); + } + + /** + * {@code super(...)} against the superclass's narrowest visible constructor, every argument cast + * so the overload is unambiguous. + */ + private static String superCall(Class superclass) { + if (superclass == null || superclass == Object.class) { + return ""; + } + Constructor narrowest = null; + for (Constructor candidate : superclass.getDeclaredConstructors()) { + if (candidate.isSynthetic() || Modifier.isPrivate(candidate.getModifiers())) { + continue; + } + if (narrowest == null + || candidate.getParameterTypes().length < narrowest.getParameterTypes().length) { + narrowest = candidate; + } + } + if (narrowest == null) { + throw new IllegalStateException( + superclass.getName() + + " has no constructor this skeleton can call," + + " so its subclass cannot be given a compilable stand-in." + + " Instrument the method instead (jvm.trace), which needs no compiler"); + } + Class[] parameters = narrowest.getParameterTypes(); + if (parameters.length == 0) { + return ""; + } + StringBuilder call = new StringBuilder("super("); + for (int i = 0; i < parameters.length; i++) { + call.append(i == 0 ? "" : ", ") + .append('(') + .append(typeName(parameters[i])) + .append(") ") + .append(zeroValue(parameters[i])); + } + return call.append(");").toString(); + } + + private static void appendParameters(StringBuilder out, Type[] parameters, boolean varargs) { + out.append('('); + for (int i = 0; i < parameters.length; i++) { + String rendered = typeName(parameters[i]); + if (varargs && i == parameters.length - 1 && rendered.endsWith("[]")) { + rendered = rendered.substring(0, rendered.length() - 2) + "..."; + } + out.append(i == 0 ? "" : ", ").append(rendered).append(" p").append(i); + } + out.append(')'); + } + + private static void appendThrows(StringBuilder out, Type[] thrown) { + for (int i = 0; i < thrown.length; i++) { + out.append(i == 0 ? " throws " : ", ").append(typeName(thrown[i])); + } + } + + private static void appendTypeParameters(StringBuilder out, TypeVariable[] variables) { + if (variables.length == 0) { + return; + } + out.append('<'); + for (int i = 0; i < variables.length; i++) { + out.append(i == 0 ? "" : ", ").append(variables[i].getName()); + int written = 0; + for (Type bound : variables[i].getBounds()) { + if (bound == Object.class) { + continue; + } + out.append(written++ == 0 ? " extends " : " & ").append(typeName(bound)); + } + } + out.append('>'); + } + + /** + * A type as it would be written in source. + * + *

{@code Type.getTypeName()} is not usable here: it renders a nested class with its binary + * {@code $} separator, and {@code com.foo.Map$Entry} is not a type reference javac resolves. + * Canonical names are, which is why every {@link Class} goes through {@link + * Class#getCanonicalName()} and the generic forms are composed from the parts rather than taken + * whole. + */ + private static String typeName(Type type) { + if (type instanceof Class) { + Class raw = (Class) type; + String canonical = raw.getCanonicalName(); + // Anonymous and local classes have no canonical name and cannot be + // written in source at all; the binary name at least names what is + // missing when javac then rejects it. + return canonical != null ? canonical : raw.getName(); + } + if (type instanceof ParameterizedType) { + ParameterizedType parameterized = (ParameterizedType) type; + StringBuilder out = new StringBuilder(typeName(parameterized.getRawType())).append('<'); + Type[] arguments = parameterized.getActualTypeArguments(); + for (int i = 0; i < arguments.length; i++) { + out.append(i == 0 ? "" : ", ").append(typeName(arguments[i])); + } + return out.append('>').toString(); + } + if (type instanceof GenericArrayType) { + return typeName(((GenericArrayType) type).getGenericComponentType()) + "[]"; + } + if (type instanceof WildcardType) { + WildcardType wildcard = (WildcardType) type; + for (Type bound : wildcard.getLowerBounds()) { + return "? super " + typeName(bound); + } + for (Type bound : wildcard.getUpperBounds()) { + if (bound != Object.class) { + return "? extends " + typeName(bound); + } + } + return "?"; + } + // A TypeVariable renders as its own name, which the enclosing + // declaration has already introduced. + return type.getTypeName(); + } + + /** + * The name a nested class is declared under here: {@code Outer$Inner}, which is both a legal Java + * identifier and the binary name a redefinition matches. + */ + private static String simpleBinaryName(Class target) { + String name = target.getName(); + int dot = name.lastIndexOf('.'); + return dot < 0 ? name : name.substring(dot + 1); + } + + private static String zeroValue(Class type) { + if (!type.isPrimitive()) { + return "null"; + } + if (type == boolean.class) { + return "false"; + } + if (type == char.class) { + return "'\\0'"; + } + if (type == long.class) { + return "0L"; + } + if (type == float.class) { + return "0F"; + } + if (type == double.class) { + return "0D"; + } + return "0"; + } + + private static boolean matches(Method method, org.objectweb.asm.commons.Method replaced) { + return method.getName().equals(replaced.getName()) + && org.objectweb.asm.Type.getMethodDescriptor(method).equals(replaced.getDescriptor()); + } + + private static boolean matches( + Constructor constructor, org.objectweb.asm.commons.Method replaced) { + return "".equals(replaced.getName()) + && org.objectweb.asm.Type.getConstructorDescriptor(constructor) + .equals(replaced.getDescriptor()); + } + + private static String modifiers(int access) { + String rendered = Modifier.toString(access); + return rendered.isEmpty() ? "" : rendered + " "; + } + + /** Deterministic output, so two skeletons of the same class compare equal. */ + private static List sorted(T[] members) { + List out = new ArrayList(Arrays.asList(members)); + out.sort(Comparator.comparing(Object::toString)); + return out; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/SourceMethodOrigins.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/SourceMethodOrigins.java new file mode 100644 index 0000000..51b63da --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/SourceMethodOrigins.java @@ -0,0 +1,29 @@ +package com.flanksource.telltale.diag.hotpatch; + +import java.io.*; +import java.util.*; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.tree.*; + +final class SourceMethodOrigins { + private SourceMethodOrigins() {} + + static Map read(Class type, MethodNode method) { + try (InputStream input = ClassArtifact.open(type)) { + if (input == null) throw unavailable(type, method, "the authoritative code-source class artifact is missing"); + ClassNode original = new ClassNode(); + new ClassReader(input).accept(original, ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES); + if (!original.name.equals(type.getName().replace('.', '/'))) + throw unavailable(type, method, "the code-source artifact names a different class"); + Map methods = new LinkedHashMap<>(); + for (MethodNode candidate : original.methods) methods.put(candidate.name + candidate.desc, candidate); + return methods; + } catch (IOException | SecurityException | IllegalArgumentException error) { + throw new IllegalStateException(unavailable(type, method, "the code-source artifact cannot be read").getMessage(), error); + } + } + + static IllegalStateException unavailable(Class type, MethodNode method, String reason) { + return new IllegalStateException("safe source editing unavailable for " + type.getName() + "#" + method.name + method.desc + ": " + reason); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/SourceTransformations.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/SourceTransformations.java new file mode 100644 index 0000000..7fae0af --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/SourceTransformations.java @@ -0,0 +1,187 @@ +package com.flanksource.telltale.diag.hotpatch; + +import java.lang.instrument.*; +import java.security.ProtectionDomain; +import java.util.*; +import org.objectweb.asm.*; +import org.objectweb.asm.tree.*; + +/** Applies authored method edits to the latest foreign transformer output. */ +public final class SourceTransformations implements ClassFileTransformer { + private static final Map, Patch> PATCHES = Collections.synchronizedMap(new WeakHashMap<>()); + private static Instrumentation instrumentation; + private static SourceTransformations transformer; + + public static synchronized void install(Instrumentation installed) { + if (!TransformOwner.owns(SourceTransformations.class)) { + TransformOwner.invoke(SourceTransformations.class, "install", new Object[] {installed}); return; + } + if (instrumentation != null) return; + if (!installed.isRetransformClassesSupported()) throw new IllegalStateException("source patches require class retransformation"); + transformer = new SourceTransformations(); + installed.addTransformer(transformer, true); + instrumentation = installed; + } + + public static synchronized void shutdown() { + if (!PATCHES.isEmpty()) throw new IllegalStateException("source patches are still active"); + if (transformer != null && !instrumentation.removeTransformer(transformer)) + throw new IllegalStateException("source transformer was not registered"); + transformer = null; + instrumentation = null; + } + + static synchronized void apply(Instrumentation installed, List definitions) { + if (!TransformOwner.owns(SourceTransformations.class)) { + TransformOwner.invoke(SourceTransformations.class, "apply", new Object[] {installed, definitions}); return; + } + install(installed); + Map, Patch> previous = new LinkedHashMap<>(); + Map, Patch> prepared = new LinkedHashMap<>(); + boolean published = false; + try { + for (ClassDefinition definition : definitions) { + Class type = definition.getDefinitionClass(); + com.flanksource.telltale.observer.ObserverRuntime.assertTargetActive(type); + if (!installed.isModifiableClass(type)) throw new IllegalStateException("source target is not modifiable: " + type.getName()); + Patch before = PATCHES.get(type); + previous.put(type, before); + ClassNode current = read(PatchService.sourceBytes(installed, type)); + ClassNode replacement = read(definition.getDefinitionClassFile()); + requireSameSchema(current, replacement); + Map methods = before == null ? new LinkedHashMap<>() : new LinkedHashMap<>(before.methods); + Map existing = methods(current); + Map origins = null; + for (MethodNode method : replacement.methods) { + String key = key(method); + if (Arrays.equals(fingerprint(existing.get(key)), fingerprint(method))) continue; + MethodNode owned = before == null ? null : before.methods.get(key); + if (owned == null || !digest(existing.get(key)).equals(digest(owned))) { + if (origins == null) origins = SourceMethodOrigins.read(type, method); + MethodNode origin = origins.get(key); + if (origin == null || !digest(existing.get(key)).equals(digest(origin))) + throw SourceMethodOrigins.unavailable(type, method, "runtime method differs from its code-source artifact; an uncoordinated transform would be overwritten"); + } + methods.put(key, method); + } + prepared.put(type, new Patch(methods)); + } + for (Map.Entry, Patch> entry : prepared.entrySet()) PATCHES.put(entry.getKey(), entry.getValue()); + published = true; + retransform(previous.keySet()); + for (Class type : previous.keySet()) + if (PATCHES.get(type).failure != null) throw new IllegalStateException("cannot patch source of " + type.getName(), PATCHES.get(type).failure); + } catch (RuntimeException | LinkageError error) { + if (!published) throw error; + for (Map.Entry, Patch> entry : previous.entrySet()) { + if (entry.getValue() == null) PATCHES.remove(entry.getKey()); else PATCHES.put(entry.getKey(), entry.getValue()); + } + retransform(previous.keySet()); + throw error; + } + } + + static synchronized void restore(Collection> types) { + if (!TransformOwner.owns(SourceTransformations.class)) { + TransformOwner.invoke(SourceTransformations.class, "restore", new Object[] {types}); return; + } + for (Class type : types) + if (!PATCHES.containsKey(type)) throw new NoSuchElementException("no authored source edit for " + type.getName()); + for (Class type : types) com.flanksource.telltale.observer.ObserverRuntime.assertTargetActive(type); + Map, Patch> previous = new LinkedHashMap<>(); + for (Class type : types) previous.put(type, PATCHES.remove(type)); + try { retransform(types); } + catch (RuntimeException | LinkageError error) { + for (Map.Entry, Patch> entry : previous.entrySet()) + if (entry.getValue() != null) PATCHES.put(entry.getKey(), entry.getValue()); + retransform(types); + throw error; + } + } + + static boolean contains(Class type) { + if (!TransformOwner.owns(SourceTransformations.class)) + return Boolean.TRUE.equals(TransformOwner.invoke(SourceTransformations.class, "contains", new Object[] {type})); + return PATCHES.containsKey(type); + } + + @SuppressWarnings("unchecked") + public static Object dispatch(String operation, Object[] args) { + switch (operation) { + case "install": install((Instrumentation) args[0]); return null; + case "apply": apply((Instrumentation) args[0], (List) args[1]); return null; + case "restore": restore((Collection>) args[0]); return null; + case "contains": return contains((Class) args[0]); + case "source": return PatchService.sourceBytes((Instrumentation) args[0], (Class) args[1]); + default: throw new IllegalArgumentException("unknown source transformation operation " + operation); + } + } + + public byte[] transform(ClassLoader loader, String name, Class type, ProtectionDomain domain, byte[] bytes) { + Patch patch = type == null ? null : PATCHES.get(type); + if (patch == null) return null; + try { + ClassNode current = read(bytes); + for (int i = 0; i < current.methods.size(); i++) { + MethodNode original = current.methods.get(i); + MethodNode replacement = patch.methods.get(key(original)); + if (replacement != null) current.methods.set(i, replacement); + } + ClassWriter writer = new ClassWriter(ClassWriter.COMPUTE_MAXS); + current.accept(writer); + return writer.toByteArray(); + } catch (Throwable error) { patch.failure = error; return null; } + } + + private static void retransform(Collection> types) { + if (types.isEmpty()) return; + try { instrumentation.retransformClasses(types.toArray(new Class[0])); } + catch (UnmodifiableClassException error) { throw new IllegalStateException("cannot retransform source patches", error); } + } + + private static ClassNode read(byte[] bytes) { + ClassNode node = new ClassNode(); + new ClassReader(bytes).accept(node, 0); + return node; + } + + private static String key(MethodNode method) { return method.name + method.desc; } + private static String digest(MethodNode method) { return BytecodeSource.digest(fingerprint(method)); } + private static Map methods(ClassNode node) { + Map methods = new LinkedHashMap<>(); + for (MethodNode method : node.methods) methods.put(key(method), method); + return methods; + } + + private static byte[] fingerprint(MethodNode method) { + ClassWriter writer = new ClassWriter(0); + writer.visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC, "MethodBody", null, "java/lang/Object", null); + method.accept(writer); + writer.visitEnd(); + ClassWriter normalized = new ClassWriter(0); + new ClassReader(writer.toByteArray()).accept(normalized, ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES); + return normalized.toByteArray(); + } + + private static void requireSameSchema(ClassNode current, ClassNode replacement) { + if (!Objects.equals(current.name, replacement.name) || current.access != replacement.access + || !Objects.equals(current.superName, replacement.superName) || !current.interfaces.equals(replacement.interfaces) + || !schema(current).equals(schema(replacement))) + throw new IllegalArgumentException("source patches may change method bodies only: " + current.name); + } + + private static Set schema(ClassNode node) { + Set schema = new TreeSet<>(); + for (FieldNode field : node.fields) schema.add("field:" + field.name + field.desc + ":" + field.access); + for (MethodNode method : node.methods) schema.add("method:" + key(method) + ":" + method.access); + return schema; + } + + private static final class Patch { + final Map methods; + volatile Throwable failure; + Patch(Map methods) { + this.methods = Collections.unmodifiableMap(methods); + } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/TargetLoaderClassWriter.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/TargetLoaderClassWriter.java new file mode 100644 index 0000000..55b4c2d --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/TargetLoaderClassWriter.java @@ -0,0 +1,38 @@ +package com.flanksource.telltale.diag.hotpatch; + +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassWriter; + +/** + * A {@link ClassWriter} that computes stack map frames against the class loader of the class being + * rewritten. + * + *

Java 8 class files (major 52) require a {@code StackMapTable}, so frames must be recomputed + * for any method whose instructions change, and computing them means asking "what is the common + * supertype of these two types?" at every merge point. {@link ClassWriter}'s default answer loads + * those types through ASM's own loader — this WAR's — which cannot see a single + * application class. Overriding {@link #getClassLoader()} points that resolution at the loader that + * defined the target, which is the only one guaranteed to resolve every type the target's own code + * mentions. + * + *

Constructing with the {@link ClassReader} is not just an optimisation: it shares the constant + * pool, which is what lets ASM copy untouched methods verbatim instead of re-deriving their frames. + * Methods this WAR does not rewrite therefore keep the exact bytes they were running, including + * whatever an APM or tracing agent had already woven into them. + */ +final class TargetLoaderClassWriter extends ClassWriter { + + private final ClassLoader loader; + + TargetLoaderClassWriter(ClassReader reader, ClassLoader loader) { + super(reader, COMPUTE_FRAMES); + this.loader = loader; + } + + @Override + protected ClassLoader getClassLoader() { + // Null is the bootstrap loader, which is the correct answer for a + // bootstrap-loaded target rather than a missing one. + return loader; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/TransformOwner.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/TransformOwner.java new file mode 100644 index 0000000..f786783 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/hotpatch/TransformOwner.java @@ -0,0 +1,26 @@ +package com.flanksource.telltale.diag.hotpatch; + +import java.lang.reflect.InvocationTargetException; + +final class TransformOwner { + private TransformOwner() {} + + static boolean owns(Class local) { return owner(local) == local; } + + static Object invoke(Class local, String operation, Object[] args) { + try { return owner(local).getMethod("dispatch", String.class, Object[].class).invoke(null, operation, args); } + catch (InvocationTargetException error) { + Throwable cause = error.getCause(); + if (cause instanceof RuntimeException) throw (RuntimeException) cause; + if (cause instanceof Error) throw (Error) cause; + throw new IllegalStateException("JVM transform owner failed: " + operation, cause); + } catch (ReflectiveOperationException error) { throw new IllegalStateException("incompatible JVM transform owner: " + operation, error); } + } + + private static Class owner(Class local) { + try { return Class.forName(local.getName(), false, com.flanksource.telltale.RuntimeOwner.loader()); } + catch (ClassNotFoundException error) { + throw new IllegalStateException("the attached agent has no JVM transform coordinator " + local.getName(), error); + } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/jmx/GcNotifications.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/jmx/GcNotifications.java new file mode 100644 index 0000000..2b10fa6 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/jmx/GcNotifications.java @@ -0,0 +1,107 @@ +package com.flanksource.telltale.diag.jmx; + +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryPoolMXBean; +import java.lang.management.MemoryType; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import javax.management.openmbean.CompositeData; +import javax.management.openmbean.TabularData; + +/** + * Decodes a {@code com.sun.management.gc.notification} — a collector reporting a finished + * collection — into a flat map. + * + *

The user data is the CompositeData form of GarbageCollectionNotificationInfo, read by key + * rather than through {@code com.sun.management} types so the WAR links on any JVM. Its start and + * end times are milliseconds of JVM uptime; they are converted to epoch milliseconds with the + * RuntimeMXBean start time so a reader can line a collection up against everything else it + * captured. + */ +final class GcNotifications { + + /** The notification type every HotSpot collector emits on a finished collection. */ + static final String TYPE = "com.sun.management.gc.notification"; + + private GcNotifications() {} + + /** + * @return {collector, action, cause, id, startTime, endTime, durationMs, pools}, each pool {name, + * heap?, usedBefore, committedBefore, usedAfter, committedAfter}. {@code heap} is left out + * for a pool the JVM no longer lists, rather than guessed. + * @throws IllegalArgumentException when the user data is not the documented shape. + */ + static Map decode(Object userData) { + if (!(userData instanceof CompositeData)) { + throw new IllegalArgumentException( + "GC notification user data is " + + (userData == null ? "null" : userData.getClass().getName()) + + ", not CompositeData"); + } + CompositeData info = (CompositeData) userData; + CompositeData gcInfo = (CompositeData) info.get("gcInfo"); + long jvmStart = ManagementFactory.getRuntimeMXBean().getStartTime(); + Map gc = new LinkedHashMap(); + gc.put("collector", info.get("gcName")); + gc.put("action", info.get("gcAction")); + gc.put("cause", info.get("gcCause")); + gc.put("id", gcInfo.get("id")); + gc.put("startTime", Long.valueOf(jvmStart + ((Number) gcInfo.get("startTime")).longValue())); + gc.put("endTime", Long.valueOf(jvmStart + ((Number) gcInfo.get("endTime")).longValue())); + gc.put("durationMs", gcInfo.get("duration")); + gc.put( + "pools", + pools( + (TabularData) gcInfo.get("memoryUsageBeforeGc"), + (TabularData) gcInfo.get("memoryUsageAfterGc"))); + return gc; + } + + private static List> pools(TabularData before, TabularData after) { + Map afterByPool = usageByPool(after); + Map types = poolTypes(); + List> pools = new ArrayList>(); + for (Map.Entry entry : usageByPool(before).entrySet()) { + CompositeData usedAfter = afterByPool.get(entry.getKey()); + if (usedAfter == null) { + throw new IllegalArgumentException( + "GC notification reports pool " + + entry.getKey() + + " before the collection and not after it"); + } + Map pool = new LinkedHashMap(); + pool.put("name", entry.getKey()); + MemoryType type = types.get(entry.getKey()); + if (type != null) { + pool.put("heap", Boolean.valueOf(type == MemoryType.HEAP)); + } + pool.put("usedBefore", entry.getValue().get("used")); + pool.put("committedBefore", entry.getValue().get("committed")); + pool.put("usedAfter", usedAfter.get("used")); + pool.put("committedAfter", usedAfter.get("committed")); + pools.add(pool); + } + return pools; + } + + /** A {@code Map} in open-type form: rows of {key, value}. */ + private static Map usageByPool(TabularData usage) { + Map byPool = new LinkedHashMap(); + for (Object row : usage.values()) { + CompositeData entry = (CompositeData) row; + byPool.put(String.valueOf(entry.get("key")), (CompositeData) entry.get("value")); + } + return byPool; + } + + private static Map poolTypes() { + Map types = new HashMap(); + for (MemoryPoolMXBean pool : ManagementFactory.getMemoryPoolMXBeans()) { + types.put(pool.getName(), pool.getType()); + } + return types; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/jmx/JmxService.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/jmx/JmxService.java new file mode 100644 index 0000000..5cb1201 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/jmx/JmxService.java @@ -0,0 +1,288 @@ +package com.flanksource.telltale.diag.jmx; + +import java.lang.management.ManagementFactory; +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.management.Attribute; +import javax.management.AttributeList; +import javax.management.MBeanAttributeInfo; +import javax.management.MBeanInfo; +import javax.management.MBeanNotificationInfo; +import javax.management.MBeanOperationInfo; +import javax.management.MBeanParameterInfo; +import javax.management.MBeanServer; +import javax.management.ObjectInstance; +import javax.management.ObjectName; +import javax.management.openmbean.CompositeData; +import javax.management.openmbean.CompositeType; +import javax.management.openmbean.TabularData; + +/** + * Generic JMX access over the process-wide platform MBeanServer — the same one application + * registers its MBeans on. Everything is keyed by {@link ObjectName}, so no application classes are + * needed. Open types (CompositeData/TabularData) are lowered to plain maps/lists for JSON. + */ +public final class JmxService { + + private final MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + + public MBeanServer server() { + return mbs; + } + + /** queryNames matching a pattern (null/"" => all). */ + public List search(String pattern) throws Exception { + ObjectName query = (pattern == null || pattern.isEmpty()) ? null : new ObjectName(pattern); + Set instances = mbs.queryMBeans(query, null); + List names = new ArrayList<>(instances.size()); + for (ObjectInstance oi : instances) { + names.add(oi.getObjectName().getCanonicalName()); + } + return names; + } + + public boolean isRegistered(String objectName) throws Exception { + return mbs.isRegistered(new ObjectName(objectName)); + } + + /** MBeanInfo lowered to a JSON-friendly map with complete operator metadata. */ + public Map info(String objectName) throws Exception { + ObjectName on = new ObjectName(objectName); + MBeanInfo mi = mbs.getMBeanInfo(on); + Map out = new LinkedHashMap<>(); + out.put("objectName", on.getCanonicalName()); + out.put("className", mi.getClassName()); + out.put("description", mi.getDescription()); + List> attrs = new ArrayList<>(); + for (MBeanAttributeInfo a : mi.getAttributes()) { + Map attr = new LinkedHashMap<>(); + attr.put("name", a.getName()); + attr.put("description", a.getDescription()); + attr.put("type", a.getType()); + attr.put("readable", a.isReadable()); + attr.put("writable", a.isWritable()); + attr.put("is", a.isIs()); + attrs.add(attr); + } + out.put("attributes", attrs); + List> ops = new ArrayList<>(); + for (MBeanOperationInfo o : mi.getOperations()) { + Map operation = new LinkedHashMap<>(); + operation.put("name", o.getName()); + operation.put("description", o.getDescription()); + operation.put("returnType", o.getReturnType()); + operation.put("impact", o.getImpact()); + List> signature = new ArrayList<>(); + for (MBeanParameterInfo parameter : o.getSignature()) { + Map item = new LinkedHashMap<>(); + item.put("name", parameter.getName()); + item.put("description", parameter.getDescription()); + item.put("type", parameter.getType()); + signature.add(item); + } + operation.put("signature", signature); + ops.add(operation); + } + out.put("operations", ops); + List> notifications = new ArrayList<>(); + for (MBeanNotificationInfo n : mi.getNotifications()) { + Map notification = new LinkedHashMap<>(); + notification.put("name", n.getName()); + notification.put("description", n.getDescription()); + notification.put("types", java.util.Arrays.asList(n.getNotifTypes())); + notifications.add(notification); + } + out.put("notifications", notifications); + return out; + } + + public Object read(String objectName, String attribute) throws Exception { + return simplify(mbs.getAttribute(new ObjectName(objectName), attribute)); + } + + public Map readAll(String objectName) throws Exception { + ObjectName on = new ObjectName(objectName); + MBeanInfo info = mbs.getMBeanInfo(on); + List names = new ArrayList<>(); + for (MBeanAttributeInfo attribute : info.getAttributes()) { + if (attribute.isReadable()) { + names.add(attribute.getName()); + } + } + AttributeList values = mbs.getAttributes(on, names.toArray(new String[0])); + Map byName = new LinkedHashMap<>(); + for (Object value : values) { + Attribute attribute = (Attribute) value; + byName.put(attribute.getName(), simplify(attribute.getValue())); + } + List> attributes = new ArrayList<>(); + for (String name : names) { + if (!byName.containsKey(name)) { + continue; + } + Map attribute = new LinkedHashMap<>(); + attribute.put("name", name); + attribute.put("value", byName.get(name)); + attributes.add(attribute); + } + Map out = new LinkedHashMap<>(); + out.put("objectName", on.getCanonicalName()); + out.put("attributes", attributes); + return out; + } + + public void write(String objectName, String attribute, Object value) throws Exception { + ObjectName on = new ObjectName(objectName); + String type = attributeType(mbs.getMBeanInfo(on), attribute); + mbs.setAttribute(on, new Attribute(attribute, convert(value, type))); + } + + public Object invoke(String objectName, String operation, Object[] args, String[] signature) + throws Exception { + if (args.length != signature.length) { + throw new IllegalArgumentException("arguments and signature must have the same length"); + } + Object[] converted = new Object[args.length]; + for (int i = 0; i < args.length; i++) { + converted[i] = convert(args[i], signature[i]); + } + return simplify(mbs.invoke(new ObjectName(objectName), operation, converted, signature)); + } + + private static String attributeType(MBeanInfo info, String name) { + for (MBeanAttributeInfo attribute : info.getAttributes()) { + if (attribute.getName().equals(name)) { + if (!attribute.isWritable()) { + throw new IllegalArgumentException("MBean attribute is not writable: " + name); + } + return attribute.getType(); + } + } + throw new IllegalArgumentException("unknown MBean attribute: " + name); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + static Object convert(Object value, String typeName) throws Exception { + if (value == null) { + if (isPrimitive(typeName)) { + throw new IllegalArgumentException("null is invalid for primitive type " + typeName); + } + return null; + } + Class type = resolveType(typeName); + if (type.isInstance(value)) { + return value; + } + String text = String.valueOf(value); + if (type == String.class) return text; + if (type == Boolean.class || type == boolean.class) { + if (!"true".equalsIgnoreCase(text) && !"false".equalsIgnoreCase(text)) + throw new IllegalArgumentException("expected true or false for " + typeName); + return Boolean.valueOf(text); + } + if (type == Byte.class || type == byte.class) return Byte.valueOf(text); + if (type == Short.class || type == short.class) return Short.valueOf(text); + if (type == Integer.class || type == int.class) return Integer.valueOf(text); + if (type == Long.class || type == long.class) return Long.valueOf(text); + if (type == Float.class || type == float.class) return Float.valueOf(text); + if (type == Double.class || type == double.class) return Double.valueOf(text); + if (type == Character.class || type == char.class) { + if (text.length() != 1) throw new IllegalArgumentException("expected one character"); + return text.charAt(0); + } + if (type == ObjectName.class) return new ObjectName(text); + if (type.isEnum()) return Enum.valueOf((Class) type, text); + throw new IllegalArgumentException("unsupported JMX value type: " + typeName); + } + + private static boolean isPrimitive(String typeName) { + return "boolean".equals(typeName) + || "byte".equals(typeName) + || "short".equals(typeName) + || "int".equals(typeName) + || "long".equals(typeName) + || "float".equals(typeName) + || "double".equals(typeName) + || "char".equals(typeName); + } + + private static Class resolveType(String typeName) throws ClassNotFoundException { + if (isPrimitive(typeName)) { + if ("boolean".equals(typeName)) return boolean.class; + if ("byte".equals(typeName)) return byte.class; + if ("short".equals(typeName)) return short.class; + if ("int".equals(typeName)) return int.class; + if ("long".equals(typeName)) return long.class; + if ("float".equals(typeName)) return float.class; + if ("double".equals(typeName)) return double.class; + return char.class; + } + ClassLoader loader = Thread.currentThread().getContextClassLoader(); + return Class.forName( + typeName, true, loader == null ? JmxService.class.getClassLoader() : loader); + } + + /** Lower JMX open types into plain maps/lists so Jackson renders them cleanly. */ + public static Object simplify(Object value) { + if (value == null) { + return null; + } + if (value instanceof CompositeData) { + CompositeData cd = (CompositeData) value; + CompositeType ct = cd.getCompositeType(); + Map m = new LinkedHashMap<>(); + for (String key : ct.keySet()) { + m.put(key, simplify(cd.get(key))); + } + return m; + } + if (value instanceof TabularData) { + List rows = new ArrayList<>(); + for (Object row : ((TabularData) value).values()) { + rows.add(simplify(row)); + } + return rows; + } + if (value instanceof Object[]) { + List list = new ArrayList<>(); + for (Object item : (Object[]) value) { + list.add(simplify(item)); + } + return list; + } + if (value instanceof Map) { + Map out = new LinkedHashMap<>(); + for (Map.Entry entry : ((Map) value).entrySet()) { + out.put(String.valueOf(entry.getKey()), simplify(entry.getValue())); + } + return out; + } + if (value instanceof Iterable) { + List out = new ArrayList<>(); + for (Object item : (Iterable) value) { + out.add(simplify(item)); + } + return out; + } + if (value.getClass().isArray()) { + List out = new ArrayList<>(); + for (int i = 0; i < Array.getLength(value); i++) { + out.add(simplify(Array.get(value, i))); + } + return out; + } + if (value instanceof Enum) { + return ((Enum) value).name(); + } + if (value instanceof String || value instanceof Number || value instanceof Boolean) { + return value; + } + // Fall back to a string form for application-specific / non-open types we can't + // introspect without their classes. + return String.valueOf(value); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/jmx/LoggingService.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/jmx/LoggingService.java new file mode 100644 index 0000000..e5a5d47 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/jmx/LoggingService.java @@ -0,0 +1,39 @@ +package com.flanksource.telltale.diag.jmx; + +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.LogManager; +import java.util.logging.Logger; + +public final class LoggingService { + public List> loggers() { + List> result = new ArrayList<>(); + LogManager manager = LogManager.getLogManager(); + Enumeration names = manager.getLoggerNames(); + while (names.hasMoreElements()) { + String name = names.nextElement(); + Logger logger = manager.getLogger(name); + // JUL keeps weak references; a logger may disappear during enumeration. + if (logger == null) continue; + Map row = new LinkedHashMap<>(); + row.put("name", name); + row.put("level", logger.getLevel() == null ? null : logger.getLevel().getName()); + row.put("parent", logger.getParent() == null ? null : logger.getParent().getName()); + result.add(row); + } + result.sort((left, right) -> ((String) left.get("name")).compareTo((String) right.get("name"))); + return result; + } + + public void setLevel(String name, String level) { + if (name == null) + throw new IllegalArgumentException("logger name is required (empty names the root)"); + Logger logger = LogManager.getLogManager().getLogger(name); + if (logger == null) throw new IllegalArgumentException("Unknown logger: " + name); + logger.setLevel(level == null ? null : Level.parse(level)); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/jmx/NotificationHub.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/jmx/NotificationHub.java new file mode 100644 index 0000000..c42b4f9 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/jmx/NotificationHub.java @@ -0,0 +1,404 @@ +package com.flanksource.telltale.diag.jmx; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.management.MBeanServer; +import javax.management.Notification; +import javax.management.NotificationListener; +import javax.management.ObjectName; + +/** + * Subscribes to JMX notifications on arbitrary MBeans and buffers them per subscriber. One shared + * {@link NotificationListener} is registered per target MBean and fans out to every subscriber of + * that MBean; subscribers poll their own bounded queue (drained by the SSE servlet). + */ +public final class NotificationHub { + + private static final Logger LOG = Logger.getLogger(NotificationHub.class.getName()); + + /** Ring size when a subscriber does not ask for one. */ + public static final int DEFAULT_CAPACITY = 500; + + /** Largest ring a subscriber may ask for; each slot pins one serialized notification. */ + public static final int MAX_CAPACITY = 50_000; + + private final MBeanServer mbs; + private final ConcurrentHashMap subscribers = new ConcurrentHashMap<>(); + private final ConcurrentHashMap listeners = + new ConcurrentHashMap<>(); + + public NotificationHub(MBeanServer mbs) { + this.mbs = mbs; + } + + public static final class Subscriber { + final String id; + + /** What the subscriber asked for: one MBean, or a pattern. */ + final ObjectName target; + + /** The MBeans it listens to: the target itself, or every MBean the pattern matched. */ + final List sources; + + final List filters; + final int capacity; + final BlockingQueue> queue; + final Deque> history; + final AtomicLong dropped = new AtomicLong(); + long writeSeq; + boolean active = true; + + Subscriber( + String id, + ObjectName target, + List sources, + List filters, + int capacity) { + this.id = id; + this.target = target; + this.sources = sources; + this.filters = filters; + this.capacity = capacity; + this.queue = new ArrayBlockingQueue>(capacity); + this.history = new ArrayDeque>(capacity); + } + + boolean accepts(Notification n) { + if (filters == null || filters.isEmpty()) { + return true; + } + String type = n.getType(); + for (String f : filters) { + if (type != null && (type.equals(f) || type.startsWith(f))) { + return true; + } + } + return false; + } + + synchronized void offer(Map event) { + if (!active) { + return; + } + if (!queue.offer(event)) { + queue.poll(); + dropped.incrementAndGet(); + queue.offer(event); + } + if (history.size() == capacity) { + history.removeFirst(); + } + history.addLast(event); + writeSeq++; + } + + synchronized Map read(long since, int limit) { + if (since < 0L) { + throw new IllegalArgumentException("notification cursor must not be negative: " + since); + } + if (limit <= 0) { + throw new IllegalArgumentException("notification limit must be positive: " + limit); + } + if (since > writeSeq) { + throw new IllegalArgumentException( + "notification cursor " + since + " is ahead of write cursor " + writeSeq); + } + long oldest = writeSeq - history.size(); + long start = Math.max(since, oldest); + long skip = start - oldest; + List> events = new ArrayList>(); + int index = 0; + for (Map event : history) { + if (index++ < skip) { + continue; + } + if (events.size() == limit) { + break; + } + events.add(event); + } + Map batch = new LinkedHashMap(); + batch.put("id", id); + batch.put("generation", id); + batch.put("events", events); + batch.put("nextSeq", Long.valueOf(start + events.size())); + batch.put("oldestSeq", Long.valueOf(oldest)); + batch.put("writeSeq", Long.valueOf(writeSeq)); + batch.put("dropped", Long.valueOf(Math.max(0L, oldest - since))); + batch.put("active", Boolean.valueOf(active)); + return batch; + } + + synchronized boolean freeze() { + boolean changed = active; + active = false; + return changed; + } + + synchronized boolean isActive() { + return active; + } + } + + /** + * Register a subscriber on {@code mbeanName} (required). A pattern ({@code + * java.lang:type=GarbageCollector,*}) subscribes to every MBean it matches at registration. + * Ensures a shared listener on each source MBean. Throws (fail loud) if the MBean is absent, a + * pattern matches nothing, a source is not a NotificationBroadcaster, or if {@code capacity} is + * outside 1..{@link #MAX_CAPACITY}. + */ + public synchronized String register(String mbeanName, List filters, int capacity) + throws Exception { + checkCapacity(capacity); + if (mbeanName == null || mbeanName.trim().isEmpty()) + throw new IllegalArgumentException("mbean is required"); + ObjectName target = new ObjectName(mbeanName.trim()); + List sources = sourcesOf(target); + try { + for (ObjectName source : sources) { + ensureListener(source); // throws if not a broadcaster + } + } catch (Exception notBroadcaster) { + // A pattern can fail on its third match; the first two must not keep + // listening for a subscriber that never existed. + for (ObjectName source : sources) { + if (!listenedToBySomeoneElse(source, null)) { + removeListener(source); + } + } + throw notBroadcaster; + } + String id = UUID.randomUUID().toString(); + subscribers.put(id, new Subscriber(id, target, sources, filters, capacity)); + return id; + } + + /** Takes a long so a JSON number beyond int range is refused rather than wrapped. */ + public static int checkCapacity(long capacity) { + if (capacity < 1L || capacity > MAX_CAPACITY) { + throw new IllegalArgumentException( + "notification capacity must be between 1 and " + MAX_CAPACITY + ", got " + capacity); + } + return (int) capacity; + } + + /** + * The registered MBeans {@code target} names: itself, or every match of a pattern, in name order. + */ + private List sourcesOf(ObjectName target) { + if (!target.isPattern()) { + if (!mbs.isRegistered(target)) { + throw new IllegalArgumentException("MBean not registered: " + target); + } + return Collections.singletonList(target); + } + List matched = new ArrayList(mbs.queryNames(target, null)); + if (matched.isEmpty()) { + throw new IllegalArgumentException("no MBean matches pattern " + target); + } + Collections.sort(matched); + return matched; + } + + private void ensureListener(ObjectName target) throws Exception { + // computeIfAbsent can't throw checked exceptions, so guard explicitly. + if (listeners.containsKey(target)) { + return; + } + synchronized (listeners) { + if (listeners.containsKey(target)) { + return; + } + NotificationListener l = + new NotificationListener() { + @Override + public void handleNotification(Notification n, Object handback) { + fanOut(target, n); + } + }; + mbs.addNotificationListener(target, l, null, null); // may throw if not a broadcaster + listeners.put(target, l); + LOG.log(Level.INFO, "telltale: listening on {0}", target); + } + } + + private void fanOut(ObjectName source, Notification n) { + Map event = serialize(n); + for (Subscriber s : subscribers.values()) { + if (!s.sources.contains(source) || !s.accepts(n)) { + continue; + } + s.offer(event); + } + } + + /** + * One notification as the cursor read serves it. A GC notification carries its decoded collection + * as {@code gc} in place of the raw user data; one whose user data will not decode keeps the raw + * data and says why in {@code gcError}, so the failure reaches the reader instead of dying in the + * listener thread. + */ + static Map serialize(Notification n) { + Map m = new LinkedHashMap<>(); + m.put("type", n.getType()); + m.put("sequenceNumber", n.getSequenceNumber()); + m.put("timeStamp", n.getTimeStamp()); + m.put("message", n.getMessage()); + m.put("source", String.valueOf(n.getSource())); + if (GcNotifications.TYPE.equals(n.getType())) { + try { + m.put("gc", GcNotifications.decode(n.getUserData())); + return m; + } catch (RuntimeException undecodable) { + m.put("gcError", String.valueOf(undecodable)); + } + } + m.put("userData", JmxService.simplify(n.getUserData())); + return m; + } + + /** Block up to {@code timeoutMs} for the next notification for a subscriber. */ + public Map poll(String id, long timeoutMs) throws InterruptedException { + Subscriber s = subscribers.get(id); + if (s == null) { + return null; + } + return s.queue.poll(timeoutMs, TimeUnit.MILLISECONDS); + } + + /** + * Non-blocking drain of all currently-buffered notifications for a subscriber. Backs the {@code + * NotificationBuffer} MBean so a Jolokia client can pull notifications via {@code exec drain(id)} + * — Jolokia 1.x supports {@code exec} even though it cannot subscribe to JMX notifications + * itself. + */ + public List> drain(String id) { + Subscriber s = subscribers.get(id); + if (s == null) { + throw new IllegalArgumentException("unknown subscription: " + id); + } + List> out = new ArrayList<>(); + s.queue.drainTo(out); + return out; + } + + /** Non-destructive cursor read used by durable record-store followers. */ + public Map read(String id, long since, int limit) { + Subscriber s = subscribers.get(id); + if (s == null) { + throw new IllegalArgumentException("unknown subscription: " + id); + } + return s.read(since, limit); + } + + /** Stops new notifications while retaining the subscriber's readable history. */ + public boolean freeze(String id) { + Subscriber s = subscribers.get(id); + if (s == null) { + throw new IllegalArgumentException("unknown subscription: " + id); + } + return s.freeze(); + } + + public boolean exists(String id) { + return subscribers.containsKey(id); + } + + public boolean active(String id) { + Subscriber s = subscribers.get(id); + return s != null && s.isActive(); + } + + public int queued(String id) { + Subscriber s = subscribers.get(id); + return s == null ? 0 : s.queue.size(); + } + + public long dropped(String id) { + Subscriber s = subscribers.get(id); + return s == null ? 0 : s.dropped.get(); + } + + public synchronized void deregister(String id) { + Subscriber s = subscribers.get(id); + if (s == null) { + return; + } + s.freeze(); + for (ObjectName source : s.sources) { + if (!listenedToBySomeoneElse(source, s)) { + removeListener(source); + } + } + subscribers.remove(id); + } + + private boolean listenedToBySomeoneElse(ObjectName source, Subscriber excluded) { + for (Subscriber other : subscribers.values()) { + if (other != excluded && other.sources.contains(source)) { + return true; + } + } + return false; + } + + private void removeListener(ObjectName source) { + NotificationListener l = listeners.get(source); + if (l == null) { + return; + } + try { + mbs.removeNotificationListener(source, l); + } catch (javax.management.InstanceNotFoundException + | javax.management.ListenerNotFoundException absent) { + // The broadcaster or listener has already gone; no resource remains to release. + } catch (Exception error) { + throw new IllegalStateException("remove notification listener from " + source, error); + } + listeners.remove(source); + } + + /** Remove all listeners — called on webapp shutdown. */ + public synchronized void shutdown() { + RuntimeException failure = null; + for (String id : new ArrayList<>(subscribers.keySet())) { + try { + deregister(id); + } catch (RuntimeException error) { + if (failure == null) failure = error; + else failure.addSuppressed(error); + } + } + if (failure != null) throw failure; + } + + /** Subscribers currently registered (for diagnostics). */ + public List> list() { + List> out = new ArrayList<>(); + for (Subscriber s : subscribers.values()) { + Map m = new LinkedHashMap<>(); + m.put("id", s.id); + m.put("mbean", s.target.getCanonicalName()); + m.put("filters", s.filters); + m.put("capacity", s.capacity); + m.put("queued", s.queue.size()); + m.put("dropped", s.dropped.get()); + m.put("active", s.isActive()); + out.add(m); + } + return out; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/jvm/HeapHistogramService.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/jvm/HeapHistogramService.java new file mode 100644 index 0000000..203c88e --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/jvm/HeapHistogramService.java @@ -0,0 +1,144 @@ +package com.flanksource.telltale.diag.jvm; + +import java.lang.management.ManagementFactory; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.management.ObjectName; + +/** + * The heap's class histogram — {@code jcmd GC.class_histogram} — read through the {@code + * DiagnosticCommand} MBean and parsed here, so a client receives rows rather than text it would have + * to parse per JDK. + * + *

{@code live:false}, the default, passes {@code -all}: every object in the heap, reachable or + * not, with no collection forced. {@code live:true} counts reachable objects only, which the JVM + * answers by running a FULL GC first — a stop-the-world pause on the JVM being diagnosed, so it is + * never the default. + * + *

{@code classPattern} is a glob over the binary class name ({@code *} any run, {@code ?} one + * character, everything else literal): class names are full of dots and dollars, which a regex would + * make every caller escape. + */ +public final class HeapHistogramService { + + static final int DEFAULT_TOP = 100; + + private static final Pattern ROW = + Pattern.compile("^\\s*(\\d+):\\s+(\\d+)\\s+(\\d+)\\s+(\\S+)(?:\\s+\\((.*)\\))?\\s*$"); + private static final Pattern TOTAL = Pattern.compile("^Total\\s+(\\d+)\\s+(\\d+)\\s*$"); + + public Map histogram(Map params) throws Exception { + int top = top(params.get("top")); + Object pattern = params.get("classPattern"); + if (pattern != null && !(pattern instanceof String)) { + throw new IllegalArgumentException("classPattern must be a string glob, got " + pattern); + } + Object live = params.get("live"); + if (live != null && !(live instanceof Boolean)) { + throw new IllegalArgumentException("live must be a boolean, got " + live); + } + boolean liveOnly = Boolean.TRUE.equals(live); + String[] args = liveOnly ? new String[0] : new String[] {"-all"}; + Object output = + ManagementFactory.getPlatformMBeanServer() + .invoke( + new ObjectName("com.sun.management:type=DiagnosticCommand"), + "gcClassHistogram", + new Object[] {args}, + new String[] {String[].class.getName()}); + return parse(String.valueOf(output), top, (String) pattern, liveOnly); + } + + private static int top(Object raw) { + if (raw == null) { + return DEFAULT_TOP; + } + if (!(raw instanceof Number) || ((Number) raw).intValue() <= 0) { + throw new IllegalArgumentException("top must be a positive integer, got " + raw); + } + return ((Number) raw).intValue(); + } + + /** + * Parses {@code GC.class_histogram} output. + * + * @throws IllegalStateException when it has no Total line: whatever the JVM printed instead (a + * refusal, a changed format) is reported, never an empty heap + */ + static Map parse(String output, int top, String classPattern, boolean live) { + Pattern glob = classPattern == null || classPattern.isEmpty() ? null : glob(classPattern); + List> rows = new ArrayList>(); + Long totalInstances = null; + Long totalBytes = null; + for (String line : output.split("\\r?\\n")) { + Matcher total = TOTAL.matcher(line); + if (total.matches()) { + totalInstances = Long.valueOf(total.group(1)); + totalBytes = Long.valueOf(total.group(2)); + continue; + } + Matcher row = ROW.matcher(line); + if (row.matches() && (glob == null || glob.matcher(row.group(4)).matches())) { + rows.add(row(row)); + } + } + if (totalBytes == null) { + throw new IllegalStateException( + "GC.class_histogram returned no Total line; it said: " + head(output)); + } + Collections.sort(rows, BY_BYTES); + Map histogram = new LinkedHashMap(); + histogram.put("collectedAt", java.time.Instant.now().toString()); + histogram.put("live", Boolean.valueOf(live)); + histogram.put("totalInstances", totalInstances); + histogram.put("totalBytes", totalBytes); + histogram.put("matched", Integer.valueOf(rows.size())); + histogram.put("rows", new ArrayList>(rows.subList(0, Math.min(top, rows.size())))); + return histogram; + } + + private static Map row(Matcher row) { + Map out = new LinkedHashMap(); + out.put("rank", Integer.valueOf(row.group(1))); + out.put("className", row.group(4)); + out.put("instances", Long.valueOf(row.group(2))); + out.put("bytes", Long.valueOf(row.group(3))); + if (row.group(5) != null) { + out.put("loader", row.group(5)); + } + return out; + } + + private static final Comparator> BY_BYTES = + new Comparator>() { + @Override + public int compare(Map a, Map b) { + return Long.compare((Long) b.get("bytes"), (Long) a.get("bytes")); + } + }; + + static Pattern glob(String glob) { + StringBuilder regex = new StringBuilder(); + for (char c : glob.toCharArray()) { + if (c == '*') { + regex.append(".*"); + } else if (c == '?') { + regex.append('.'); + } else { + regex.append(Pattern.quote(String.valueOf(c))); + } + } + return Pattern.compile(regex.toString()); + } + + private static String head(String output) { + String trimmed = output.trim(); + return trimmed.length() <= 500 ? trimmed : trimmed.substring(0, 500) + "…"; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/jvm/JvmMetricsService.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/jvm/JvmMetricsService.java new file mode 100644 index 0000000..85ea292 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/jvm/JvmMetricsService.java @@ -0,0 +1,182 @@ +package com.flanksource.telltale.diag.jvm; + +import com.flanksource.telltale.diag.jmx.JmxService; +import java.lang.management.BufferPoolMXBean; +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryPoolMXBean; +import java.lang.management.MemoryType; +import java.lang.management.MemoryUsage; +import java.lang.management.RuntimeMXBean; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import javax.management.ObjectName; + +/** Captures the target JVM metrics used by the dashboard and timeseries sampler. */ +public final class JvmMetricsService { + + private final JmxService jmx; + + public JvmMetricsService() { + this(new JmxService()); + } + + JvmMetricsService(JmxService jmx) { + this.jmx = jmx; + } + + public Map snapshot() throws Exception { + Map out = new LinkedHashMap<>(); + out.put("capturedAt", Instant.now().toString()); + out.put("memory", memory()); + out.put("garbageCollectors", garbageCollectors()); + out.put("runtime", runtime()); + Object processCPU = processCPUPercent(); + if (processCPU != null) out.put("processCpuPercent", processCPU); + + List> connectors = connectorMetrics(); + List> dataSources = dataSourceMetrics(); + out.put("connectors", connectors); + out.put("dataSources", dataSources); + + Map series = new LinkedHashMap<>(); + series.put("collectors", names(garbageCollectors(), "name")); + series.put("connectors", names(connectors, "name")); + series.put("dataSources", names(dataSources, "name")); + out.put("series", series); + return out; + } + + private static Map memory() { + List> heap = new ArrayList<>(); + List> nonHeap = new ArrayList<>(); + for (MemoryPoolMXBean pool : ManagementFactory.getMemoryPoolMXBeans()) { + Map row = memoryRow(pool.getName(), pool.getType().name(), pool.getUsage()); + if (pool.getType() == MemoryType.HEAP) heap.add(row); + else nonHeap.add(row); + } + List> buffers = new ArrayList<>(); + for (BufferPoolMXBean pool : ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) { + Map row = new LinkedHashMap<>(); + row.put("name", pool.getName()); + row.put("type", "BUFFER"); + row.put("used", pool.getMemoryUsed()); + row.put("committed", pool.getTotalCapacity()); + row.put("max", -1L); + buffers.add(row); + } + Map memory = new LinkedHashMap<>(); + memory.put("heap", heap); + memory.put("nonHeap", nonHeap); + memory.put("buffer", buffers); + return memory; + } + + private static Map memoryRow(String name, String type, MemoryUsage usage) { + Map row = new LinkedHashMap<>(); + row.put("name", name); + row.put("type", type); + row.put("used", usage == null ? 0L : usage.getUsed()); + row.put("committed", usage == null ? 0L : usage.getCommitted()); + row.put("max", usage == null ? -1L : usage.getMax()); + return row; + } + + private static List> garbageCollectors() { + List> collectors = new ArrayList<>(); + for (GarbageCollectorMXBean collector : ManagementFactory.getGarbageCollectorMXBeans()) { + Map row = new LinkedHashMap<>(); + row.put("name", collector.getName()); + row.put("collectionCount", collector.getCollectionCount()); + row.put("collectionTime", collector.getCollectionTime()); + collectors.add(row); + } + return collectors; + } + + private static Map runtime() { + RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean(); + java.lang.management.OperatingSystemMXBean os = ManagementFactory.getOperatingSystemMXBean(); + Map out = new LinkedHashMap<>(); + out.put("javaVersion", System.getProperty("java.version")); + out.put("javaHome", System.getProperty("java.home")); + out.put("osName", os.getName()); + out.put("osVersion", os.getVersion()); + out.put("processors", os.getAvailableProcessors()); + out.put("systemLoadAverage", os.getSystemLoadAverage()); + out.put("uptime", runtime.getUptime()); + return out; + } + + private Object processCPUPercent() { + try { + Object value = jmx.read("java.lang:type=OperatingSystem", "ProcessCpuLoad"); + if (value instanceof Number && ((Number) value).doubleValue() >= 0) { + return ((Number) value).doubleValue() * 100D; + } + } catch (Exception ignored) { + // ProcessCpuLoad is not present on every supported JVM. + } + return null; + } + + private List> connectorMetrics() throws Exception { + List> rows = new ArrayList<>(); + for (String name : jmx.search("Catalina:type=ThreadPool,*")) { + rows.add(metricRow(name, Arrays.asList("currentThreadsBusy", "maxThreads"))); + } + return rows; + } + + private List> dataSourceMetrics() throws Exception { + List names = new ArrayList<>(); + names.addAll(jmx.search("com.zaxxer.hikari:type=Pool (*)")); + names.addAll(jmx.search("openejb.management:ObjectType=datasources,*")); + names.addAll(jmx.search("*:type=ConnectionPool,*")); + List> rows = new ArrayList<>(); + for (String name : names) { + if (name.contains("connections=")) continue; + rows.add( + metricRow( + name, + Arrays.asList( + "ActiveConnections", + "IdleConnections", + "TotalConnections", + "Active", + "Idle", + "Size"))); + } + return rows; + } + + private Map metricRow(String objectName, List wanted) throws Exception { + @SuppressWarnings("unchecked") + List> attributes = + (List>) jmx.readAll(objectName).get("attributes"); + Map row = new LinkedHashMap<>(); + ObjectName parsed = new ObjectName(objectName); + String label = parsed.getKeyProperty("name"); + if (label == null) label = parsed.getKeyProperty("DataSource"); + row.put("name", label == null ? parsed.getCanonicalName() : label); + row.put("objectName", parsed.getCanonicalName()); + for (Map attribute : attributes) { + String name = String.valueOf(attribute.get("name")); + if (wanted.contains(name)) row.put(name, attribute.get("value")); + } + return row; + } + + private static List names(List> rows, String field) { + List out = new ArrayList<>(); + for (Map row : rows) { + Object value = row.get(field); + if (value != null) out.add(String.valueOf(value)); + } + return out; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/jvm/NativeTelemetryService.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/jvm/NativeTelemetryService.java new file mode 100644 index 0000000..6e80f10 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/jvm/NativeTelemetryService.java @@ -0,0 +1,63 @@ +package com.flanksource.telltale.diag.jvm; + +import java.lang.instrument.Instrumentation; +import java.lang.reflect.InvocationTargetException; +import java.time.Instant; +import java.util.*; +import com.flanksource.telltale.diag.hotpatch.ClassIndex; + +public final class NativeTelemetryService { + private static final String TELEMETRY = "com.oipa.telemetry.Telemetry"; + private final Instrumentation instrumentation; + + public NativeTelemetryService(Instrumentation instrumentation) { + this.instrumentation = Objects.requireNonNull(instrumentation, "instrumentation"); + } + + public Map snapshot() throws Exception { + List> applications = new ArrayList<>(); + for (Class type : instrumentation.getAllLoadedClasses()) { + if (!type.getName().equals(TELEMETRY)) continue; + ClassLoader caller = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(type.getClassLoader()); + Map snapshot = read(type); + Map application = new LinkedHashMap<>(); + application.put("loader", ClassIndex.hash(type.getClassLoader())); + application.put("label", ClassIndex.label(type.getClassLoader())); + application.put("status", snapshot.get("status")); + application.put("histograms", snapshot.get("histograms")); + applications.add(application); + } finally { + Thread.currentThread().setContextClassLoader(caller); + } + } + applications.sort(Comparator.comparing(row -> (String) row.get("loader"))); + Map result = new LinkedHashMap<>(); + result.put("capturedAt", Instant.now().toString()); + result.put("applications", applications); + return result; + } + + private Map read(Class type) throws Exception { + Object value; + try { + value = type.getMethod("snapshot").invoke(null); + } catch (NoSuchMethodException error) { + throw new IllegalStateException(TELEMETRY + " in loader " + ClassIndex.hash(type.getClassLoader()) + + " has no native snapshot API; update the telemetry extension and restart the JVM", error); + } catch (InvocationTargetException error) { + if (error.getCause() instanceof Error) throw (Error) error.getCause(); + if (error.getCause() instanceof Exception) throw (Exception) error.getCause(); + throw new IllegalStateException("native telemetry snapshot failed", error.getCause()); + } + if (!(value instanceof Map)) throw new IllegalStateException("native telemetry snapshot must be an object"); + Map snapshot = (Map) value; + if (!(snapshot.get("status") instanceof Map) || !(snapshot.get("histograms") instanceof List)) + throw new IllegalStateException("native telemetry snapshot must contain status and histograms"); + Map status = (Map) snapshot.get("status"); + if (!(status.get("armed") instanceof Boolean) || !(status.get("mode") instanceof String)) + throw new IllegalStateException("native telemetry status must contain armed and mode"); + return snapshot; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/jvm/ResourceSettings.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/jvm/ResourceSettings.java new file mode 100644 index 0000000..d88a2e7 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/jvm/ResourceSettings.java @@ -0,0 +1,74 @@ +package com.flanksource.telltale.diag.jvm; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.util.*; + +public final class ResourceSettings { + private ResourceSettings() {} + + public static Map snapshot() throws IOException { + Map out = new LinkedHashMap<>(); + List> disks = new ArrayList<>(); + for (File root : File.listRoots()) { + Map disk = new LinkedHashMap<>(); + disk.put("path", root.getAbsolutePath()); + disk.put("total", root.getTotalSpace()); + disk.put("usable", root.getUsableSpace()); + disk.put("free", root.getFreeSpace()); + disks.add(disk); + } + out.put("filesystems", disks); + if (!System.getProperty("os.name").equals("Linux")) { + out.put("linux", Collections.singletonMap("status", "not-applicable")); + return out; + } + for (String file : Arrays.asList("limits", "status", "cgroup", "mountinfo")) { + out.put(file, read(Paths.get("/proc/self", file))); + } + out.put("cgroups", cgroups(Paths.get("/proc/self"))); + return out; + } + + static Map cgroups(Path proc) throws IOException { + Map out = new LinkedHashMap<>(); + for (String group : read(proc.resolve("cgroup")).split("\n")) { + String[] membership = group.split(":", 3); + if (membership.length != 3) throw new IOException("invalid cgroup membership: " + group); + for (String mount : read(proc.resolve("mountinfo")).split("\n")) { + String[] fields = mount.split(" "); + int separator = Arrays.asList(fields).indexOf("-"); + if (separator < 0 || fields.length <= separator + 3) throw new IOException("invalid mountinfo"); + boolean v2 = fields[separator + 1].equals("cgroup2") && membership[1].isEmpty(); + boolean v1 = fields[separator + 1].equals("cgroup") && !membership[1].isEmpty() + && Arrays.asList(fields[separator + 3].split(",")).containsAll(Arrays.asList(membership[1].split(","))); + if (!v1 && !v2) continue; + String root = fields[3].replace("\\040", " "); + String path = membership[2]; + if (!root.equals("/") && path.startsWith(root + "/")) path = path.substring(root.length()); + else if (path.equals(root)) path = "/"; + Path directory = Paths.get(fields[4].replace("\\040", " ")).resolve(path.substring(1)).normalize(); + Map limits = new LinkedHashMap<>(); + for (String file : Arrays.asList("cpu.max", "cpu.stat", "cpuset.cpus.effective", "memory.max", "memory.current", "memory.events", "cpu.cfs_quota_us", "cpu.cfs_period_us", "memory.limit_in_bytes", "memory.usage_in_bytes")) { + Path setting = directory.resolve(file); + if (Files.exists(setting)) limits.put(file, read(setting)); + } + out.put(directory.toString(), limits); + } + } + return out; + } + + private static String read(Path path) throws IOException { + try (InputStream input = Files.newInputStream(path); ByteArrayOutputStream out = new ByteArrayOutputStream()) { + byte[] buffer = new byte[8192]; + int count; + while ((count = input.read(buffer)) != -1) { + if (out.size() + count > 1024 * 1024) throw new IOException(path + " exceeds 1 MiB"); + out.write(buffer, 0, count); + } + return new String(out.toByteArray(), StandardCharsets.UTF_8); + } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/jvm/SettingsService.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/jvm/SettingsService.java new file mode 100644 index 0000000..7868225 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/jvm/SettingsService.java @@ -0,0 +1,26 @@ +package com.flanksource.telltale.diag.jvm; + +import com.flanksource.telltale.diag.jmx.JmxService; +import java.lang.management.*; +import java.util.*; +import javax.management.*; + +public final class SettingsService { + private SettingsService() {} + + public static Map snapshot() throws Exception { + Map out = new LinkedHashMap<>(); + out.put("properties", new TreeMap<>(ManagementFactory.getRuntimeMXBean().getSystemProperties())); + out.put("environment", new TreeMap<>(System.getenv())); + out.put("runtime", new JmxService().readAll("java.lang:type=Runtime")); + out.put("operatingSystem", new JmxService().readAll("java.lang:type=OperatingSystem")); + return out; + } + + public static String command(String command, String... args) throws Exception { + MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + ObjectName name = new ObjectName("com.sun.management:type=DiagnosticCommand"); + if (!server.isRegistered(name)) throw new UnsupportedOperationException("HotSpot DiagnosticCommand MBean is unavailable"); + return (String) server.invoke(name, command, new Object[] {args}, new String[] {String[].class.getName()}); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/diag/jvm/ThreadDumpService.java b/sdk/java/src/main/java/com/flanksource/telltale/diag/jvm/ThreadDumpService.java new file mode 100644 index 0000000..a38f2da --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/diag/jvm/ThreadDumpService.java @@ -0,0 +1,345 @@ +package com.flanksource.telltale.diag.jvm; + +import java.lang.management.LockInfo; +import java.lang.management.ManagementFactory; +import java.lang.management.MonitorInfo; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; + +/** + * Read-only thread diagnostics over {@link ThreadMXBean}: a CPU-sampled thread table, one thread's + * full stack, and a jstack-style dump of every thread. + * + *

Nothing is switched on to answer: thread contention monitoring stays as the JVM has it, so + * blocked/waited times are {@code null} when it is off rather than a {@code 0} that reads as "never + * waited". The same holds for CPU time. + * + *

The text is formatted here rather than by {@link ThreadInfo#toString()}, which stops after + * eight frames — the frames below them are usually the ones that say which application call the + * thread is in. + */ +public final class ThreadDumpService { + + public static final int DEFAULT_SAMPLE_MS = 200; + public static final int MIN_SAMPLE_MS = 50; + public static final int MAX_SAMPLE_MS = 2000; + + private static final long NANOS_PER_MS = 1_000_000L; + + private final ThreadMXBean threads; + + public ThreadDumpService() { + this(ManagementFactory.getThreadMXBean()); + } + + ThreadDumpService(ThreadMXBean threads) { + this.threads = threads; + } + + /** + * Every live thread, with CPU % measured as the thread CPU time it used over {@code sampleMs} of + * wall clock. The request blocks for that window. + */ + public Map snapshot(int sampleMs) throws InterruptedException { + if (sampleMs < MIN_SAMPLE_MS || sampleMs > MAX_SAMPLE_MS) { + throw new IllegalArgumentException( + "sampleMs must be between " + + MIN_SAMPLE_MS + + " and " + + MAX_SAMPLE_MS + + ", got " + + sampleMs); + } + boolean cpuEnabled = cpuTimeEnabled(); + Map before = cpuEnabled ? cpuTimes(threads.getAllThreadIds()) : null; + long startedAt = System.nanoTime(); + Thread.sleep(sampleMs); + long[] ids = threads.getAllThreadIds(); + Map after = cpuEnabled ? cpuTimes(ids) : null; + long elapsed = System.nanoTime() - startedAt; + + ThreadInfo[] infos = threads.getThreadInfo(ids); + Map live = liveThreads(); + List> rows = new ArrayList<>(); + for (ThreadInfo info : infos) { + // Null, or absent from the live set, is a thread that ended during the + // read: it no longer exists to be diagnosed. + if (info == null || !live.containsKey(info.getThreadId())) continue; + long id = info.getThreadId(); + Double cpuPercent = null; + Long cpuTimeMs = null; + if (cpuEnabled) { + Long used = after.get(id); + if (used == null) continue; + // A thread started inside the window had used no CPU before it. + Long base = before.containsKey(id) ? before.get(id) : 0L; + cpuPercent = Math.round((used - base) * 1000D / elapsed) / 10D; + cpuTimeMs = used / NANOS_PER_MS; + } + rows.add(row(info, live.get(id), cpuPercent, cpuTimeMs)); + } + + Map out = new LinkedHashMap<>(); + out.put("sampledMs", sampleMs); + out.put("cpuTimeEnabled", cpuEnabled); + out.put("deadlocked", deadlocked()); + out.put("threads", rows); + return out; + } + + /** One thread's row, its jstack-format text, and its frames, at full depth. */ + public Map stack(long id) { + ThreadInfo info = + threads + .getThreadInfo( + new long[] {id}, + threads.isObjectMonitorUsageSupported(), + threads.isSynchronizerUsageSupported())[0]; + Thread thread = liveThreads().get(id); + if (info == null || thread == null) { + throw new NoSuchElementException("thread " + id + " is not alive in this JVM"); + } + Long cpuTimeMs = null; + if (cpuTimeEnabled()) { + long used = threads.getThreadCpuTime(id); + if (used < 0) + throw new NoSuchElementException("thread " + id + " ended while it was being read"); + cpuTimeMs = used / NANOS_PER_MS; + } + List> frames = new ArrayList<>(); + for (StackTraceElement element : info.getStackTrace()) { + Map frame = new LinkedHashMap<>(); + frame.put("className", element.getClassName()); + frame.put("methodName", element.getMethodName()); + frame.put("fileName", element.getFileName()); + frame.put("lineNumber", element.getLineNumber()); + frames.add(frame); + } + Map out = new LinkedHashMap<>(); + out.put("thread", row(info, thread, null, cpuTimeMs)); + out.put("text", format(info, thread)); + out.put("frames", frames); + return out; + } + + /** Every thread in jstack format, then the deadlocks, if any. */ + public String dump() { + ThreadInfo[] infos = + threads.dumpAllThreads( + threads.isObjectMonitorUsageSupported(), threads.isSynchronizerUsageSupported()); + Map live = liveThreads(); + StringBuilder out = new StringBuilder(); + Map byId = new HashMap<>(); + for (ThreadInfo info : infos) { + if (info == null) continue; + byId.put(info.getThreadId(), info); + out.append(format(info, live.get(info.getThreadId()))).append('\n'); + } + out.append(deadlockReport(deadlocked(), byId)); + return out.toString(); + } + + private boolean cpuTimeEnabled() { + return threads.isThreadCpuTimeSupported() && threads.isThreadCpuTimeEnabled(); + } + + /** Thread id to CPU nanoseconds, leaving out threads that ended (-1). */ + private Map cpuTimes(long[] ids) { + Map out = new HashMap<>(); + for (long id : ids) { + long used = threads.getThreadCpuTime(id); + if (used >= 0) out.put(id, used); + } + return out; + } + + /** + * Live {@link Thread}s by id: {@code ThreadInfo.isDaemon()} and {@code getPriority()} are Java + * 9+, and this WAR runs on Java 8. + */ + private static Map liveThreads() { + Map out = new HashMap<>(); + for (Thread thread : Thread.getAllStackTraces().keySet()) { + out.put(thread.getId(), thread); + } + return out; + } + + private List deadlocked() { + long[] ids = threads.findDeadlockedThreads(); + if (ids == null) return Collections.emptyList(); + List out = new ArrayList<>(); + for (long id : ids) out.add(id); + return out; + } + + private Map row( + ThreadInfo info, Thread thread, Double cpuPercent, Long cpuTimeMs) { + boolean contention = + threads.isThreadContentionMonitoringSupported() + && threads.isThreadContentionMonitoringEnabled(); + Map row = new LinkedHashMap<>(); + row.put("id", info.getThreadId()); + row.put("name", info.getThreadName()); + row.put("state", info.getThreadState().name()); + row.put("daemon", thread.isDaemon()); + row.put("priority", thread.getPriority()); + row.put("cpuPercent", cpuPercent); + row.put("cpuTimeMs", cpuTimeMs); + row.put("blockedCount", info.getBlockedCount()); + row.put("blockedTimeMs", contention ? info.getBlockedTime() : null); + row.put("waitedCount", info.getWaitedCount()); + row.put("waitedTimeMs", contention ? info.getWaitedTime() : null); + row.put("lockName", info.getLockName()); + row.put("lockOwnerId", info.getLockOwnerId() < 0 ? null : info.getLockOwnerId()); + row.put("lockOwnerName", info.getLockOwnerName()); + row.put("inNative", info.isInNative()); + row.put("suspended", info.isSuspended()); + return row; + } + + /** One thread in jstack format; {@code thread} is null when it ended mid-dump. */ + static String format(ThreadInfo info, Thread thread) { + StringBuilder out = new StringBuilder(); + out.append('"').append(info.getThreadName()).append("\" #").append(info.getThreadId()); + if (thread != null) { + if (thread.isDaemon()) out.append(" daemon"); + out.append(" prio=").append(thread.getPriority()); + } + out.append('\n'); + out.append(" java.lang.Thread.State: ").append(info.getThreadState()).append('\n'); + StackTraceElement[] stack = info.getStackTrace(); + MonitorInfo[] monitors = info.getLockedMonitors(); + for (int depth = 0; depth < stack.length; depth++) { + out.append("\tat ").append(frame(stack[depth])).append('\n'); + if (depth == 0) appendBlockedOn(out, info, stack[0]); + for (MonitorInfo monitor : monitors) { + if (monitor.getLockedStackDepth() == depth) { + out.append("\t- locked ").append(lock(monitor)).append('\n'); + } + } + } + out.append("\n Locked ownable synchronizers:\n"); + LockInfo[] synchronizers = info.getLockedSynchronizers(); + if (synchronizers.length == 0) { + out.append("\t- None\n"); + } + for (LockInfo synchronizer : synchronizers) { + out.append("\t- ").append(lock(synchronizer)).append('\n'); + } + return out.toString(); + } + + /** The jstack line under frame 0 naming what the thread is stopped on. */ + private static void appendBlockedOn(StringBuilder out, ThreadInfo info, StackTraceElement top) { + LockInfo lock = info.getLockInfo(); + if (lock == null) return; + String verb; + switch (info.getThreadState()) { + case BLOCKED: + verb = "waiting to lock "; + break; + case WAITING: + case TIMED_WAITING: + boolean parked = + top.getClassName().endsWith(".Unsafe") && "park".equals(top.getMethodName()); + verb = parked ? "parking to wait for " : "waiting on "; + break; + default: + return; + } + out.append("\t- ").append(verb).append(lock(lock)); + if (info.getLockOwnerName() != null) { + out.append(" owned by \"") + .append(info.getLockOwnerName()) + .append("\" Id=") + .append(info.getLockOwnerId()); + } + out.append('\n'); + } + + private static String frame(StackTraceElement element) { + String location; + if (element.isNativeMethod()) { + location = "Native Method"; + } else if (element.getFileName() == null) { + location = "Unknown Source"; + } else if (element.getLineNumber() >= 0) { + location = element.getFileName() + ":" + element.getLineNumber(); + } else { + location = element.getFileName(); + } + return element.getClassName() + "." + element.getMethodName() + "(" + location + ")"; + } + + private static String lock(LockInfo lock) { + return String.format("<0x%08x> (a %s)", lock.getIdentityHashCode(), lock.getClassName()); + } + + /** + * jstack's trailer: each deadlock cycle, as who waits for what and who holds it. {@code + * findDeadlockedThreads} returns the threads, not the cycles, so the cycles are recovered by + * following lock owners. + */ + private static String deadlockReport(List deadlocked, Map byId) { + if (deadlocked.isEmpty()) return ""; + List> cycles = deadlockCycles(deadlocked, byId); + StringBuilder out = new StringBuilder(); + out.append("Found ") + .append(cycles.size()) + .append(cycles.size() == 1 ? " deadlock:" : " deadlocks:") + .append("\n"); + for (List cycle : cycles) { + out.append('\n'); + for (ThreadInfo info : cycle) { + out.append('"') + .append(info.getThreadName()) + .append("\" Id=") + .append(info.getThreadId()) + .append(":\n waiting for ") + .append(lock(info.getLockInfo())) + .append(",\n which is held by \"") + .append(info.getLockOwnerName()) + .append("\" Id=") + .append(info.getLockOwnerId()) + .append('\n'); + } + } + return out.toString(); + } + + private static List> deadlockCycles( + List deadlocked, Map byId) { + Set visited = new HashSet<>(); + List> cycles = new ArrayList<>(); + for (Long start : deadlocked) { + LinkedHashSet path = new LinkedHashSet<>(); + Long current = start; + while (current != null && !visited.contains(current) && byId.containsKey(current)) { + path.add(current); + visited.add(current); + long owner = byId.get(current).getLockOwnerId(); + current = owner < 0 ? null : owner; + } + if (current == null || !path.contains(current)) continue; + List cycle = new ArrayList<>(); + boolean inCycle = false; + for (Long id : path) { + inCycle = inCycle || id.equals(current); + if (inCycle) cycle.add(byId.get(id)); + } + cycles.add(cycle); + } + return cycles; + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/observer/CallbackIndex.java b/sdk/java/src/main/java/com/flanksource/telltale/observer/CallbackIndex.java new file mode 100644 index 0000000..fc04520 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/observer/CallbackIndex.java @@ -0,0 +1,69 @@ +package com.flanksource.telltale.observer; + +import java.lang.ref.*; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.*; +import com.flanksource.telltale.diag.hotpatch.ClassArtifact; +import org.objectweb.asm.ClassReader; + +/** Classfile summaries contain no Class or strong ClassLoader references. */ +final class CallbackIndex { + private static final int MAX_CLASSES = 65536; + private final ReferenceQueue released = new ReferenceQueue<>(); + private final Map classes = new LinkedHashMap(1024, .75f, true) { + protected boolean removeEldestEntry(Map.Entry entry) { return size() > MAX_CLASSES; } + }; + + synchronized Boolean known(ClassLoader loader, String name) { + purge(); return classes.get(new Key(loader, name, null)); + } + + Boolean known(Class type) { + Boolean known = known(type.getClassLoader(), type.getName()); + if (known != null) return known; + try (InputStream input = ClassArtifact.open(type); ByteArrayOutputStream output = new ByteArrayOutputStream()) { + if (input == null) return null; + byte[] buffer = new byte[4096]; + for (int count; (count = input.read(buffer)) != -1;) output.write(buffer, 0, count); + byte[] bytes = output.toByteArray(); + if (!new ClassReader(bytes).getClassName().equals(type.getName().replace('.', '/'))) { + throw new IllegalArgumentException("code-source artifact names a different class"); + } + return remember(type.getClassLoader(), type.getName(), bytes); + } catch (IOException | SecurityException | IllegalArgumentException error) { + throw new IllegalStateException("cannot inspect callback artifact for " + type.getName(), error); + } + } + + boolean remember(ClassLoader loader, String name, byte[] bytes) { + String pool = new String(bytes, StandardCharsets.ISO_8859_1); + boolean callback = pool.contains("/jms/MessageListener") || pool.contains("/jms/CompletionListener"); + synchronized (this) { purge(); classes.put(new Key(loader, name, released), callback); } + return callback; + } + + private void purge() { + for (Reference dead; (dead = released.poll()) != null;) classes.remove(dead); + } + + private static final class Key extends WeakReference { + final String name; + final int hash; + final boolean bootstrap; + Key(ClassLoader loader, String name, ReferenceQueue queue) { + super(loader, queue); this.name = name; bootstrap = loader == null; + hash = 31 * System.identityHashCode(loader) + name.hashCode(); + } + public int hashCode() { return hash; } + public boolean equals(Object other) { + if (this == other) return true; + if (!(other instanceof Key)) return false; + Key key = (Key) other; + return hash == key.hash && name.equals(key.name) && bootstrap == key.bootstrap + && (bootstrap || get() != null && get() == key.get()); + } + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/observer/LoaderLifecycle.java b/sdk/java/src/main/java/com/flanksource/telltale/observer/LoaderLifecycle.java new file mode 100644 index 0000000..aa9157d --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/observer/LoaderLifecycle.java @@ -0,0 +1,35 @@ +package com.flanksource.telltale.observer; + +import java.util.*; + +/** A retired application loader forbids verification and is no longer a live capture target. */ +final class LoaderLifecycle { + private LoaderLifecycle() {} + + static Map retired(ClassLoader loader) { + if (loader == null) return null; + Class contract = loader.getClass(); + String target = OtelTargets.name("loader.tomcat.type"); + while (contract != null && !contract.getName().equals(target)) contract = contract.getSuperclass(); + if (contract == null) return null; + try { + Object value = contract.getMethod("getState").invoke(loader); + if (!(value instanceof Enum)) throw new IllegalStateException("Tomcat loader lifecycle state is not an enum: " + value); + String state = ((Enum) value).name(); + if (!Arrays.asList(OtelTargets.name("loader.tomcat.retiredStates").split(",")).contains(state)) return null; + Map retired = new LinkedHashMap<>(); + retired.put("id", Integer.toHexString(System.identityHashCode(loader))); + retired.put("class", loader.getClass().getName()); + retired.put("state", state); + return retired; + } catch (ReflectiveOperationException error) { + throw new IllegalStateException("read application loader lifecycle " + loader.getClass().getName(), error); + } + } + + static void requireActive(Class type) { + Map retired = retired(type.getClassLoader()); + if (retired != null) throw new IllegalStateException("cannot instrument retired application class " + + type.getName() + ": loader " + retired.get("id") + " is " + retired.get("state")); + } +} diff --git a/sdk/java/src/main/java/com/flanksource/telltale/observer/MethodFrames.java b/sdk/java/src/main/java/com/flanksource/telltale/observer/MethodFrames.java new file mode 100644 index 0000000..04ffa55 --- /dev/null +++ b/sdk/java/src/main/java/com/flanksource/telltale/observer/MethodFrames.java @@ -0,0 +1,95 @@ +package com.flanksource.telltale.observer; + +import java.util.*; +import org.objectweb.asm.*; +import org.objectweb.asm.commons.AnalyzerAdapter; +import org.objectweb.asm.tree.*; + +final class MethodFrames extends MethodVisitor { + private final AnalyzerAdapter analyzer; + final Map hookLocals = new LinkedHashMap<>(); + int lastStore; + + MethodFrames(String owner, int access, String name, String descriptor, MethodVisitor delegate, boolean mapped) { + super(Opcodes.ASM9, mapped ? new AnalyzerAdapter(owner, access, name, descriptor, delegate) : delegate); + analyzer = mapped ? (AnalyzerAdapter) mv : null; + } + + public void visitVarInsn(int opcode, int variable) { + super.visitVarInsn(opcode, variable); + if (opcode >= Opcodes.ISTORE && opcode <= Opcodes.ASTORE) lastStore = variable; + } + + Snapshot snapshot() { + if (analyzer == null) return null; + if (analyzer.locals == null || analyzer.stack == null) + throw new IllegalStateException("callback has no verifier state"); + return new Snapshot(new ArrayList<>(analyzer.locals), new ArrayList<>(analyzer.stack)); + } + + void emit(Snapshot snapshot) { + if (analyzer == null) return; + List locals = new ArrayList<>(snapshot.locals); + for (Map.Entry local : hookLocals.entrySet()) { + while (locals.size() <= local.getKey()) locals.add(Opcodes.TOP); + locals.set(local.getKey(), local.getValue()); + } + Object[] localTypes = compact(locals), stackTypes = compact(snapshot.stack); + mv.visitFrame(Opcodes.F_NEW, localTypes.length, localTypes, stackTypes.length, stackTypes); + } + + void handler(int tokenSlot) { + if (analyzer == null) return; + List locals = new ArrayList<>(Collections.nCopies(tokenSlot + 1, Opcodes.TOP)); + locals.set(tokenSlot, "[Ljava/lang/Object;"); + Object[] localTypes = compact(locals); + mv.visitFrame(Opcodes.F_NEW, localTypes.length, localTypes, 1, new Object[] {"java/lang/Throwable"}); + } + + private static Object[] compact(List slots) { + List entries = new ArrayList<>(); + for (int index = 0; index < slots.size(); index++) { + Object value = slots.get(index); + entries.add(value); + if (Objects.equals(value, Opcodes.LONG) || Objects.equals(value, Opcodes.DOUBLE)) { + if (++index >= slots.size() || !Objects.equals(slots.get(index), Opcodes.TOP)) + throw new IllegalStateException("invalid category-two verifier slots"); + } + } + while (!entries.isEmpty() && Objects.equals(entries.get(entries.size() - 1), Opcodes.TOP)) entries.remove(entries.size() - 1); + return entries.toArray(); + } + + static final class Snapshot { + final List locals, stack; + Snapshot(List locals, List stack) { this.locals = locals; this.stack = stack; } + Snapshot handler() { return new Snapshot(locals, Collections.singletonList("java/lang/Throwable")); } + } + + static final class Buffered extends MethodNode { + private final MethodVisitor target; + final Set