From e834add891f1fc27509425f879bd932f81fbf4de Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 14 Nov 2026 05:13:04 +0800 Subject: [PATCH 01/15] =?UTF-8?q?=E2=9C=A8=20feat:=20emit=20compact=20meas?= =?UTF-8?q?urement=20results=20with=20a=20quality=20verdict?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 100-sample measurement printed ~120 KB of inline event pairs, which crowds out an agent's context while the statistics and quality fields it needs are buried. Measurement results move to schema 3.0: - `evidence` keeps one exemplar pair per latency rank (min, p50, p90, p99, max), using the same nearest-rank indices as the statistics, plus the artifact path; `--evidence full` still inlines every pair. - Live captures always write the bounded event capture, to a private temporary directory unless `--events-out` is set, so summary output never loses evidence. `measure --input` re-correlates it. - `quality` gives a deterministic `good`/`caution` verdict with coded issues for timeouts, heuristic or ambiguous correlation, unmatched events, clock error, and sample count. Unchanged records, including Event JSONL, stay at schema 2.0, so older artifacts remain readable. Live tests that inspect every pair request `--evidence full`. --- README.md | 11 +- benchmarks/cuda-multiprocess/run-container.pl | 7 +- docs/architecture.md | 11 +- docs/cli-contract.md | 15 +- schemas/measurement-result.schema.json | 185 ++++++++- skills/xprobe-measure-latency/SKILL.md | 5 +- .../references/result-quality.md | 7 +- tests/integration/run-linux-live.sh | 5 + tests/integration/run-nvtx-live.sh | 2 + tests/integration/test_linux.py | 8 +- tests/integration/test_nvtx.py | 6 +- tests/integration/test_pytorch_cuda.py | 4 +- tests/integration/test_pytorch_symbols.py | 6 +- xprobe/cli/src/args.rs | 28 +- xprobe/cli/src/dev.rs | 3 +- xprobe/cli/src/main.rs | 192 +++++---- xprobe/cli/src/render.rs | 11 +- xprobe/cli/tests/measure.rs | 75 +++- xprobe/correlator/src/lib.rs | 69 ++-- xprobe/correlator/src/quality.rs | 368 ++++++++++++++++++ xprobe/protocol/src/lib.rs | 10 +- xprobe/protocol/src/measurement.rs | 89 ++++- xprobe/protocol/src/version.rs | 17 + xprobe/protocol/tests/contracts.rs | 55 ++- 24 files changed, 1047 insertions(+), 142 deletions(-) create mode 100644 xprobe/correlator/src/quality.rs diff --git a/README.md b/README.md index fe7f8eb..8234982 100644 --- a/README.md +++ b/README.md @@ -72,11 +72,12 @@ GPU operation durations, transfers, NVTX ranges, and CPU/GPU paths after selecting the correct process. `measure` also accepts completed `--input` captures and versioned live -`--spec` files. Evidence can be exported as `jsonl` or `chrome`. JSON results -contain every matched start/end event, latency statistics, unmatched and -ambiguous counts, collection completeness, CUPTI buffer usage, clock quality, -correlation confidence, and warnings. With `--events-out`, the bounded capture -is also preserved when correlation or clock validation fails. +`--spec` files. JSON results contain latency statistics, a quality verdict, +rank exemplar pairs, unmatched and ambiguous counts, collection completeness, +CUPTI buffer usage, clock quality, correlation confidence, and warnings. Live +captures always write the bounded event capture (`jsonl` or `chrome`, private +temporary file unless `--events-out` is set), including when correlation or +clock validation fails; `--evidence full` inlines every matched pair. ## Public CLI diff --git a/benchmarks/cuda-multiprocess/run-container.pl b/benchmarks/cuda-multiprocess/run-container.pl index e7bb828..94d363c 100755 --- a/benchmarks/cuda-multiprocess/run-container.pl +++ b/benchmarks/cuda-multiprocess/run-container.pl @@ -200,13 +200,14 @@ sub warning_present { } sub load_json { - my ($path, $description) = @_; + my ($path, $description, $schema_version) = @_; + $schema_version //= '2.0'; my $value = eval { $JSON->decode(read_text($path)) }; fail("malformed $description JSON at $path: $@") if $@; require_condition(ref($value) eq 'HASH', "malformed $description JSON at $path: expected object"); require_condition(defined($value->{schema_version}) - && $value->{schema_version} eq '2.0', + && $value->{schema_version} eq $schema_version, "unexpected schema in $description"); return $value; } @@ -802,7 +803,7 @@ sub run_batch_body { "worker $worker->{ordinal} exact measurement command failed"); verify_identity($worker, 'after exact measurement'); my $exact = load_json($worker->{files}{measure_stdout}, - 'exact measurement'); + 'exact measurement', '3.0'); my $quality = validate_exact_result($exact, $worker); my $injection_warning = warning_present($exact, 'CUPTI_AGENT_INJECTED'); diff --git a/docs/architecture.md b/docs/architecture.md index 93b5dfc..00c694f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -199,9 +199,14 @@ All sources normalize into the versioned `Event` type. `measure` supports: | `stack-nested` | Per-thread LIFO host entry/return | high | | `stream-order` | Ordered activity within device/context/stream | high | -Every result includes full matched start/end events and `latency_ns` in -`evidence`. Statistics are derived from those pairs. Drops, unmatched events, -ambiguity, policy, confidence, and clock quality remain visible. A normalized +Statistics are derived from every matched start/end pair. Measurement results +use schema `3.0`: by default `evidence` inlines one exemplar pair per latency +rank (min, p50, p90, p99, max) and references the bounded event artifact that +live captures always write; `--evidence full` also inlines every pair. The +artifact re-correlates deterministically with `measure --input`. A +deterministic `quality` verdict summarizes timeout, correlation confidence, +ambiguity, unmatched events, clock error, and sample count. Drops, unmatched +events, ambiguity, policy, confidence, and clock quality remain visible. A normalized CUPTI event without a reported interpolation bound makes `estimated_error_ns: null` and emits `CLOCK_ERROR_UNAVAILABLE`. diff --git a/docs/cli-contract.md b/docs/cli-contract.md index 351a758..7695ac9 100644 --- a/docs/cli-contract.md +++ b/docs/cli-contract.md @@ -301,19 +301,26 @@ Completed inputs may be CUPTI ABI binary, bounded host-capture JSON, or Event JSONL. Repeated inputs are merged after target-PID checks. Unknown records, mixed targets, malformed captures, and excessive event counts fail explicitly. -Results conform to `schemas/measurement-result.schema.json` and contain: +Results conform to `schemas/measurement-result.schema.json` (schema `3.0`) and +contain: - status and latency statistics; +- a `quality` verdict (`good` or `caution`) with coded issues; - matched, unmatched, ambiguous, and dropped counts; - method, confidence, and score; - clock alignment and optional estimated error; - collection completeness, host/CUDA totals, and CUPTI capacity, observed, retained, dropped, and utilization values when an ABI envelope provides them; -- full start/end events plus latency for each evidence pair; +- `evidence`: one exemplar start/end pair per latency rank (min, p50, p90, + p99, max), the artifact path, format, and event count, and with + `--evidence full` every matched pair; - structured warnings. -`--events-out PATH` atomically writes the complete bounded capture with mode -`0600`. The default format is JSONL; `--format chrome` writes Chrome Trace Event +Live captures always write the complete bounded capture with mode `0600`: +`--events-out PATH` chooses the file; otherwise it goes to a new mode `0700` +directory under the temporary directory. `measure --input` with the same +selectors re-correlates it deterministically. Completed `--input` measurements +write an artifact only with `--events-out`. The default format is JSONL; `--format chrome` writes Chrome Trace Event Format. Once collection has produced a capture, correlation, clock, drop, or capacity failures still write it and identify its path, format, and event count in the JSON error. Artifact failure returns `TRACE_EXPORT_FAILED` and preserves diff --git a/schemas/measurement-result.schema.json b/schemas/measurement-result.schema.json index c6e3e7c..d3a2e70 100644 --- a/schemas/measurement-result.schema.json +++ b/schemas/measurement-result.schema.json @@ -13,10 +13,7 @@ "$ref": "#/$defs/CorrelationSummary" }, "evidence": { - "type": "array", - "items": { - "$ref": "#/$defs/MatchedEventPair" - } + "$ref": "#/$defs/Evidence" }, "measurement": { "$ref": "#/$defs/Measurement" @@ -24,8 +21,11 @@ "ok": { "type": "boolean" }, + "quality": { + "$ref": "#/$defs/ResultQuality" + }, "schema_version": { - "$ref": "#/$defs/SchemaVersion" + "$ref": "#/$defs/SchemaVersion3" }, "session_id": { "type": "string" @@ -48,6 +48,7 @@ "session_id", "status", "measurement", + "quality", "correlation", "clock", "collection", @@ -531,6 +532,102 @@ "marker" ] }, + "Evidence": { + "description": "Matched pair evidence. `summary` keeps representative exemplars inline and\nreferences the bounded capture artifact; `full` also inlines every pair.", + "type": "object", + "properties": { + "artifact": { + "anyOf": [ + { + "$ref": "#/$defs/EvidenceArtifact" + }, + { + "type": "null" + } + ] + }, + "exemplars": { + "type": "array", + "items": { + "$ref": "#/$defs/EvidenceExemplar" + } + }, + "mode": { + "$ref": "#/$defs/EvidenceMode" + }, + "pairs": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/$defs/MatchedEventPair" + } + } + }, + "additionalProperties": false, + "required": [ + "mode", + "exemplars" + ] + }, + "EvidenceArtifact": { + "description": "Bounded capture written by `measure`; re-correlate it with `measure --input`.", + "type": "object", + "properties": { + "events": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "format": { + "$ref": "#/$defs/ExportFormat" + }, + "path": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "path", + "format", + "events" + ] + }, + "EvidenceExemplar": { + "description": "One matched pair selected for a latency rank. Ranks use the same\nnearest-rank indices as `measurement.latency_ns`, so an exemplar's latency\nequals the reported statistic.", + "type": "object", + "properties": { + "pair": { + "$ref": "#/$defs/MatchedEventPair" + }, + "ranks": { + "type": "array", + "items": { + "$ref": "#/$defs/LatencyRank" + } + } + }, + "additionalProperties": false, + "required": [ + "ranks", + "pair" + ] + }, + "EvidenceMode": { + "type": "string", + "enum": [ + "summary", + "full" + ] + }, + "ExportFormat": { + "type": "string", + "enum": [ + "jsonl", + "chrome" + ] + }, "HostEvent": { "type": "object", "properties": { @@ -602,6 +699,16 @@ "usdt" ] }, + "LatencyRank": { + "type": "string", + "enum": [ + "min", + "p50", + "p90", + "p99", + "max" + ] + }, "LatencyStatistics": { "type": "object", "properties": { @@ -745,6 +852,67 @@ "process" ] }, + "QualityIssue": { + "type": "object", + "properties": { + "code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "severity": { + "$ref": "#/$defs/QualitySeverity" + } + }, + "additionalProperties": false, + "required": [ + "code", + "severity", + "message" + ] + }, + "QualitySeverity": { + "type": "string", + "enum": [ + "info", + "caution" + ] + }, + "QualityVerdict": { + "oneOf": [ + { + "description": "No caution-level issue; info issues may still be listed.", + "type": "string", + "const": "good" + }, + { + "description": "At least one issue limits how the latency may be interpreted.", + "type": "string", + "const": "caution" + } + ] + }, + "ResultQuality": { + "description": "Deterministic interpretation of the result's quality fields.", + "type": "object", + "properties": { + "issues": { + "type": "array", + "items": { + "$ref": "#/$defs/QualityIssue" + } + }, + "verdict": { + "$ref": "#/$defs/QualityVerdict" + } + }, + "additionalProperties": false, + "required": [ + "verdict", + "issues" + ] + }, "SampleSummary": { "type": "object", "properties": { @@ -789,6 +957,13 @@ "2.0" ] }, + "SchemaVersion3": { + "description": "Version of records whose shape changed in xprobe 0.6.\n\nRecords keep their own version so unchanged contracts, such as Event JSONL\nartifacts, remain readable at `2.0`.", + "type": "string", + "enum": [ + "3.0" + ] + }, "SessionStatus": { "type": "string", "enum": [ diff --git a/skills/xprobe-measure-latency/SKILL.md b/skills/xprobe-measure-latency/SKILL.md index 2b829e1..da94212 100644 --- a/skills/xprobe-measure-latency/SKILL.md +++ b/skills/xprobe-measure-latency/SKILL.md @@ -76,8 +76,9 @@ justify unrelated collection. ## Preserve these invariants -- Use JSON mode and require schema version `2.0`. Treat malformed output and - unknown versions as errors. +- Use JSON mode. Measurement results use schema version `3.0`; other records + and Event JSONL use `2.0`. Treat malformed output and unknown versions as + errors. - Identify a live process by PID plus procfs start time and verify it around validation and attachment. Never substitute a reused PID. - Bound every capture by duration or samples, timeout, and record/group/thread diff --git a/skills/xprobe-measure-latency/references/result-quality.md b/skills/xprobe-measure-latency/references/result-quality.md index daa9656..161374c 100644 --- a/skills/xprobe-measure-latency/references/result-quality.md +++ b/skills/xprobe-measure-latency/references/result-quality.md @@ -8,8 +8,11 @@ function. Use `stream-order` only for GPU activity endpoints on the same device, context, and stream. Treat `first-after` and `nearest` as temporal heuristics, never request causality. -Inspect every evidence pair for selector scope, process identity, correlation -IDs, device/context/stream, timestamps, and clock domains. A result is not +Start from `quality.verdict` and its issues, then inspect the rank exemplar +pairs for selector scope, process identity, correlation IDs, +device/context/stream, timestamps, and clock domains. When a claim depends on +individual pairs, use `--evidence full` or re-correlate the evidence artifact +with `measure --input`. A result is not sufficient when records were dropped, no samples matched, target identity changed, collection was incomplete, or clock alignment failed. Report unmatched and ambiguous counts with matched samples. `estimated_error_ns: null` means no diff --git a/tests/integration/run-linux-live.sh b/tests/integration/run-linux-live.sh index 846e54b..be13213 100755 --- a/tests/integration/run-linux-live.sh +++ b/tests/integration/run-linux-live.sh @@ -32,6 +32,7 @@ cleanup_python_gc() { trap 'cleanup_python_gc; cleanup' EXIT "${binary}" measure \ + --evidence full \ --pid "${python_gc_pid}" \ --from python:gc_start \ --to python:gc_end \ @@ -51,6 +52,7 @@ trap 'cleanup_python_gc; cleanup' EXIT --json --non-interactive --no-color >"${syscall_inventory}" "${binary}" measure \ + --evidence full \ --pid "${target_pid}" \ --from syscall:mmap:entry \ --to syscall:mmap:exit \ @@ -61,6 +63,7 @@ trap 'cleanup_python_gc; cleanup' EXIT --json --non-interactive --no-color >"${mmap_capture}" "${binary}" measure \ + --evidence full \ --pid "${target_pid}" \ --from syscall:munmap:entry \ --to syscall:munmap:exit \ @@ -71,6 +74,7 @@ trap 'cleanup_python_gc; cleanup' EXIT --json --non-interactive --no-color >"${munmap_capture}" "${binary}" measure \ + --evidence full \ --pid "${target_pid}" \ --from tracepoint:raw_syscalls:sys_enter \ --to tracepoint:raw_syscalls:sys_exit \ @@ -81,6 +85,7 @@ trap 'cleanup_python_gc; cleanup' EXIT --json --non-interactive --no-color >"${tracepoint_capture}" if "${binary}" measure \ + --evidence full \ --pid "${target_pid}" \ --from syscall:mmap:entry \ --to syscall:mmap:exit \ diff --git a/tests/integration/run-nvtx-live.sh b/tests/integration/run-nvtx-live.sh index 09800bf..82011ca 100755 --- a/tests/integration/run-nvtx-live.sh +++ b/tests/integration/run-nvtx-live.sh @@ -51,6 +51,7 @@ measure() mv "${mode}.next" "${mode}" sleep 0.05 "${xprobe}" measure \ + --evidence full \ --pid "${pid}" \ --agent "${agent}" \ --from "cuda:nvtx_range_start:name~${pattern}" \ @@ -72,6 +73,7 @@ mv "${mode}.next" "${mode}" sleep 0.05 set +e "${xprobe}" measure \ + --evidence full \ --pid "${pid}" \ --agent "${agent}" \ --from 'cuda:nvtx_range_start:name~^xprobe_outer$' \ diff --git a/tests/integration/test_linux.py b/tests/integration/test_linux.py index c598e16..ef4f693 100755 --- a/tests/integration/test_linux.py +++ b/tests/integration/test_linux.py @@ -62,8 +62,8 @@ def assert_syscall_capture(result: dict, name: str) -> None: assert result["collection"]["host_events"] == 8 assert result["collection"]["cuda_events"] == 0 assert result["collection"]["dropped_events"] == 0 - assert len(result["evidence"]) == 3 - for pair in result["evidence"]: + assert len(result["evidence"]["pairs"]) == 3 + for pair in result["evidence"]["pairs"]: start = pair["start"] end = pair["end"] assert start["event_type"] == "syscall_entry" @@ -88,7 +88,7 @@ def assert_tracepoint_capture(result: dict) -> None: assert result["correlation"]["method"] == "first_after" assert result["correlation"]["confidence"] == "heuristic" assert result["collection"]["dropped_events"] == 0 - for pair in result["evidence"]: + for pair in result["evidence"]["pairs"]: assert pair["start"]["host"]["probe_kind"] == "tracepoint" assert pair["start"]["host"]["symbol"] == "sys_enter" assert pair["end"]["host"]["symbol"] == "sys_exit" @@ -122,7 +122,7 @@ def assert_python_gc_capture(result: dict) -> None: assert result["correlation"]["method"] == "exact_python_gc_tid_lifecycle" assert result["correlation"]["confidence"] == "exact" assert result["collection"]["dropped_events"] == 0 - for pair in result["evidence"]: + for pair in result["evidence"]["pairs"]: start = pair["start"] end = pair["end"] assert start["event_type"] == "python_gc_start" diff --git a/tests/integration/test_nvtx.py b/tests/integration/test_nvtx.py index be5224d..962c875 100755 --- a/tests/integration/test_nvtx.py +++ b/tests/integration/test_nvtx.py @@ -67,12 +67,12 @@ def main() -> None: pair["start"]["tid"] != pair["end"]["tid"] and pair["start"]["nvtx"]["start_tid"] == pair["start"]["tid"] and pair["end"]["nvtx"]["start_tid"] == pair["start"]["tid"] - for pair in cross["evidence"] + for pair in cross["evidence"]["pairs"] ) assert all( pair["start"]["nvtx"]["name_complete"] is False and pair["end"]["nvtx"]["name_complete"] is False - for pair in long_name["evidence"] + for pair in long_name["evidence"]["pairs"] ) print( json.dumps( @@ -96,7 +96,7 @@ def assert_measurement(result: dict, range_kind: str) -> None: pair["start"]["nvtx"]["range_kind"] == range_kind and pair["end"]["nvtx"]["range_kind"] == range_kind and pair["start"]["nvtx"]["range_id"] == pair["end"]["nvtx"]["range_id"] - for pair in result["evidence"] + for pair in result["evidence"]["pairs"] ) diff --git a/tests/integration/test_pytorch_cuda.py b/tests/integration/test_pytorch_cuda.py index 652f7b4..ff12125 100644 --- a/tests/integration/test_pytorch_cuda.py +++ b/tests/integration/test_pytorch_cuda.py @@ -154,7 +154,7 @@ def run_inner(xprobe: pathlib.Path) -> None: assert all( pair["start"]["nvtx"]["range_kind"] == "thread" and pair["start"]["nvtx"]["range_id"] == pair["end"]["nvtx"]["range_id"] - for pair in nvtx["evidence"] + for pair in nvtx["evidence"]["pairs"] ) print( @@ -329,6 +329,8 @@ def exact_measure( "exact", "--samples", "8", + "--evidence", + "full", "--max-events", "4096", "--timeout-ms", diff --git a/tests/integration/test_pytorch_symbols.py b/tests/integration/test_pytorch_symbols.py index 2348117..771997b 100755 --- a/tests/integration/test_pytorch_symbols.py +++ b/tests/integration/test_pytorch_symbols.py @@ -67,8 +67,8 @@ def main() -> None: assert torch_probe["object_kind"] == "shared_library" if measurement is not None: assert measurement["measurement"]["samples"]["matched"] == 8 - assert len(measurement["evidence"]) == 8 - for evidence in measurement["evidence"]: + assert len(measurement["evidence"]["pairs"]) == 8 + for evidence in measurement["evidence"]["pairs"]: assert evidence["start"]["host"]["symbol_demangled"] == MM_SYMBOL assert evidence["end"]["host"]["symbol_demangled"] == MM_SYMBOL print( @@ -160,6 +160,8 @@ def measure( [ binary, "measure", + "--evidence", + "full", "--pid", str(pid), "--from", diff --git a/xprobe/cli/src/args.rs b/xprobe/cli/src/args.rs index e71ce84..fd4efdd 100644 --- a/xprobe/cli/src/args.rs +++ b/xprobe/cli/src/args.rs @@ -76,6 +76,21 @@ pub(crate) enum ExportFormatArg { Chrome, } +#[derive(Debug, Clone, Copy, ValueEnum)] +pub(crate) enum EvidenceArg { + Summary, + Full, +} + +impl EvidenceArg { + pub(crate) const fn mode(self) -> xprobe_protocol::EvidenceMode { + match self { + Self::Summary => xprobe_protocol::EvidenceMode::Summary, + Self::Full => xprobe_protocol::EvidenceMode::Full, + } + } +} + #[derive(Debug, Args)] pub(crate) struct ExportArgs { /// Completed CUPTI binary, host capture JSON, or Event JSONL; repeat to merge. @@ -243,10 +258,21 @@ pub(crate) struct MeasureArgs { #[arg(long)] pub(crate) agent: Option, - /// Write the complete bounded event capture to this file. + /// Write the complete bounded event capture to this file. Live captures + /// default to a private file under the temporary directory. #[arg(long)] pub(crate) events_out: Option, + /// Inline matched pairs: `summary` keeps rank exemplars and the artifact + /// path; `full` also inlines every pair. + #[arg( + long, + value_enum, + default_value_t = EvidenceArg::Summary, + conflicts_with_all = ["cpu_sample", "syscall_aggregate", "aggregate"] + )] + pub(crate) evidence: EvidenceArg, + /// Evidence event format; defaults to jsonl when --events-out is set. #[arg(long, value_enum)] pub(crate) format: Option, diff --git a/xprobe/cli/src/dev.rs b/xprobe/cli/src/dev.rs index 505ed6b..d55ef53 100644 --- a/xprobe/cli/src/dev.rs +++ b/xprobe/cli/src/dev.rs @@ -40,6 +40,7 @@ use xprobe_correlator::MeasureOptions; use xprobe_exporter::events_to_chrome_trace; use xprobe_exporter::events_to_jsonl; use xprobe_protocol::ErrorCode; +use xprobe_protocol::EvidenceMode; use xprobe_protocol::ExportFormat; use xprobe_protocol::MeasurementMode; use xprobe_protocol::MeasurementSpec; @@ -143,7 +144,7 @@ pub(crate) fn run_trace(args: TraceArgs) -> ExitCode { }, }; match collect_live_measurement(&request) { - Ok(execution) => emit_measurement(&execution.result, json), + Ok(execution) => emit_measurement(execution.result, EvidenceMode::Full, json), Err(error) => emit_command_failure(error, json), } } diff --git a/xprobe/cli/src/main.rs b/xprobe/cli/src/main.rs index 5573467..2820787 100644 --- a/xprobe/cli/src/main.rs +++ b/xprobe/cli/src/main.rs @@ -1,9 +1,10 @@ use std::{ - fs::{self}, + fs::{self, DirBuilder}, + os::unix::fs::DirBuilderExt, path::{Path, PathBuf}, process::ExitCode, sync::atomic::AtomicU64, - time::Duration, + time::{Duration, SystemTime, UNIX_EPOCH}, }; static EXPORT_SEQUENCE: AtomicU64 = AtomicU64::new(0); @@ -11,12 +12,12 @@ static EXPORT_SEQUENCE: AtomicU64 = AtomicU64::new(0); use clap::Parser; use xprobe_collector::completed; use xprobe_core::{cpu_sampling, discover, doctor, inspect, syscall_aggregate, validate}; -use xprobe_correlator::{MeasureOptions, measure}; +use xprobe_correlator::{MeasureOptions, finalize, measure}; use xprobe_exporter::{events_to_chrome_trace, events_to_jsonl}; use xprobe_protocol::{ AggregateInventoryResult, ClockDomain, CpuSamplingValidationResult, ErrorCode, Event, - MatchPolicy, MeasurementMode, MeasurementResult, MeasurementSpec, - SyscallAggregateValidationResult, + EvidenceArtifact, EvidenceMode, ExportFormat, MatchPolicy, MeasurementMode, MeasurementResult, + MeasurementSpec, SyscallAggregateValidationResult, }; mod args; @@ -69,6 +70,7 @@ fn run_measure(args: MeasureArgs) -> ExitCode { cupti_socket, agent, events_out, + evidence, format, timeout_ms, from, @@ -94,15 +96,13 @@ fn run_measure(args: MeasureArgs) -> ExitCode { non_interactive: _, }, } = args; + let output = EvidenceOutput { + events_out: events_out.as_deref(), + format, + mode: evidence.mode(), + }; if let Some(spec) = spec { - return run_measure_spec( - &spec, - cupti_socket, - agent, - events_out.as_deref(), - format, - json, - ); + return run_measure_spec(&spec, cupti_socket, agent, output, json); } if cpu_sample { let Some(pid) = pid else { @@ -223,8 +223,8 @@ fn run_measure(args: MeasureArgs) -> ExitCode { options, }; return match collect_live_measurement(&request) { - Ok(execution) => finish_measurement(&execution, events_out.as_deref(), format, json), - Err(error) => finish_measurement_failure(error, events_out.as_deref(), format, json), + Ok(execution) => finish_measurement(&execution, output, true, json), + Err(error) => finish_measurement_failure(error, output, true, json), }; } run_completed_measurement( @@ -232,8 +232,7 @@ fn run_measure(args: MeasureArgs) -> ExitCode { &options, cupti_socket.is_some(), agent.is_some(), - events_out.as_deref(), - format, + output, json, ) } @@ -336,8 +335,7 @@ fn run_completed_measurement( options: &MeasureOptions, has_cupti_socket: bool, has_agent: bool, - events_out: Option<&Path>, - format: Option, + output: EvidenceOutput<'_>, json: bool, ) -> ExitCode { let invalid = if input.is_empty() { @@ -359,8 +357,8 @@ fn run_completed_measurement( return emit_error(code, message.to_owned(), true, json); } match measure_completed_inputs(input, options) { - Ok(execution) => finish_measurement(&execution, events_out, format, json), - Err(error) => finish_measurement_failure(error, events_out, format, json), + Ok(execution) => finish_measurement(&execution, output, false, json), + Err(error) => finish_measurement_failure(error, output, false, json), } } @@ -369,8 +367,7 @@ fn run_measure_spec( path: &Path, cupti_socket: Option, agent_path: Option, - events_out: Option<&Path>, - format: Option, + output: EvidenceOutput<'_>, json: bool, ) -> ExitCode { let bytes = match fs::read(path) { @@ -400,7 +397,7 @@ fn run_measure_spec( value, path, cupti_socket.is_some() || agent_path.is_some(), - events_out.is_some() || format.is_some(), + output.requests_artifact(), json, ); } @@ -409,7 +406,7 @@ fn run_measure_spec( value, path, cupti_socket.is_some() || agent_path.is_some(), - events_out.is_some() || format.is_some(), + output.requests_artifact(), json, ); } @@ -461,11 +458,11 @@ fn run_measure_spec( }; match request.mode { MeasurementMode::Exact => match collect_live_measurement(&request) { - Ok(execution) => finish_measurement(&execution, events_out, format, json), - Err(error) => finish_measurement_failure(error, events_out, format, json), + Ok(execution) => finish_measurement(&execution, output, true, json), + Err(error) => finish_measurement_failure(error, output, true, json), }, MeasurementMode::Aggregate => { - if events_out.is_some() || format.is_some() { + if output.requests_artifact() { return emit_error( ErrorCode::TraceExportFailed, "aggregate measurement does not produce event evidence artifacts".to_owned(), @@ -514,61 +511,123 @@ fn measurement_spec_capacity(spec: &MeasurementSpec) -> Result { + events_out: Option<&'a Path>, + format: Option, + mode: EvidenceMode, +} + +impl EvidenceOutput<'_> { + const fn requests_artifact(&self) -> bool { + self.events_out.is_some() || self.format.is_some() + } + + /// Resolve where the bounded capture goes. Live captures default to a + /// private temporary file so summary output never loses evidence. + fn artifact_path(&self, live: bool) -> Result, CommandFailure> { + if self.format.is_some() && self.events_out.is_none() { + return Err(CommandFailure::new( + ErrorCode::TraceExportFailed, + "--format requires --events-out", + true, + )); + } + match self.events_out { + Some(path) => Ok(Some(path.to_path_buf())), + None if live => default_artifact_path().map(Some), + None => Ok(None), + } + } +} + +fn default_artifact_path() -> Result { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_nanos()); + let directory = std::env::temp_dir().join(format!("xprobe-{}-{nanos}", std::process::id())); + DirBuilder::new() + .mode(0o700) + .create(&directory) + .map_err(|error| { + CommandFailure::new( + ErrorCode::TraceExportFailed, + format!( + "failed to create evidence directory {}: {error}", + directory.display() + ), + true, + ) + .with_hint("pass --events-out PATH to choose a writable artifact location") + })?; + Ok(directory.join("events.jsonl")) +} + fn finish_measurement( execution: &MeasurementExecution, - events_out: Option<&Path>, - format: Option, + output: EvidenceOutput<'_>, + live: bool, json: bool, ) -> ExitCode { - if format.is_some() && events_out.is_none() { - return emit_error( - ErrorCode::TraceExportFailed, - "--format requires --events-out".to_owned(), - true, - json, - ); - } - if let Some(path) = events_out { - let format = format.unwrap_or(ExportFormatArg::Jsonl); - if let Err(error) = export_measurement_capture(&execution.events, path, format) { + let mut result = execution.result.clone(); + let path = match output.artifact_path(live) { + Ok(path) => path, + Err(error) => return emit_command_failure(error, json), + }; + if let Some(path) = path { + let format = output.format.unwrap_or(ExportFormatArg::Jsonl); + if let Err(error) = export_measurement_capture(&execution.events, &path, format) { return emit_command_failure(error, json); } + result.evidence.artifact = Some(EvidenceArtifact { + path: path.display().to_string(), + format: protocol_export_format(format), + events: execution.events.len() as u64, + }); } - emit_measurement(&execution.result, json) + emit_measurement(result, output.mode, json) } fn finish_measurement_failure( mut error: CommandFailure, - events_out: Option<&Path>, - format: Option, + output: EvidenceOutput<'_>, + live: bool, json: bool, ) -> ExitCode { - if format.is_some() && events_out.is_none() { - return emit_error( - ErrorCode::TraceExportFailed, - "--format requires --events-out".to_owned(), - true, + let Some(events) = error.artifact_events.take() else { + if let Err(invalid) = output.artifact_path(false) { + return emit_command_failure(invalid, json); + } + return emit_command_failure(error, json); + }; + let path = match output.artifact_path(live) { + Ok(Some(path)) => path, + Ok(None) => return emit_command_failure(error, json), + Err(invalid) => return emit_command_failure(invalid, json), + }; + let format = output.format.unwrap_or(ExportFormatArg::Jsonl); + if let Err(export) = export_measurement_capture(&events, &path, format) { + return emit_command_failure( + export + .with_detail("original_error_code", error.code.to_string()) + .with_hint("the measurement also failed; rerun after fixing artifact output"), json, ); } - if let (Some(path), Some(events)) = (events_out, error.artifact_events.take()) { - let format = format.unwrap_or(ExportFormatArg::Jsonl); - if let Err(export) = export_measurement_capture(&events, path, format) { - return emit_command_failure( - export - .with_detail("original_error_code", error.code.to_string()) - .with_hint("the measurement also failed; rerun after fixing artifact output"), - json, - ); - } - error = error - .with_detail("artifact_path", path.display().to_string()) - .with_detail("artifact_format", export_format_name(format)) - .with_detail("artifact_event_count", events.len() as u64); - } + error = error + .with_detail("artifact_path", path.display().to_string()) + .with_detail("artifact_format", export_format_name(format)) + .with_detail("artifact_event_count", events.len() as u64); emit_command_failure(error, json) } +const fn protocol_export_format(format: ExportFormatArg) -> ExportFormat { + match format { + ExportFormatArg::Jsonl => ExportFormat::Jsonl, + ExportFormatArg::Chrome => ExportFormat::Chrome, + } +} + fn export_measurement_capture( events: &[Event], path: &Path, @@ -662,14 +721,15 @@ const fn clock_domain_name(domain: &ClockDomain) -> &'static str { } } -fn emit_measurement(result: &MeasurementResult, json: bool) -> ExitCode { +fn emit_measurement(mut result: MeasurementResult, mode: EvidenceMode, json: bool) -> ExitCode { + finalize(&mut result, mode); if json { println!( "{}", serde_json::to_string_pretty(&result).expect("measurement result must serialize") ); } else { - print_measurement_result(result); + print_measurement_result(&result); } ExitCode::SUCCESS } diff --git a/xprobe/cli/src/render.rs b/xprobe/cli/src/render.rs index 217af2c..5c54d58 100644 --- a/xprobe/cli/src/render.rs +++ b/xprobe/cli/src/render.rs @@ -195,7 +195,16 @@ pub(crate) fn print_measurement_result(result: &MeasurementResult) { result.measurement.name.as_deref().unwrap_or("unnamed") ); println!("Matched samples: {}", result.measurement.samples.matched); - println!("Evidence pairs: {}", result.evidence.len()); + println!("Quality: {:?}", result.quality.verdict); + for issue in &result.quality.issues { + println!(" {:?} {}: {}", issue.severity, issue.code, issue.message); + } + if let Some(artifact) = &result.evidence.artifact { + println!( + "Evidence artifact: {} ({} events)", + artifact.path, artifact.events + ); + } println!("Correlation: {}", result.correlation.method); println!("Clock: {}", result.clock.alignment); println!("Latency (ns):"); diff --git a/xprobe/cli/tests/measure.rs b/xprobe/cli/tests/measure.rs index ba23b10..57b31b3 100644 --- a/xprobe/cli/tests/measure.rs +++ b/xprobe/cli/tests/measure.rs @@ -4,8 +4,8 @@ use std::{ use xprobe_protocol::{ CaptureCompleteness, ClockDomain, CorrelationConfidence, ErrorCode, ErrorResponse, Event, - EventSource, EventType, HostCaptureResult, HostEvent, HostProbeKind, MeasurementResult, - SchemaVersion, SessionStatus, TargetIdentity, + EventSource, EventType, EvidenceMode, HostCaptureResult, HostEvent, HostProbeKind, LatencyRank, + MeasurementResult, QualityVerdict, SchemaVersion, SessionStatus, TargetIdentity, }; const HEADER_SIZE: usize = 88; @@ -190,8 +190,25 @@ fn measures_exact_kernel_durations_from_a_completed_capture() { assert_eq!(cupti.retained_records, 4); assert_eq!(cupti.dropped_records, 0); assert!((cupti.buffer_utilization - 1.0).abs() < f64::EPSILON); - assert_eq!(result.evidence.len(), 2); - assert_eq!(result.evidence[0].latency_ns, 50); + assert_eq!(result.quality.verdict, QualityVerdict::Good); + assert_eq!(result.evidence.mode, EvidenceMode::Summary); + assert_eq!(result.evidence.pairs, None); + let exemplars = &result.evidence.exemplars; + assert_eq!(exemplars.len(), 2); + assert_eq!(exemplars[0].ranks, [LatencyRank::Min, LatencyRank::P50]); + assert_eq!(exemplars[0].pair.latency_ns, 50); + assert_eq!( + exemplars[1].ranks, + [LatencyRank::P90, LatencyRank::P99, LatencyRank::Max] + ); + assert_eq!(exemplars[1].pair.latency_ns, 80); + let artifact = result + .evidence + .artifact + .as_ref() + .expect("explicit events-out must be referenced"); + assert_eq!(artifact.path, evidence_path.to_string_lossy()); + assert_eq!(artifact.events, 4); let evidence = fs::read_to_string(&evidence_path).expect("evidence must be written"); let mode = fs::metadata(&evidence_path).unwrap().permissions().mode() & 0o777; fs::remove_file(evidence_path).expect("evidence fixture must be removed"); @@ -199,6 +216,56 @@ fn measures_exact_kernel_durations_from_a_completed_capture() { assert_eq!(mode, 0o600); } +#[test] +fn inlines_every_pair_with_full_evidence() { + let path = capture_path("full-evidence"); + write_capture( + &path, + &[ + record(3, 100, 11, "test_kernel"), + record(4, 150, 11, "test_kernel"), + record(3, 200, 12, "test_kernel"), + record(4, 280, 12, "test_kernel"), + ], + ); + let output = Command::new(env!("CARGO_BIN_EXE_xprobe")) + .args([ + "measure", + "--input", + path.to_str().expect("temporary path must be UTF-8"), + "--from", + "cuda:kernel_start:name~test.*", + "--to", + "cuda:kernel_end:name~test.*", + "--match", + "exact", + "--samples", + "2", + "--evidence", + "full", + "--json", + "--non-interactive", + "--no-color", + ]) + .output() + .expect("xprobe measure must run"); + fs::remove_file(path).expect("capture fixture must be removed"); + + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let result: MeasurementResult = + serde_json::from_slice(&output.stdout).expect("stdout must contain measurement JSON"); + assert_eq!(result.evidence.mode, EvidenceMode::Full); + assert_eq!(result.evidence.artifact, None); + let pairs = result.evidence.pairs.expect("full evidence inlines pairs"); + assert_eq!(pairs.len(), 2); + assert_eq!(pairs[0].latency_ns, 50); + assert_eq!(pairs[1].latency_ns, 80); +} + #[test] fn rejects_a_capture_that_reached_the_agent_record_limit() { let path = capture_path("record-limit"); diff --git a/xprobe/correlator/src/lib.rs b/xprobe/correlator/src/lib.rs index 418038a..51e5b01 100644 --- a/xprobe/correlator/src/lib.rs +++ b/xprobe/correlator/src/lib.rs @@ -7,9 +7,13 @@ use xprobe_protocol::{ CaptureCompleteness, ClockDomain, ClockQuality, CollectionSummary, CorrelationConfidence, CorrelationSummary, ErrorCode, Event, EventSource, EventType, HostProbeKind, LatencyStatistics, MatchPolicy, MatchedEventPair, Measurement, MeasurementResult, MemcpyKind, NvtxRangeKind, - SampleSummary, SchemaVersion, SessionStatus, Warning, + QualityVerdict, ResultQuality, SampleSummary, SchemaVersion3, SessionStatus, Warning, }; +mod quality; + +pub use quality::{assess as assess_quality, finalize}; + #[derive(Debug, Clone)] pub struct MeasureOptions { pub session_id: String, @@ -781,8 +785,10 @@ pub fn measure( let warnings = measurement_warnings(options, &starts, &ends); let (host_events, cuda_events) = collection_event_counts(events); - Ok(MeasurementResult { - schema_version: SchemaVersion::current(), + let estimated_error_ns = maximum_timestamp_error(&outcome.pairs); + let pairs = matched_pairs(outcome.pairs); + let mut result = MeasurementResult { + schema_version: SchemaVersion3::current(), ok: true, session_id: options.session_id.clone(), status: SessionStatus::Completed, @@ -797,19 +803,18 @@ pub fn measure( }, latency_ns: statistics(&latencies), }, + quality: ResultQuality { + verdict: QualityVerdict::Good, + issues: Vec::new(), + }, correlation: CorrelationSummary { method: method.to_owned(), confidence, score: ratio(matched, denominator), }, clock: ClockQuality { - alignment: match clock_domain { - ClockDomain::HostMonotonic => "host_monotonic", - ClockDomain::Cupti => "cupti_same_domain", - ClockDomain::CuptiNormalizedToHostMonotonic => "cupti_normalized_to_host_monotonic", - } - .to_owned(), - estimated_error_ns: maximum_timestamp_error(&outcome.pairs), + alignment: clock_alignment(&clock_domain).to_owned(), + estimated_error_ns, }, collection: CollectionSummary { completeness: CaptureCompleteness::Complete, @@ -818,17 +823,30 @@ pub fn measure( dropped_events: options.dropped_events, cupti: None, }, - evidence: outcome - .pairs - .into_iter() - .map(|pair| MatchedEventPair { - start: pair.start.clone(), - end: pair.end.clone(), - latency_ns: pair.latency_ns, - }) - .collect(), + evidence: quality::full_evidence(pairs), warnings, - }) + }; + result.quality = assess_quality(&result); + Ok(result) +} + +fn matched_pairs(pairs: Vec>) -> Vec { + pairs + .into_iter() + .map(|pair| MatchedEventPair { + start: pair.start.clone(), + end: pair.end.clone(), + latency_ns: pair.latency_ns, + }) + .collect() +} + +const fn clock_alignment(domain: &ClockDomain) -> &'static str { + match domain { + ClockDomain::HostMonotonic => "host_monotonic", + ClockDomain::Cupti => "cupti_same_domain", + ClockDomain::CuptiNormalizedToHostMonotonic => "cupti_normalized_to_host_monotonic", + } } fn collection_event_counts(events: &[Event]) -> (u64, u64) { @@ -1464,8 +1482,7 @@ fn mean(sum: u128, count: usize) -> f64 { } fn percentile(sorted: &[u64], percentile: usize) -> u64 { - let rank = (percentile * sorted.len()).div_ceil(100); - sorted[rank.saturating_sub(1)] + sorted[quality::rank_index(sorted.len(), percentile)] } fn maximum_timestamp_error(pairs: &[Pair<'_>]) -> Option { @@ -1703,8 +1720,8 @@ mod tests { assert_eq!(result.measurement.latency_ns.max, 80); assert_eq!(result.correlation.method, "exact_syscall_tid_lifecycle"); assert_eq!(result.correlation.confidence, CorrelationConfidence::Exact); - assert_eq!(result.evidence[0].start.tid, 10); - assert_eq!(result.evidence[0].end.tid, 10); + assert_eq!(result.evidence.pairs.as_ref().unwrap()[0].start.tid, 10); + assert_eq!(result.evidence.pairs.as_ref().unwrap()[0].end.tid, 10); } #[test] @@ -1797,8 +1814,8 @@ mod tests { assert_eq!(result.measurement.latency_ns.min, 50); assert_eq!(result.measurement.latency_ns.max, 80); assert_eq!(result.correlation.method, "exact_nvtx_range_id"); - assert_eq!(result.evidence[0].end.tid, 10); - assert_eq!(result.evidence[1].end.tid, 30); + assert_eq!(result.evidence.pairs.as_ref().unwrap()[0].end.tid, 10); + assert_eq!(result.evidence.pairs.as_ref().unwrap()[1].end.tid, 30); } #[test] diff --git a/xprobe/correlator/src/quality.rs b/xprobe/correlator/src/quality.rs new file mode 100644 index 0000000..c53c94d --- /dev/null +++ b/xprobe/correlator/src/quality.rs @@ -0,0 +1,368 @@ +//! Evidence summarization and deterministic result-quality assessment. + +use xprobe_protocol::{ + CorrelationConfidence, Evidence, EvidenceExemplar, EvidenceMode, LatencyRank, MatchedEventPair, + MeasurementResult, QualityIssue, QualitySeverity, QualityVerdict, ResultQuality, SessionStatus, +}; + +/// Clock error at or above this share of the p50 latency is a caution. +const CLOCK_ERROR_CAUTION_DIVISOR: u64 = 10; +/// Fewer matched samples than this make tail ranks coarse. +const TAIL_SAMPLE_FLOOR: u64 = 100; + +/// Nearest-rank index shared by latency statistics and exemplars. +pub(crate) fn rank_index(len: usize, percentile: usize) -> usize { + (percentile * len).div_ceil(100).saturating_sub(1) +} + +/// Evidence as produced by correlation: every pair inline plus exemplars. +pub(crate) fn full_evidence(pairs: Vec) -> Evidence { + Evidence { + mode: EvidenceMode::Full, + exemplars: exemplars(&pairs), + pairs: Some(pairs), + artifact: None, + } +} + +/// Select one pair per latency rank, merging ranks that land on the same pair. +pub(crate) fn exemplars(pairs: &[MatchedEventPair]) -> Vec { + if pairs.is_empty() { + return Vec::new(); + } + let mut order = (0..pairs.len()).collect::>(); + order.sort_by_key(|&index| (pairs[index].latency_ns, index)); + let ranks = [ + (LatencyRank::Min, 0), + (LatencyRank::P50, rank_index(order.len(), 50)), + (LatencyRank::P90, rank_index(order.len(), 90)), + (LatencyRank::P99, rank_index(order.len(), 99)), + (LatencyRank::Max, order.len() - 1), + ]; + let mut selected: Vec<(usize, Vec)> = Vec::new(); + for (rank, position) in ranks { + match selected.last_mut() { + Some((last, labels)) if *last == position => labels.push(rank), + _ => selected.push((position, vec![rank])), + } + } + selected + .into_iter() + .map(|(position, ranks)| EvidenceExemplar { + ranks, + pair: pairs[order[position]].clone(), + }) + .collect() +} + +/// Recompute quality and apply the requested evidence mode. +/// +/// Call this after every caller-side change to status, collection, or +/// warnings so the verdict describes the emitted result. +pub fn finalize(result: &mut MeasurementResult, mode: EvidenceMode) { + result.quality = assess(result); + result.evidence.mode = mode; + if mode == EvidenceMode::Summary { + result.evidence.pairs = None; + } +} + +/// Interpret status, correlation, sample, and clock fields deterministically. +#[must_use] +pub fn assess(result: &MeasurementResult) -> ResultQuality { + let mut issues = Vec::new(); + let samples = &result.measurement.samples; + match result.status { + SessionStatus::Completed => {} + SessionStatus::TimedOut => issues.push(caution( + "TIMED_OUT", + "capture reached its timeout before the sample or duration bound; statistics are partial" + .to_owned(), + )), + SessionStatus::Cancelled | SessionStatus::Failed => issues.push(caution( + "INCOMPLETE_SESSION", + "session did not complete; statistics are partial".to_owned(), + )), + } + match result.correlation.confidence { + CorrelationConfidence::Exact | CorrelationConfidence::High => {} + CorrelationConfidence::Heuristic => issues.push(caution( + "HEURISTIC_CORRELATION", + "pairs are matched by time order; they do not prove request-level causality".to_owned(), + )), + CorrelationConfidence::Ambiguous => issues.push(caution( + "AMBIGUOUS_CORRELATION", + "the correlation method cannot uniquely pair events".to_owned(), + )), + } + if samples.ambiguous != 0 { + issues.push(caution( + "AMBIGUOUS_PAIRS", + format!("{} candidate pairs were ambiguous", samples.ambiguous), + )); + } + let unmatched = samples.unmatched_start + samples.unmatched_end; + if unmatched != 0 { + let message = format!( + "{} start and {} end events were unmatched alongside {} matched pairs", + samples.unmatched_start, samples.unmatched_end, samples.matched + ); + issues.push(if unmatched > samples.matched { + caution("UNMATCHED_EVENTS", message) + } else { + info("UNMATCHED_EVENTS", message) + }); + } + match result.clock.estimated_error_ns { + None => issues.push(info( + "CLOCK_ERROR_UNAVAILABLE", + "no error bound is reported for GPU-to-host timestamp interpolation".to_owned(), + )), + Some(error) + if result.measurement.latency_ns.p50 != 0 + && error.saturating_mul(CLOCK_ERROR_CAUTION_DIVISOR) + >= result.measurement.latency_ns.p50 => + { + issues.push(caution( + "CLOCK_ERROR_LARGE", + format!( + "timestamp error bound {error} ns is at least 10% of the p50 latency {} ns", + result.measurement.latency_ns.p50 + ), + )); + } + Some(_) => {} + } + if samples.matched < TAIL_SAMPLE_FLOOR { + issues.push(info( + "FEW_SAMPLES", + format!( + "tail ranks such as p99 are estimated from {} matched samples", + samples.matched + ), + )); + } + let verdict = if issues + .iter() + .any(|issue| issue.severity == QualitySeverity::Caution) + { + QualityVerdict::Caution + } else { + QualityVerdict::Good + }; + ResultQuality { verdict, issues } +} + +fn caution(code: &str, message: String) -> QualityIssue { + QualityIssue { + code: code.to_owned(), + severity: QualitySeverity::Caution, + message, + } +} + +fn info(code: &str, message: String) -> QualityIssue { + QualityIssue { + code: code.to_owned(), + severity: QualitySeverity::Info, + message, + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use xprobe_protocol::{ + CaptureCompleteness, ClockDomain, ClockQuality, CollectionSummary, CorrelationConfidence, + CorrelationSummary, Event, EventSource, EventType, EvidenceMode, LatencyRank, + LatencyStatistics, MatchedEventPair, Measurement, MeasurementResult, QualitySeverity, + QualityVerdict, ResultQuality, SampleSummary, SchemaVersion, SchemaVersion3, SessionStatus, + }; + + use super::{assess, exemplars, finalize, full_evidence, rank_index}; + + fn event(timestamp_ns: u64) -> Event { + Event { + schema_version: SchemaVersion::current(), + session_id: "s".to_owned(), + event_id: format!("e{timestamp_ns}"), + sequence: timestamp_ns, + source: EventSource::Ebpf, + event_type: EventType::SyscallEntry, + pid: 1, + tid: 1, + cpu: Some(0), + timestamp_raw: timestamp_ns, + timestamp_ns, + clock_domain: ClockDomain::HostMonotonic, + timestamp_error_ns: None, + process_start_time: Some(1), + host: None, + cuda: None, + nvtx: None, + attributes: BTreeMap::new(), + } + } + + fn pair(latency_ns: u64) -> MatchedEventPair { + MatchedEventPair { + start: event(0), + end: event(latency_ns), + latency_ns, + } + } + + fn result(matched: u64, p50: u64) -> MeasurementResult { + MeasurementResult { + schema_version: SchemaVersion3::current(), + ok: true, + session_id: "s".to_owned(), + status: SessionStatus::Completed, + measurement: Measurement { + name: None, + samples: SampleSummary { + matched, + unmatched_start: 0, + unmatched_end: 0, + ambiguous: 0, + dropped: 0, + }, + latency_ns: LatencyStatistics { + min: p50, + mean: 0.0, + p50, + p90: p50, + p95: p50, + p99: p50, + max: p50, + }, + }, + quality: ResultQuality { + verdict: QualityVerdict::Good, + issues: Vec::new(), + }, + correlation: CorrelationSummary { + method: "exact".to_owned(), + confidence: CorrelationConfidence::Exact, + score: 1.0, + }, + clock: ClockQuality { + alignment: "host_monotonic".to_owned(), + estimated_error_ns: Some(0), + }, + collection: CollectionSummary { + completeness: CaptureCompleteness::Complete, + host_events: 0, + cuda_events: 0, + dropped_events: 0, + cupti: None, + }, + evidence: full_evidence(vec![pair(p50)]), + warnings: Vec::new(), + } + } + + fn codes(result: &MeasurementResult) -> Vec<(String, QualitySeverity)> { + assess(result) + .issues + .into_iter() + .map(|issue| (issue.code, issue.severity)) + .collect() + } + + #[test] + fn exemplars_follow_statistic_ranks_and_merge_shared_pairs() { + let pairs = (1..=200).rev().map(pair).collect::>(); + let selected = exemplars(&pairs); + let ranks = selected + .iter() + .map(|exemplar| (exemplar.ranks.clone(), exemplar.pair.latency_ns)) + .collect::>(); + assert_eq!( + ranks, + [ + (vec![LatencyRank::Min], 1), + (vec![LatencyRank::P50], 100), + (vec![LatencyRank::P90], 180), + (vec![LatencyRank::P99], 198), + (vec![LatencyRank::Max], 200), + ] + ); + assert_eq!(rank_index(200, 99), 197); + let single = exemplars(&[pair(7)]); + assert_eq!(single.len(), 1); + assert_eq!(single[0].ranks.len(), 5); + assert_eq!(exemplars(&[]), Vec::new()); + } + + #[test] + fn clean_exact_result_is_good() { + let quality = assess(&result(100, 1_000)); + assert_eq!(quality.verdict, QualityVerdict::Good); + assert_eq!(quality.issues, []); + } + + #[test] + fn few_samples_and_minor_unmatched_are_informational() { + let mut value = result(10, 1_000); + value.measurement.samples.unmatched_end = 2; + let quality = assess(&value); + assert_eq!(quality.verdict, QualityVerdict::Good); + assert_eq!( + codes(&value), + [ + ("UNMATCHED_EVENTS".to_owned(), QualitySeverity::Info), + ("FEW_SAMPLES".to_owned(), QualitySeverity::Info), + ] + ); + } + + #[test] + fn interpretation_limits_are_cautions() { + let mut value = result(100, 1_000); + value.status = SessionStatus::TimedOut; + value.correlation.confidence = CorrelationConfidence::Heuristic; + value.measurement.samples.ambiguous = 1; + value.measurement.samples.unmatched_start = 101; + value.clock.estimated_error_ns = Some(100); + assert_eq!(assess(&value).verdict, QualityVerdict::Caution); + assert_eq!( + codes(&value), + [ + ("TIMED_OUT".to_owned(), QualitySeverity::Caution), + ("HEURISTIC_CORRELATION".to_owned(), QualitySeverity::Caution), + ("AMBIGUOUS_PAIRS".to_owned(), QualitySeverity::Caution), + ("UNMATCHED_EVENTS".to_owned(), QualitySeverity::Caution), + ("CLOCK_ERROR_LARGE".to_owned(), QualitySeverity::Caution), + ] + ); + } + + #[test] + fn unknown_clock_error_is_reported() { + let mut value = result(100, 1_000); + value.clock.estimated_error_ns = None; + assert_eq!( + codes(&value), + [("CLOCK_ERROR_UNAVAILABLE".to_owned(), QualitySeverity::Info)] + ); + value.clock.estimated_error_ns = Some(99); + assert_eq!(codes(&value), []); + } + + #[test] + fn finalize_recomputes_quality_and_trims_summary_pairs() { + let mut value = result(100, 1_000); + value.status = SessionStatus::TimedOut; + finalize(&mut value, EvidenceMode::Summary); + assert_eq!(value.quality.verdict, QualityVerdict::Caution); + assert_eq!(value.evidence.mode, EvidenceMode::Summary); + assert_eq!(value.evidence.pairs, None); + assert_eq!(value.evidence.exemplars.len(), 1); + + let mut value = result(100, 1_000); + finalize(&mut value, EvidenceMode::Full); + assert_eq!(value.evidence.pairs.map(|pairs| pairs.len()), Some(1)); + } +} diff --git a/xprobe/protocol/src/lib.rs b/xprobe/protocol/src/lib.rs index 97d28dd..c2cabd8 100644 --- a/xprobe/protocol/src/lib.rs +++ b/xprobe/protocol/src/lib.rs @@ -35,9 +35,11 @@ pub use export::{ExportFormat, TraceExportResult}; pub use measurement::{ AggregateActivity, AggregateCollectionSummary, AggregateDuration, AggregateGroup, AggregateInventory, AggregateInventoryResult, CaptureCompleteness, ClockQuality, - CollectionSummary, CorrelationConfidence, CorrelationSummary, CuptiCollectionSummary, - LatencyStatistics, MatchPolicy, MatchedEventPair, Measurement, MeasurementMode, - MeasurementResult, MeasurementSpec, SampleSummary, SessionStatus, TargetIdentity, + CollectionSummary, CorrelationConfidence, CorrelationSummary, CuptiCollectionSummary, Evidence, + EvidenceArtifact, EvidenceExemplar, EvidenceMode, LatencyRank, LatencyStatistics, MatchPolicy, + MatchedEventPair, Measurement, MeasurementMode, MeasurementResult, MeasurementSpec, + QualityIssue, QualitySeverity, QualityVerdict, ResultQuality, SampleSummary, SessionStatus, + TargetIdentity, }; pub use process::{CgroupEntry, ProcessCredentials, ProcessCudaState, ProcessReport}; pub use resolve::{ElfObjectKind, ProcessMapping, ResolvedProbe}; @@ -51,6 +53,6 @@ pub use validate::{ ResolvedCudaSelector, ResolvedLinuxSelector, ValidatedEndpoint, ValidationIssue, ValidationRequirements, ValidationResult, }; -pub use version::SchemaVersion; +pub use version::{SchemaVersion, SchemaVersion3}; pub const SCHEMA_VERSION: &str = "2.0"; diff --git a/xprobe/protocol/src/measurement.rs b/xprobe/protocol/src/measurement.rs index e62ffc3..40113d5 100644 --- a/xprobe/protocol/src/measurement.rs +++ b/xprobe/protocol/src/measurement.rs @@ -1,7 +1,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::{Event, SchemaVersion, Warning}; +use crate::{Event, ExportFormat, SchemaVersion, SchemaVersion3, Warning}; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] @@ -60,19 +60,102 @@ pub enum SessionStatus { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct MeasurementResult { - pub schema_version: SchemaVersion, + pub schema_version: SchemaVersion3, pub ok: bool, pub session_id: String, pub status: SessionStatus, pub measurement: Measurement, + pub quality: ResultQuality, pub correlation: CorrelationSummary, pub clock: ClockQuality, pub collection: CollectionSummary, - pub evidence: Vec, + pub evidence: Evidence, #[serde(default)] pub warnings: Vec, } +/// Matched pair evidence. `summary` keeps representative exemplars inline and +/// references the bounded capture artifact; `full` also inlines every pair. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Evidence { + pub mode: EvidenceMode, + pub exemplars: Vec, + pub pairs: Option>, + pub artifact: Option, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum EvidenceMode { + #[default] + Summary, + Full, +} + +/// One matched pair selected for a latency rank. Ranks use the same +/// nearest-rank indices as `measurement.latency_ns`, so an exemplar's latency +/// equals the reported statistic. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct EvidenceExemplar { + pub ranks: Vec, + pub pair: MatchedEventPair, +} + +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum LatencyRank { + Min, + P50, + P90, + P99, + Max, +} + +/// Bounded capture written by `measure`; re-correlate it with `measure --input`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct EvidenceArtifact { + pub path: String, + pub format: ExportFormat, + pub events: u64, +} + +/// Deterministic interpretation of the result's quality fields. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ResultQuality { + pub verdict: QualityVerdict, + pub issues: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum QualityVerdict { + /// No caution-level issue; info issues may still be listed. + Good, + /// At least one issue limits how the latency may be interpreted. + Caution, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct QualityIssue { + pub code: String, + pub severity: QualitySeverity, + pub message: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum QualitySeverity { + Info, + Caution, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct MatchedEventPair { diff --git a/xprobe/protocol/src/version.rs b/xprobe/protocol/src/version.rs index 4c0929c..d2a25cf 100644 --- a/xprobe/protocol/src/version.rs +++ b/xprobe/protocol/src/version.rs @@ -13,3 +13,20 @@ impl SchemaVersion { Self::V2 } } + +/// Version of records whose shape changed in xprobe 0.6. +/// +/// Records keep their own version so unchanged contracts, such as Event JSONL +/// artifacts, remain readable at `2.0`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub enum SchemaVersion3 { + #[serde(rename = "3.0")] + V3, +} + +impl SchemaVersion3 { + #[must_use] + pub const fn current() -> Self { + Self::V3 + } +} diff --git a/xprobe/protocol/tests/contracts.rs b/xprobe/protocol/tests/contracts.rs index 790c90d..dc6b625 100644 --- a/xprobe/protocol/tests/contracts.rs +++ b/xprobe/protocol/tests/contracts.rs @@ -406,7 +406,7 @@ fn trace_export_contract_round_trips() { #[test] fn measurement_result_contract_round_trips() { assert_round_trip::(&json!({ - "schema_version": "2.0", + "schema_version": "3.0", "ok": true, "session_id": "xp_test", "status": "completed", @@ -429,6 +429,21 @@ fn measurement_result_contract_round_trips() { "max": 13_000 } }, + "quality": { + "verdict": "caution", + "issues": [ + { + "code": "HEURISTIC_CORRELATION", + "severity": "caution", + "message": "pairs are matched by time order" + }, + { + "code": "FEW_SAMPLES", + "severity": "info", + "message": "tail ranks are estimated from 1 matched samples" + } + ] + }, "correlation": { "method": "first_after", "confidence": "heuristic", @@ -445,11 +460,47 @@ fn measurement_result_contract_round_trips() { "dropped_events": 0, "cupti": null }, - "evidence": [], + "evidence": { + "mode": "summary", + "exemplars": [], + "pairs": null, + "artifact": { + "path": "/tmp/xprobe-1-2/events.jsonl", + "format": "jsonl", + "events": 2 + } + }, "warnings": [] })); } +#[test] +fn measurement_results_require_schema_three() { + let mut value = serde_json::to_value( + serde_json::from_value::(json!({ + "schema_version": "3.0", + "ok": true, + "session_id": "xp_test", + "status": "completed", + "measurement": { + "name": null, + "samples": {"matched": 1, "unmatched_start": 0, "unmatched_end": 0, "ambiguous": 0, "dropped": 0}, + "latency_ns": {"min": 1, "mean": 1.0, "p50": 1, "p90": 1, "p95": 1, "p99": 1, "max": 1} + }, + "quality": {"verdict": "good", "issues": []}, + "correlation": {"method": "exact", "confidence": "exact", "score": 1.0}, + "clock": {"alignment": "host_monotonic", "estimated_error_ns": 0}, + "collection": {"completeness": "complete", "host_events": 2, "cuda_events": 0, "dropped_events": 0, "cupti": null}, + "evidence": {"mode": "full", "exemplars": [], "pairs": [], "artifact": null}, + "warnings": [] + })) + .expect("3.0 measurement result must parse"), + ) + .expect("measurement result must serialize"); + value["schema_version"] = json!("2.0"); + assert!(serde_json::from_value::(value).is_err()); +} + #[test] fn process_report_contract_round_trips() { assert_round_trip::(&json!({ From fd01a8661a4caf0f366058d4f3a3c97b53987cc4 Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 14 Nov 2026 05:19:56 +0800 Subject: [PATCH 02/15] =?UTF-8?q?=F0=9F=90=9B=20fix:=20keep=20discovery=20?= =?UTF-8?q?working=20on=20shared=20GPUs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NVML reports compute processes from every container on the GPU. When a listed PID was not visible in the caller's PID namespace, the ancestry walk read a missing /proc entry and failed the whole command with TARGET_EXITED, so discovery broke whenever another container used the GPU. Such a PID cannot descend from the root, so classify it as unavailable, re-walk once when an ancestor exits mid-walk, and skip workers that exit before inspection. Report skipped PIDs in a NVML_PROCESS_UNAVAILABLE warning; root exit and other inspection failures stay errors. Also update the Skill invariant checked by the contract test for the schema 3.0 measurement wording. --- Cargo.lock | 1 + docs/cli-contract.md | 6 + tests/agent-contract/test_contract.py | 2 +- xprobe/core/Cargo.toml | 1 + xprobe/core/src/discover.rs | 180 ++++++++++++++++++++++---- 5 files changed, 164 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5dd424d..b846c10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -761,6 +761,7 @@ dependencies = [ "nix", "object", "regex", + "serde_json", "xprobe-protocol", ] diff --git a/docs/cli-contract.md b/docs/cli-contract.md index 7695ac9..361e18c 100644 --- a/docs/cli-contract.md +++ b/docs/cli-contract.md @@ -50,6 +50,12 @@ GPU UUIDs. `limit` must be positive. `total_candidates` and `truncated` describe bounded output. The caller chooses one candidate and passes its PID to `validate` and `measure`; xprobe does not guess among workers. +NVML lists compute processes from every container sharing the GPU. A listed PID +that is not visible in the caller's PID namespace, or that exits before +inspection, cannot be a candidate; it is reported in a +`NVML_PROCESS_UNAVAILABLE` warning with its `nvml_pids` instead of failing the +command. Root exit and other inspection failures remain errors. + This process-candidate response is discovery schema `2.0`. xprobe does not emit or maintain the pre-0.3 event-discovery shape. diff --git a/tests/agent-contract/test_contract.py b/tests/agent-contract/test_contract.py index b7f1232..a66ee99 100755 --- a/tests/agent-contract/test_contract.py +++ b/tests/agent-contract/test_contract.py @@ -74,7 +74,7 @@ def check_skill(workspace: pathlib.Path) -> None: ): assert adaptive_rule in normalized_skill for invariant in ( - "schema version `2.0`", + "Measurement results use schema version `3.0`", "PID plus procfs start time", "Pass the selected exact endpoints through read-only `validate`", "Bound every capture", diff --git a/xprobe/core/Cargo.toml b/xprobe/core/Cargo.toml index 6e95266..03a0c00 100644 --- a/xprobe/core/Cargo.toml +++ b/xprobe/core/Cargo.toml @@ -11,6 +11,7 @@ cpp_demangle.workspace = true object.workspace = true nix.workspace = true regex.workspace = true +serde_json.workspace = true xprobe-protocol.workspace = true [lints] diff --git a/xprobe/core/src/discover.rs b/xprobe/core/src/discover.rs index b852778..e70dc7d 100644 --- a/xprobe/core/src/discover.rs +++ b/xprobe/core/src/discover.rs @@ -6,7 +6,7 @@ use std::{ }; use xprobe_protocol::{ - CudaProcessCandidate, DiscoveryResult, ErrorCode, ProcessReport, SchemaVersion, + CudaProcessCandidate, DiscoveryResult, ErrorCode, ProcessReport, SchemaVersion, Warning, }; use crate::inspect::{self, InspectError}; @@ -105,18 +105,33 @@ pub fn run(report: &ProcessReport, limit: usize) -> Result {} + Ancestry::Unrelated => continue, + Ancestry::Unavailable => { + unavailable.push(pid); + continue; + } } - let candidate_report = inspect::run(pid)?; - candidates.push(CudaProcessCandidate { - target: candidate_report.target, - parent_pid: inspect::read_parent_pid(pid)?, - executable: candidate_report.executable, - command_line: candidate_report.command_line, - gpu_uuids: gpu_uuids.into_iter().collect(), - }); + let candidate = match inspect::run(pid).and_then(|candidate_report| { + Ok(CudaProcessCandidate { + parent_pid: inspect::read_parent_pid(pid)?, + target: candidate_report.target, + executable: candidate_report.executable, + command_line: candidate_report.command_line, + gpu_uuids: gpu_uuids.into_iter().collect(), + }) + }) { + Ok(candidate) => candidate, + Err(InspectError::TargetExited { .. } | InspectError::TargetNotFound { .. }) => { + unavailable.push(pid); + continue; + } + Err(error) => return Err(error.into()), + }; + candidates.push(candidate); } candidates.sort_by_key(|candidate| candidate.target.pid); let total_candidates = candidates.len(); @@ -130,7 +145,73 @@ pub fn run(report: &ProcessReport, limit: usize) -> Result candidates.len(), candidates, - warnings: Vec::new(), + warnings: unavailable_warning(&unavailable).into_iter().collect(), + }) +} + +/// Relationship between an NVML compute PID and the requested root. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Ancestry { + Descendant, + Unrelated, + /// The PID is not visible in this PID namespace or exited before its + /// ancestry could be read. Either way it cannot be inspected as a worker. + Unavailable, +} + +/// Walk parent links from `pid` to `root`. NVML reports compute processes from +/// every container sharing the GPU, so a PID that this namespace cannot see is +/// not a descendant of `root`. If an ancestor exits mid-walk the candidate has +/// been reparented; walk once more from the candidate. +fn ancestry( + pid: u32, + root: u32, + parent_of: impl Fn(u32) -> Result, +) -> Result { + for _ in 0..2 { + match walk_ancestry(pid, root, &parent_of) { + Ok(true) => return Ok(Ancestry::Descendant), + Ok(false) => return Ok(Ancestry::Unrelated), + Err(InspectError::TargetExited { pid: missing }) if missing == pid => { + return Ok(Ancestry::Unavailable); + } + Err(InspectError::TargetExited { .. }) => {} + Err(error) => return Err(error.into()), + } + } + Ok(Ancestry::Unavailable) +} + +fn walk_ancestry( + mut pid: u32, + root: u32, + parent_of: &impl Fn(u32) -> Result, +) -> Result { + let mut visited = BTreeSet::new(); + loop { + if pid == root { + return Ok(true); + } + if pid == 0 || !visited.insert(pid) { + return Ok(false); + } + pid = parent_of(pid)?; + } +} + +fn unavailable_warning(pids: &[u32]) -> Option { + if pids.is_empty() { + return None; + } + Some(Warning { + code: "NVML_PROCESS_UNAVAILABLE".to_owned(), + message: format!( + "{} NVML compute processes are not visible in this PID namespace or exited before inspection", + pids.len() + ), + details: [("nvml_pids".to_owned(), serde_json::json!(pids))] + .into_iter() + .collect(), }) } @@ -153,22 +234,71 @@ fn parse_compute_processes(output: &str) -> Result Result { - let mut visited = BTreeSet::new(); - loop { - if pid == root { - return Ok(true); - } - if pid == 0 || !visited.insert(pid) { - return Ok(false); +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use crate::inspect::InspectError; + + use super::{Ancestry, DiscoverError, ancestry, parse_compute_processes, unavailable_warning}; + + fn tree(edges: &[(u32, u32)]) -> impl Fn(u32) -> Result { + let parents = edges.iter().copied().collect::>(); + move |pid| { + parents + .get(&pid) + .copied() + .ok_or(InspectError::TargetExited { pid }) } - pid = inspect::read_parent_pid(pid)?; } -} -#[cfg(test)] -mod tests { - use super::{DiscoverError, parse_compute_processes}; + #[test] + fn classifies_descendants_unrelated_and_invisible_processes() { + let parents = tree(&[(30, 20), (20, 10), (10, 1), (40, 1), (1, 0)]); + assert_eq!(ancestry(30, 10, &parents).unwrap(), Ancestry::Descendant); + assert_eq!(ancestry(10, 10, &parents).unwrap(), Ancestry::Descendant); + assert_eq!(ancestry(40, 10, &parents).unwrap(), Ancestry::Unrelated); + assert_eq!(ancestry(111, 10, &parents).unwrap(), Ancestry::Unavailable); + } + + #[test] + fn rewalks_once_when_an_ancestor_exits_mid_walk() { + let calls = std::cell::Cell::new(0); + let parents = |pid: u32| { + calls.set(calls.get() + 1); + match (pid, calls.get()) { + (30, 1) => Ok(20), + (20, 2) => Err(InspectError::TargetExited { pid: 20 }), + (30, _) => Ok(10), + (pid, _) => Err(InspectError::TargetExited { pid }), + } + }; + assert_eq!(ancestry(30, 10, parents).unwrap(), Ancestry::Descendant); + } + + #[test] + fn propagates_other_inspection_failures() { + let denied = |pid: u32| { + Err(InspectError::PermissionDenied { + pid, + path: format!("/proc/{pid}/stat").into(), + }) + }; + assert!(matches!( + ancestry(30, 10, denied), + Err(DiscoverError::Inspect( + InspectError::PermissionDenied { .. } + )) + )); + } + + #[test] + fn reports_unavailable_nvml_processes() { + assert!(unavailable_warning(&[]).is_none()); + let warning = unavailable_warning(&[111, 222]).unwrap(); + assert_eq!(warning.code, "NVML_PROCESS_UNAVAILABLE"); + assert_eq!(warning.details["nvml_pids"], serde_json::json!([111, 222])); + } #[test] fn groups_compute_processes_across_devices() { From a69cfe53926740b541d2727c9ff06b2d4a3740d9 Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 14 Nov 2026 05:21:25 +0800 Subject: [PATCH 03/15] =?UTF-8?q?=E2=9C=A8=20feat:=20default=20to=20JSON?= =?UTF-8?q?=20when=20stdout=20is=20not=20a=20terminal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agents and pipelines had to pass `--json --non-interactive --no-color` on every call, and the docs repeated that triplet dozens of times. Commands now emit their versioned JSON whenever stdout is not a terminal and keep the human report on a terminal. A global `--human` forces the report in a pipe and conflicts with `--json`. Existing flags remain accepted; `dev uprobe --jsonl` streams are unchanged. --- docs/cli-contract.md | 16 +++-- xprobe/cli/src/args.rs | 128 +++++++++++++++++++++++++++++++++++-- xprobe/cli/src/main.rs | 10 ++- xprobe/cli/tests/doctor.rs | 26 +++++++- 4 files changed, 164 insertions(+), 16 deletions(-) diff --git a/docs/cli-contract.md b/docs/cli-contract.md index 361e18c..347d1b1 100644 --- a/docs/cli-contract.md +++ b/docs/cli-contract.md @@ -5,12 +5,16 @@ and `measure`. ## Common behavior -All commands support `--json --non-interactive --no-color`. JSON mode writes one -versioned document to stdout. Diagnostics, including the online-injection -warning, go to stderr. Commands never prompt. - -All success and error records carry `schema_version: "2.0"`. Unknown JSON -fields and unsupported schema versions are rejected. xprobe maintains only the +When stdout is not a terminal, every command writes one versioned JSON document +to stdout; on a terminal it prints a human-readable report. `--json` forces +JSON and `--human` forces the report; they cannot be combined. +`--non-interactive` and `--no-color` are accepted for compatibility: commands +never prompt and never color. Diagnostics, including the online-injection +warning, go to stderr. + +Measurement results carry `schema_version: "3.0"`; every other success record, +error record, and Event JSONL line carries `"2.0"`. Unknown JSON fields and +unsupported schema versions are rejected. xprobe maintains only the current schema during the pre-1.0 protocol transition. | Exit | Meaning | diff --git a/xprobe/cli/src/args.rs b/xprobe/cli/src/args.rs index fd4efdd..d98c30a 100644 --- a/xprobe/cli/src/args.rs +++ b/xprobe/cli/src/args.rs @@ -7,10 +7,56 @@ use std::path::PathBuf; #[derive(Debug, Parser)] #[command(name = "xprobe", version, about = "Runtime host-to-GPU latency probe")] pub(crate) struct Cli { + /// Print human-readable output even when stdout is not a terminal. + #[arg(long, global = true)] + pub(crate) human: bool, + #[command(subcommand)] pub(crate) command: Command, } +impl Cli { + /// Choose the output form. JSON is the default whenever stdout is not a + /// terminal, so agents and pipelines need no flags; `--human` keeps text. + /// + /// # Errors + /// + /// Returns a message when `--human` and `--json` are both set. + pub(crate) fn resolve_output(&mut self, stdout_is_terminal: bool) -> Result<(), &'static str> { + let human = self.human; + let Some(json) = self.command.json_flag() else { + return Ok(()); + }; + if human && *json { + return Err("--human cannot be used with --json"); + } + if !human && !stdout_is_terminal { + *json = true; + } + Ok(()) + } +} + +impl Command { + fn json_flag(&mut self) -> Option<&mut bool> { + match self { + Self::Doctor(args) => Some(&mut args.json), + Self::Discover(args) => Some(&mut args.json), + Self::Inspect(args) => Some(&mut args.json), + Self::Resolve(args) => Some(&mut args.json), + Self::Validate(args) => Some(&mut args.output.json), + Self::Measure(args) => Some(&mut args.output.json), + Self::Trace(args) => Some(&mut args.json), + Self::Export(args) => Some(&mut args.json), + Self::Capture(args) | Self::Dev(args) => match &mut args.command { + DevCommand::Uprobe(args) if args.output.jsonl => None, + DevCommand::Uprobe(args) => Some(&mut args.output.json), + DevCommand::Cupti(args) => Some(&mut args.json), + }, + } + } +} + #[derive(Debug, Subcommand)] pub(crate) enum Command { /// Inspect local tracing and GPU capabilities. @@ -57,7 +103,8 @@ pub(crate) struct TraceArgs { #[arg(long, hide = true)] pub(crate) cupti_socket: Option, - /// Emit only the versioned JSON result on stdout. + /// Emit only the versioned JSON result on stdout (default when stdout is + /// not a terminal). #[arg(long)] pub(crate) json: bool, @@ -128,7 +175,8 @@ pub(crate) enum DevCommand { #[derive(Debug, Clone, Copy, Args)] pub(crate) struct DoctorArgs { - /// Emit only the versioned JSON result on stdout. + /// Emit only the versioned JSON result on stdout (default when stdout is + /// not a terminal). #[arg(long)] pub(crate) json: bool, @@ -151,7 +199,8 @@ pub(crate) struct DiscoverArgs { #[arg(long, default_value_t = 200)] pub(crate) limit: usize, - /// Emit only the versioned JSON result on stdout. + /// Emit only the versioned JSON result on stdout (default when stdout is + /// not a terminal). #[arg(long)] pub(crate) json: bool, @@ -170,7 +219,8 @@ pub(crate) struct InspectArgs { #[arg(long)] pub(crate) pid: u32, - /// Emit only the versioned JSON result on stdout. + /// Emit only the versioned JSON result on stdout (default when stdout is + /// not a terminal). #[arg(long)] pub(crate) json: bool, @@ -193,7 +243,8 @@ pub(crate) struct ResolveArgs { #[arg(long, alias = "event")] pub(crate) selector: String, - /// Emit only the versioned JSON result on stdout. + /// Emit only the versioned JSON result on stdout (default when stdout is + /// not a terminal). #[arg(long)] pub(crate) json: bool, @@ -376,7 +427,8 @@ pub(crate) struct MeasureArgs { #[derive(Debug, Clone, Copy, Args)] pub(crate) struct CommonOutputArgs { - /// Emit only the versioned JSON result on stdout. + /// Emit only the versioned JSON result on stdout (default when stdout is + /// not a terminal). #[arg(long)] pub(crate) json: bool, @@ -433,7 +485,8 @@ pub(crate) struct UprobeArgs { #[derive(Debug, Clone, Copy, Args)] pub(crate) struct EventOutputArgs { - /// Emit only the versioned JSON result on stdout. + /// Emit only the versioned JSON result on stdout (default when stdout is + /// not a terminal). #[arg(long)] pub(crate) json: bool, @@ -472,3 +525,64 @@ pub(crate) struct CuptiArgs { #[arg(long)] pub(crate) non_interactive: bool, } + +#[cfg(test)] +mod tests { + use clap::Parser; + + use super::{Cli, Command}; + + fn json(arguments: &[&str], terminal: bool) -> Result { + let mut cli = Cli::try_parse_from(arguments).expect("arguments must parse"); + cli.resolve_output(terminal)?; + Ok(match cli.command { + Command::Doctor(args) => args.json, + Command::Measure(args) => args.output.json, + _ => unreachable!("test covers doctor and measure"), + }) + } + + #[test] + fn defaults_to_json_only_when_stdout_is_not_a_terminal() { + assert_eq!(json(&["xprobe", "doctor"], false), Ok(true)); + assert_eq!(json(&["xprobe", "doctor"], true), Ok(false)); + assert_eq!(json(&["xprobe", "doctor", "--json"], true), Ok(true)); + assert_eq!( + json(&["xprobe", "measure", "--input", "a.jsonl"], false), + Ok(true) + ); + } + + #[test] + fn human_keeps_text_output_and_conflicts_with_json() { + assert_eq!(json(&["xprobe", "doctor", "--human"], false), Ok(false)); + assert_eq!(json(&["xprobe", "--human", "doctor"], false), Ok(false)); + assert!(json(&["xprobe", "doctor", "--human", "--json"], true).is_err()); + } + + #[test] + fn leaves_jsonl_event_streams_alone() { + let mut cli = Cli::try_parse_from([ + "xprobe", + "dev", + "uprobe", + "--pid", + "1", + "--binary", + "/bin/true", + "--symbol", + "main", + "--jsonl", + ]) + .expect("uprobe arguments must parse"); + cli.resolve_output(false).expect("jsonl output is valid"); + let Command::Dev(args) = cli.command else { + unreachable!("dev command parsed") + }; + let super::DevCommand::Uprobe(args) = args.command else { + unreachable!("uprobe parsed") + }; + assert!(args.output.jsonl); + assert!(!args.output.json); + } +} diff --git a/xprobe/cli/src/main.rs b/xprobe/cli/src/main.rs index 2820787..27d569c 100644 --- a/xprobe/cli/src/main.rs +++ b/xprobe/cli/src/main.rs @@ -1,5 +1,6 @@ use std::{ fs::{self, DirBuilder}, + io::{self, IsTerminal}, os::unix::fs::DirBuilderExt, path::{Path, PathBuf}, process::ExitCode, @@ -9,7 +10,7 @@ use std::{ static EXPORT_SEQUENCE: AtomicU64 = AtomicU64::new(0); -use clap::Parser; +use clap::{CommandFactory, Parser}; use xprobe_collector::completed; use xprobe_core::{cpu_sampling, discover, doctor, inspect, syscall_aggregate, validate}; use xprobe_correlator::{MeasureOptions, finalize, measure}; @@ -47,7 +48,12 @@ use render::{ }; fn main() -> ExitCode { - let cli = Cli::parse(); + let mut cli = Cli::parse(); + if let Err(message) = cli.resolve_output(io::stdout().is_terminal()) { + Cli::command() + .error(clap::error::ErrorKind::ArgumentConflict, message) + .exit(); + } match cli.command { Command::Doctor(args) => run_doctor(args), Command::Discover(args) => run_discover(args), diff --git a/xprobe/cli/tests/doctor.rs b/xprobe/cli/tests/doctor.rs index 3a77499..af69e78 100644 --- a/xprobe/cli/tests/doctor.rs +++ b/xprobe/cli/tests/doctor.rs @@ -25,7 +25,7 @@ fn doctor_json_is_machine_readable() { #[test] fn doctor_has_a_human_readable_report() { let output = Command::new(env!("CARGO_BIN_EXE_xprobe")) - .arg("doctor") + .args(["doctor", "--human"]) .output() .expect("xprobe doctor must run"); @@ -34,3 +34,27 @@ fn doctor_has_a_human_readable_report() { assert!(stdout.contains("Capabilities:")); assert!(stdout.contains("Checks:")); } + +#[test] +fn doctor_defaults_to_json_when_stdout_is_piped() { + let output = Command::new(env!("CARGO_BIN_EXE_xprobe")) + .arg("doctor") + .output() + .expect("xprobe doctor must run"); + + assert!(output.status.success()); + let report: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("piped stdout must be JSON"); + assert_eq!(report["schema_version"], "2.0"); +} + +#[test] +fn human_and_json_flags_conflict() { + let output = Command::new(env!("CARGO_BIN_EXE_xprobe")) + .args(["doctor", "--human", "--json"]) + .output() + .expect("xprobe doctor must run"); + + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("--human cannot be used with --json")); +} From a79ecec492d73e065db782af66a0cf71236f885f Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 14 Nov 2026 05:28:18 +0800 Subject: [PATCH 04/15] =?UTF-8?q?=E2=9C=A8=20feat:=20read=20thread=20time,?= =?UTF-8?q?=20cgroup=20throttling,=20and=20GPU=20health?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add deterministic readers for the triage window: per-thread on-CPU, runqueue, and blocked time from procfs schedstat; CFS throttling and quota for cgroup v1 and v2; and NVML utilization, clock event reasons, temperature, power, and PCIe link sampled through nvidia-smi. --- xprobe/core/src/cgroup_cpu.rs | 286 +++++++++++++++++++++++++++++++++ xprobe/core/src/gpu_health.rs | 276 +++++++++++++++++++++++++++++++ xprobe/core/src/lib.rs | 3 + xprobe/core/src/thread_time.rs | 277 +++++++++++++++++++++++++++++++ xprobe/protocol/src/lib.rs | 4 + xprobe/protocol/src/triage.rs | 85 ++++++++++ 6 files changed, 931 insertions(+) create mode 100644 xprobe/core/src/cgroup_cpu.rs create mode 100644 xprobe/core/src/gpu_health.rs create mode 100644 xprobe/core/src/thread_time.rs create mode 100644 xprobe/protocol/src/triage.rs diff --git a/xprobe/core/src/cgroup_cpu.rs b/xprobe/core/src/cgroup_cpu.rs new file mode 100644 index 0000000..9a51235 --- /dev/null +++ b/xprobe/core/src/cgroup_cpu.rs @@ -0,0 +1,286 @@ +//! CFS bandwidth throttling counters for a process's CPU cgroup. + +use std::{ + fs, io, + path::{Path, PathBuf}, +}; + +use xprobe_protocol::{CgroupCpuThrottling, CgroupVersion}; + +use crate::inspect::InspectError; + +/// The CPU controller directory for one process. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CpuCgroup { + pub version: CgroupVersion, + pub path: String, + directory: PathBuf, +} + +/// A configured CFS bandwidth limit. +#[derive(Debug, Clone, Copy, PartialEq)] +enum CpuLimit { + Unlimited, + Cpus(f64), +} + +impl CpuLimit { + const fn cpus(self) -> Option { + match self { + Self::Unlimited => None, + Self::Cpus(cpus) => Some(cpus), + } + } +} + +/// Cumulative throttling counters at one point in time. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct CpuCounters { + pub periods: u64, + pub throttled_periods: u64, + pub throttled_ns: u64, +} + +/// Find the CPU cgroup of `pid` as seen from this mount namespace. +/// +/// Returns `Ok(None)` when no CPU controller directory is mounted for it. +/// +/// # Errors +/// +/// Returns [`InspectError`] when `/proc//cgroup` is unreadable or +/// malformed. +pub fn locate(pid: u32) -> Result, InspectError> { + let path = PathBuf::from(format!("/proc/{pid}/cgroup")); + let text = fs::read_to_string(&path).map_err(|source| match source.kind() { + io::ErrorKind::NotFound => InspectError::TargetExited { pid }, + _ => InspectError::Io { + path: path.clone(), + source, + }, + })?; + let Some((version, relative)) = cpu_cgroup_path(&text) else { + return Ok(None); + }; + let root = match version { + CgroupVersion::V2 => PathBuf::from("/sys/fs/cgroup"), + CgroupVersion::V1 => ["/sys/fs/cgroup/cpu,cpuacct", "/sys/fs/cgroup/cpu"] + .into_iter() + .map(PathBuf::from) + .find(|root| root.is_dir()) + .unwrap_or_else(|| PathBuf::from("/sys/fs/cgroup/cpu")), + }; + let directory = root.join(relative.trim_start_matches('/')); + if !directory.join("cpu.stat").is_file() { + return Ok(None); + } + Ok(Some(CpuCgroup { + version, + path: relative, + directory, + })) +} + +/// Pick the v2 unified entry, or the v1 entry carrying the `cpu` controller. +fn cpu_cgroup_path(text: &str) -> Option<(CgroupVersion, String)> { + let mut unified = None; + for line in text.lines() { + let mut fields = line.splitn(3, ':'); + let (Some(_), Some(controllers), Some(path)) = + (fields.next(), fields.next(), fields.next()) + else { + continue; + }; + if controllers.split(',').any(|controller| controller == "cpu") { + return Some((CgroupVersion::V1, path.to_owned())); + } + if controllers.is_empty() { + unified = Some((CgroupVersion::V2, path.to_owned())); + } + } + unified +} + +impl CpuCgroup { + /// Read cumulative throttling counters. + /// + /// # Errors + /// + /// Returns [`InspectError`] when `cpu.stat` is unreadable. + pub fn counters(&self) -> Result { + let text = read(&self.directory.join("cpu.stat"))?; + Ok(parse_cpu_stat(&text, self.version)) + } + + /// Read the configured quota as a CPU count, `None` when unlimited. + /// + /// # Errors + /// + /// Returns [`InspectError`] when present limit files are unreadable or + /// malformed. + pub fn quota_cpus(&self) -> Result, InspectError> { + match self.version { + CgroupVersion::V2 => { + let path = self.directory.join("cpu.max"); + if !path.is_file() { + return Ok(None); + } + parse_cpu_max(&read(&path)?) + .map(CpuLimit::cpus) + .ok_or_else(|| malformed(&path)) + } + CgroupVersion::V1 => { + let quota_path = self.directory.join("cpu.cfs_quota_us"); + let period_path = self.directory.join("cpu.cfs_period_us"); + if !quota_path.is_file() || !period_path.is_file() { + return Ok(None); + } + let quota = read("a_path)?; + let period = read(&period_path)?; + parse_cfs_quota("a, &period) + .map(CpuLimit::cpus) + .ok_or_else(|| malformed("a_path)) + } + } + } +} + +/// Throttling between two counter snapshots. +#[must_use] +pub fn throttling( + cgroup: &CpuCgroup, + quota_cpus: Option, + before: CpuCounters, + after: CpuCounters, +) -> CgroupCpuThrottling { + CgroupCpuThrottling { + version: cgroup.version, + cgroup_path: cgroup.path.clone(), + quota_cpus, + periods: after.periods.saturating_sub(before.periods), + throttled_periods: after + .throttled_periods + .saturating_sub(before.throttled_periods), + throttled_ns: after.throttled_ns.saturating_sub(before.throttled_ns), + } +} + +fn read(path: &Path) -> Result { + fs::read_to_string(path).map_err(|source| InspectError::Io { + path: path.to_path_buf(), + source, + }) +} + +fn malformed(path: &Path) -> InspectError { + InspectError::Io { + path: path.to_path_buf(), + source: io::Error::new(io::ErrorKind::InvalidData, "malformed CPU limit"), + } +} + +/// Missing keys read as zero: v2 omits bandwidth keys without the controller. +fn parse_cpu_stat(text: &str, version: CgroupVersion) -> CpuCounters { + let mut counters = CpuCounters::default(); + for line in text.lines() { + let mut fields = line.split_whitespace(); + let (Some(key), Some(value)) = ( + fields.next(), + fields.next().and_then(|v| v.parse::().ok()), + ) else { + continue; + }; + match (version, key) { + (_, "nr_periods") => counters.periods = value, + (_, "nr_throttled") => counters.throttled_periods = value, + (CgroupVersion::V2, "throttled_usec") => { + counters.throttled_ns = value.saturating_mul(1_000); + } + (CgroupVersion::V1, "throttled_time") => counters.throttled_ns = value, + _ => {} + } + } + counters +} + +#[allow(clippy::cast_precision_loss)] +fn parse_cpu_max(text: &str) -> Option { + let mut fields = text.split_whitespace(); + let quota = fields.next()?; + let period = fields + .next()? + .parse::() + .ok() + .filter(|period| *period > 0)?; + if quota == "max" { + return Some(CpuLimit::Unlimited); + } + Some(CpuLimit::Cpus( + quota.parse::().ok()? as f64 / period as f64, + )) +} + +#[allow(clippy::cast_precision_loss)] +fn parse_cfs_quota(quota: &str, period: &str) -> Option { + let quota = quota.trim().parse::().ok()?; + let period = period + .trim() + .parse::() + .ok() + .filter(|period| *period > 0)?; + if quota < 0 { + return Some(CpuLimit::Unlimited); + } + Some(CpuLimit::Cpus(quota as f64 / period as f64)) +} + +#[cfg(test)] +mod tests { + use xprobe_protocol::CgroupVersion; + + use super::{ + CpuCounters, CpuLimit, cpu_cgroup_path, parse_cfs_quota, parse_cpu_max, parse_cpu_stat, + }; + + #[test] + fn selects_v2_and_v1_cpu_paths() { + assert_eq!( + cpu_cgroup_path("0::/kubepods/pod1/abc\n"), + Some((CgroupVersion::V2, "/kubepods/pod1/abc".to_owned())) + ); + assert_eq!( + cpu_cgroup_path("12:memory:/a\n4:cpu,cpuacct:/docker/x\n0::/\n"), + Some((CgroupVersion::V1, "/docker/x".to_owned())) + ); + assert_eq!(cpu_cgroup_path("5:memory:/a\n"), None); + } + + #[test] + fn parses_counters_and_limits() { + let v2 = "usage_usec 1\nnr_periods 10\nnr_throttled 4\nthrottled_usec 2500\n"; + assert_eq!( + parse_cpu_stat(v2, CgroupVersion::V2), + CpuCounters { + periods: 10, + throttled_periods: 4, + throttled_ns: 2_500_000 + } + ); + let v1 = "nr_periods 3\nnr_throttled 1\nthrottled_time 777\n"; + assert_eq!(parse_cpu_stat(v1, CgroupVersion::V1).throttled_ns, 777); + assert_eq!( + parse_cpu_stat("usage_usec 5\n", CgroupVersion::V2), + CpuCounters::default() + ); + assert_eq!(parse_cpu_max("50000 100000\n"), Some(CpuLimit::Cpus(0.5))); + assert_eq!(parse_cpu_max("max 100000\n"), Some(CpuLimit::Unlimited)); + assert_eq!(parse_cpu_max("garbage"), None); + assert_eq!( + parse_cfs_quota("-1\n", "100000\n"), + Some(CpuLimit::Unlimited) + ); + assert_eq!( + parse_cfs_quota("200000\n", "100000\n"), + Some(CpuLimit::Cpus(2.0)) + ); + } +} diff --git a/xprobe/core/src/gpu_health.rs b/xprobe/core/src/gpu_health.rs new file mode 100644 index 0000000..06d72e9 --- /dev/null +++ b/xprobe/core/src/gpu_health.rs @@ -0,0 +1,276 @@ +//! NVML device health sampled through `nvidia-smi`. + +use std::{collections::BTreeMap, error::Error, fmt, io, process::Command}; + +use xprobe_protocol::{GpuDeviceHealth, PcieLink}; + +const FIELDS: &str = "index,uuid,utilization.gpu,utilization.memory,clocks_event_reasons.active,\ +temperature.gpu,power.draw,pcie.link.gen.current,pcie.link.gen.max,pcie.link.width.current,\ +pcie.link.width.max"; +const LEGACY_FIELDS: &str = "index,uuid,utilization.gpu,utilization.memory,clocks_throttle_reasons.active,\ +temperature.gpu,power.draw,pcie.link.gen.current,pcie.link.gen.max,pcie.link.width.current,\ +pcie.link.width.max"; + +/// NVML clock event reason bits and their names. +const CLOCK_EVENT_REASONS: [(u64, &str); 9] = [ + (0x1, "gpu_idle"), + (0x2, "applications_clocks_setting"), + (0x4, "sw_power_cap"), + (0x8, "hw_slowdown"), + (0x10, "sync_boost"), + (0x20, "sw_thermal_slowdown"), + (0x40, "hw_thermal_slowdown"), + (0x80, "hw_power_brake_slowdown"), + (0x100, "display_clock_setting"), +]; + +/// One device row from one query. +#[derive(Debug, Clone, PartialEq)] +pub struct DeviceSample { + pub index: u32, + pub uuid: String, + pub utilization_pct: Option, + pub memory_utilization_pct: Option, + pub clock_event_reasons: Option, + pub temperature_c: Option, + pub power_w: Option, + pub pcie_generation: Option<(u32, u32)>, + pub pcie_width: Option<(u32, u32)>, +} + +#[derive(Debug)] +pub enum GpuHealthError { + Command(io::Error), + Failed { status: Option, stderr: String }, + InvalidOutput(String), +} + +impl fmt::Display for GpuHealthError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Command(error) => write!(formatter, "failed to run nvidia-smi: {error}"), + Self::Failed { status, stderr } => { + write!( + formatter, + "nvidia-smi failed with status {status:?}: {stderr}" + ) + } + Self::InvalidOutput(line) => write!(formatter, "invalid nvidia-smi row: {line:?}"), + } + } +} + +impl Error for GpuHealthError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Command(error) => Some(error), + _ => None, + } + } +} + +/// Query every visible device once. +/// +/// # Errors +/// +/// Returns [`GpuHealthError`] when `nvidia-smi` is missing, fails with both +/// field spellings, or prints a malformed row. +pub fn query() -> Result, GpuHealthError> { + match run(FIELDS) { + Err(GpuHealthError::Failed { .. }) => run(LEGACY_FIELDS), + result => result, + } +} + +fn run(fields: &str) -> Result, GpuHealthError> { + let output = Command::new("nvidia-smi") + .args([ + &format!("--query-gpu={fields}"), + "--format=csv,noheader,nounits", + ]) + .output() + .map_err(GpuHealthError::Command)?; + if !output.status.success() { + return Err(GpuHealthError::Failed { + status: output.status.code(), + stderr: String::from_utf8_lossy(&output.stderr).trim().to_owned(), + }); + } + let text = String::from_utf8(output.stdout) + .map_err(|error| GpuHealthError::InvalidOutput(error.to_string()))?; + text.lines() + .filter(|line| !line.trim().is_empty()) + .map(parse_row) + .collect() +} + +fn parse_row(line: &str) -> Result { + let invalid = || GpuHealthError::InvalidOutput(line.to_owned()); + let fields = line.split(',').map(str::trim).collect::>(); + if fields.len() != 11 { + return Err(invalid()); + } + let index = fields[0].parse().map_err(|_| invalid())?; + if fields[1].is_empty() { + return Err(invalid()); + } + let pair = |left: Option, right: Option| left.zip(right); + Ok(DeviceSample { + index, + uuid: fields[1].to_owned(), + utilization_pct: optional(fields[2]).map_err(|()| invalid())?, + memory_utilization_pct: optional(fields[3]).map_err(|()| invalid())?, + clock_event_reasons: optional_hex(fields[4]).map_err(|()| invalid())?, + temperature_c: optional(fields[5]).map_err(|()| invalid())?, + power_w: optional(fields[6]).map_err(|()| invalid())?, + pcie_generation: pair( + optional(fields[7]).map_err(|()| invalid())?, + optional(fields[8]).map_err(|()| invalid())?, + ), + pcie_width: pair( + optional(fields[9]).map_err(|()| invalid())?, + optional(fields[10]).map_err(|()| invalid())?, + ), + }) +} + +/// `[N/A]` and `[Not Supported]` read as absent; anything else must parse. +fn optional(field: &str) -> Result, ()> { + if field.starts_with('[') { + return Ok(None); + } + field.parse().map(Some).map_err(|_| ()) +} + +fn optional_hex(field: &str) -> Result, ()> { + if field.starts_with('[') { + return Ok(None); + } + let digits = field.strip_prefix("0x").ok_or(())?; + u64::from_str_radix(digits, 16).map(Some).map_err(|_| ()) +} + +/// Summarize repeated queries per device, ordered by index. +#[must_use] +#[allow(clippy::cast_precision_loss)] +pub fn summarize(rounds: &[Vec]) -> Vec { + let mut devices = BTreeMap::<(u32, &str), Vec<&DeviceSample>>::new(); + for sample in rounds.iter().flatten() { + devices + .entry((sample.index, &sample.uuid)) + .or_default() + .push(sample); + } + devices + .into_iter() + .map(|((index, uuid), samples)| { + let mean = |values: Vec| { + (!values.is_empty()) + .then(|| f64::from(values.iter().sum::()) / values.len() as f64) + }; + let utilization = samples + .iter() + .filter_map(|s| s.utilization_pct) + .collect::>(); + let memory = samples + .iter() + .filter_map(|s| s.memory_utilization_pct) + .collect::>(); + let reasons = samples + .iter() + .filter_map(|s| s.clock_event_reasons) + .fold(0, |union, bits| union | bits); + let generation = samples.iter().filter_map(|s| s.pcie_generation); + let width = samples.iter().filter_map(|s| s.pcie_width); + let pcie = generation + .clone() + .map(|(current, _)| current) + .min() + .zip(generation.map(|(_, max)| max).max()) + .zip( + width + .clone() + .map(|(current, _)| current) + .min() + .zip(width.map(|(_, max)| max).max()), + ) + .map( + |((generation_min, generation_max), (width_min, width_max))| PcieLink { + generation_min, + generation_max, + width_min, + width_max, + }, + ); + GpuDeviceHealth { + index, + uuid: uuid.to_owned(), + samples: samples.len() as u64, + utilization_max_pct: utilization.iter().copied().max(), + utilization_mean_pct: mean(utilization), + memory_utilization_mean_pct: mean(memory), + clock_event_reasons: CLOCK_EVENT_REASONS + .iter() + .filter(|(bit, _)| reasons & bit != 0) + .map(|(_, name)| (*name).to_owned()) + .collect(), + temperature_max_c: samples.iter().filter_map(|s| s.temperature_c).max(), + power_max_w: samples + .iter() + .filter_map(|s| s.power_w) + .fold(None, |max: Option, value| { + Some(max.map_or(value, |max| max.max(value))) + }), + pcie, + } + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::{parse_row, summarize}; + + const ROW: &str = "0, GPU-a, 37, 12, 0x0000000000000004, 71, 95.5, 3, 4, 8, 16"; + + #[test] + fn parses_rows_including_unavailable_fields() { + let sample = parse_row(ROW).unwrap(); + assert_eq!(sample.utilization_pct, Some(37)); + assert_eq!(sample.clock_event_reasons, Some(0x4)); + assert_eq!(sample.pcie_width, Some((8, 16))); + let partial = + parse_row("1, GPU-b, 0, 0, [N/A], 40, [Not Supported], 4, 4, 16, 16").unwrap(); + assert_eq!(partial.clock_event_reasons, None); + assert_eq!(partial.power_w, None); + assert!(parse_row("0, GPU-a, high, 0, 0x0, 1, 1, 1, 1, 1, 1").is_err()); + assert!(parse_row("0, GPU-a, 1").is_err()); + } + + #[test] + fn summarizes_rounds_per_device() { + let rounds = vec![ + vec![parse_row(ROW).unwrap()], + vec![ + parse_row("0, GPU-a, 63, 20, 0x0000000000000041, 74, 101.0, 4, 4, 16, 16").unwrap(), + ], + ]; + let health = summarize(&rounds); + assert_eq!(health.len(), 1); + let device = &health[0]; + assert_eq!(device.samples, 2); + assert_eq!(device.utilization_mean_pct, Some(50.0)); + assert_eq!(device.utilization_max_pct, Some(63)); + assert_eq!( + device.clock_event_reasons, + ["gpu_idle", "sw_power_cap", "hw_thermal_slowdown"] + ); + assert_eq!(device.temperature_max_c, Some(74)); + assert_eq!(device.power_max_w, Some(101.0)); + let pcie = device.pcie.as_ref().unwrap(); + assert_eq!( + (pcie.generation_min, pcie.width_min, pcie.width_max), + (3, 8, 16) + ); + } +} diff --git a/xprobe/core/src/lib.rs b/xprobe/core/src/lib.rs index 1f6520c..64ebd12 100644 --- a/xprobe/core/src/lib.rs +++ b/xprobe/core/src/lib.rs @@ -1,14 +1,17 @@ //! Measurement orchestration and domain logic for xprobe. +pub mod cgroup_cpu; pub mod cpu_sampling; pub mod cpu_sampling_analysis; pub mod cupti_compat; pub mod discover; pub mod doctor; pub mod gpu_aggregate; +pub mod gpu_health; pub mod inject; pub mod inspect; pub mod resolve; pub mod syscall_aggregate; +pub mod thread_time; mod usdt; pub mod validate; diff --git a/xprobe/core/src/thread_time.rs b/xprobe/core/src/thread_time.rs new file mode 100644 index 0000000..05e5306 --- /dev/null +++ b/xprobe/core/src/thread_time.rs @@ -0,0 +1,277 @@ +//! Per-thread on-CPU, runqueue, and blocked time from procfs `schedstat`. + +use std::{collections::BTreeMap, fs, io, path::Path}; + +use xprobe_protocol::{ThreadTime, ThreadTimeBreakdown}; + +use crate::inspect::InspectError; + +/// One thread's cumulative scheduler counters at a point in time. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ThreadCounters { + pub tid: u32, + /// procfs start time in clock ticks; distinguishes a reused TID. + pub start_time: u64, + pub name: String, + pub cpu_ns: u64, + pub runqueue_ns: u64, +} + +/// Read every current thread of `pid`. +/// +/// Threads that exit while being read are skipped; the caller's target +/// identity check covers the process itself. +/// +/// # Errors +/// +/// Returns [`InspectError`] when the task directory cannot be listed or a +/// present thread has malformed or unreadable counters. +pub fn snapshot(pid: u32) -> Result, InspectError> { + let task_dir = format!("/proc/{pid}/task"); + let entries = fs::read_dir(&task_dir).map_err(|source| match source.kind() { + io::ErrorKind::NotFound => InspectError::TargetExited { pid }, + io::ErrorKind::PermissionDenied => InspectError::PermissionDenied { + pid, + path: task_dir.clone().into(), + }, + _ => InspectError::Io { + path: task_dir.clone().into(), + source, + }, + })?; + let mut threads = Vec::new(); + for entry in entries { + let entry = entry.map_err(|source| InspectError::Io { + path: task_dir.clone().into(), + source, + })?; + let Some(tid) = entry + .file_name() + .to_str() + .and_then(|name| name.parse().ok()) + else { + continue; + }; + match read_thread(&entry.path(), tid) { + Ok(Some(thread)) => threads.push(thread), + Ok(None) => {} + Err(error) => return Err(error), + } + } + threads.sort_by_key(|thread| thread.tid); + Ok(threads) +} + +fn read_thread(dir: &Path, tid: u32) -> Result, InspectError> { + let read = |entry: &str| -> Result, InspectError> { + let path = dir.join(entry); + match fs::read_to_string(&path) { + Ok(text) => Ok(Some(text)), + Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(None), + Err(source) => Err(InspectError::Io { path, source }), + } + }; + let (Some(schedstat), Some(stat), Some(comm)) = + (read("schedstat")?, read("stat")?, read("comm")?) + else { + return Ok(None); + }; + let (cpu_ns, runqueue_ns) = parse_schedstat(&schedstat) + .ok_or_else(|| malformed(dir, "schedstat", "expected three integer fields"))?; + let start_time = + parse_start_time(&stat).ok_or_else(|| malformed(dir, "stat", "missing start time"))?; + Ok(Some(ThreadCounters { + tid, + start_time, + name: comm.trim_end().to_owned(), + cpu_ns, + runqueue_ns, + })) +} + +fn malformed(dir: &Path, entry: &str, reason: &str) -> InspectError { + InspectError::Io { + path: dir.join(entry), + source: io::Error::new(io::ErrorKind::InvalidData, reason.to_owned()), + } +} + +/// Parse `cpu_ns runqueue_ns timeslices` from a `schedstat` line. +fn parse_schedstat(text: &str) -> Option<(u64, u64)> { + let mut fields = text.split_whitespace(); + let cpu = fields.next()?.parse().ok()?; + let runqueue = fields.next()?.parse().ok()?; + fields.next()?.parse::().ok()?; + fields.next().is_none().then_some((cpu, runqueue)) +} + +/// Field 22 of `/proc//task//stat`, after the parenthesized name. +fn parse_start_time(stat: &str) -> Option { + stat.get(stat.rfind(')')? + 1..)? + .split_whitespace() + .nth(19)? + .parse() + .ok() +} + +/// Split each whole-window thread's time into on-CPU, runqueue, and blocked. +/// +/// A thread counts as whole-window only when the same TID and start time +/// appear in both snapshots; others are counted as started or exited. +#[must_use] +#[allow(clippy::cast_precision_loss)] +pub fn breakdown( + before: &[ThreadCounters], + after: &[ThreadCounters], + window_ns: u64, + top: usize, +) -> ThreadTimeBreakdown { + let key = |thread: &ThreadCounters| (thread.tid, thread.start_time); + let earlier = before + .iter() + .map(|thread| (key(thread), thread)) + .collect::>(); + let later = after + .iter() + .map(|thread| (key(thread), thread)) + .collect::>(); + let mut threads = Vec::new(); + for (thread_key, end) in &later { + let Some(start) = earlier.get(thread_key) else { + continue; + }; + let on_cpu_ns = end.cpu_ns.saturating_sub(start.cpu_ns).min(window_ns); + let runqueue_ns = end + .runqueue_ns + .saturating_sub(start.runqueue_ns) + .min(window_ns - on_cpu_ns); + let blocked_ns = window_ns - on_cpu_ns - runqueue_ns; + let share = |value: u64| { + if window_ns == 0 { + 0.0 + } else { + value as f64 / window_ns as f64 + } + }; + threads.push(ThreadTime { + tid: end.tid, + name: end.name.clone(), + on_cpu_ns, + runqueue_ns, + blocked_ns, + on_cpu_share: share(on_cpu_ns), + runqueue_share: share(runqueue_ns), + blocked_share: share(blocked_ns), + }); + } + let observed = threads.len() as u64; + let started = later + .keys() + .filter(|key| !earlier.contains_key(key)) + .count() as u64; + let exited = earlier + .keys() + .filter(|key| !later.contains_key(key)) + .count() as u64; + let on_cpu_ns = threads.iter().map(|thread| thread.on_cpu_ns).sum(); + let runqueue_ns = threads.iter().map(|thread| thread.runqueue_ns).sum(); + let blocked_ns = threads.iter().map(|thread| thread.blocked_ns).sum(); + threads.sort_by(|left, right| { + (right.on_cpu_ns + right.runqueue_ns) + .cmp(&(left.on_cpu_ns + left.runqueue_ns)) + .then(left.tid.cmp(&right.tid)) + }); + threads.truncate(top); + ThreadTimeBreakdown { + window_ns, + threads_observed: observed, + threads_started: started, + threads_exited: exited, + on_cpu_ns, + runqueue_ns, + blocked_ns, + top_threads: threads, + } +} + +#[cfg(test)] +mod tests { + use super::{ThreadCounters, breakdown, parse_schedstat, parse_start_time, snapshot}; + + fn thread(tid: u32, start_time: u64, cpu_ns: u64, runqueue_ns: u64) -> ThreadCounters { + ThreadCounters { + tid, + start_time, + name: format!("t{tid}"), + cpu_ns, + runqueue_ns, + } + } + + #[test] + fn parses_schedstat_and_stat_fields() { + assert_eq!( + parse_schedstat("23392057514 158397840 51596\n"), + Some((23_392_057_514, 158_397_840)) + ); + assert_eq!(parse_schedstat("1 2"), None); + assert_eq!(parse_schedstat("1 2 3 4"), None); + let stat = "42 (a) b) S 1 42 42 0 -1 4194560 1 0 0 0 5 6 0 0 20 0 1 0 98765 1 2"; + assert_eq!(parse_start_time(stat), Some(98_765)); + } + + #[test] + fn splits_whole_window_threads_and_counts_churn() { + let before = [ + thread(1, 10, 1_000, 0), + thread(2, 10, 0, 0), + thread(3, 10, 0, 0), + thread(4, 10, 0, 0), + ]; + let after = [ + thread(1, 10, 701_000, 100_000), + thread(2, 10, 100_000, 0), + thread(3, 20, 5, 0), + thread(5, 30, 5, 0), + ]; + let split = breakdown(&before, &after, 1_000_000, 1); + assert_eq!(split.threads_observed, 2); + assert_eq!(split.threads_started, 2); + assert_eq!(split.threads_exited, 2); + assert_eq!(split.on_cpu_ns, 800_000); + assert_eq!(split.runqueue_ns, 100_000); + assert_eq!(split.blocked_ns, 1_100_000); + assert_eq!(split.top_threads.len(), 1); + let busiest = &split.top_threads[0]; + assert_eq!( + ( + busiest.tid, + busiest.on_cpu_ns, + busiest.runqueue_ns, + busiest.blocked_ns + ), + (1, 700_000, 100_000, 200_000) + ); + assert!((busiest.blocked_share - 0.2).abs() < 1e-9); + } + + #[test] + fn clamps_counter_deltas_to_the_window() { + let split = breakdown(&[thread(1, 1, 0, 0)], &[thread(1, 1, 900, 900)], 1_000, 4); + let only = &split.top_threads[0]; + assert_eq!( + (only.on_cpu_ns, only.runqueue_ns, only.blocked_ns), + (900, 100, 0) + ); + } + + #[test] + fn reads_the_current_process() { + let threads = snapshot(std::process::id()).expect("own threads are readable"); + assert!( + threads + .iter() + .any(|thread| thread.tid == std::process::id()) + ); + } +} diff --git a/xprobe/protocol/src/lib.rs b/xprobe/protocol/src/lib.rs index c2cabd8..59884bd 100644 --- a/xprobe/protocol/src/lib.rs +++ b/xprobe/protocol/src/lib.rs @@ -12,6 +12,7 @@ mod process; mod resolve; pub mod schema; mod syscall_aggregate; +mod triage; mod validate; mod version; @@ -48,6 +49,9 @@ pub use syscall_aggregate::{ SyscallAggregateInventory, SyscallAggregateRequirements, SyscallAggregateResult, SyscallAggregateSpec, SyscallAggregateValidationResult, }; +pub use triage::{ + CgroupCpuThrottling, CgroupVersion, GpuDeviceHealth, PcieLink, ThreadTime, ThreadTimeBreakdown, +}; pub use validate::{ AgentActivation, EndpointSource, PolicyRecommendation, PolicyRecommendationReason, ResolvedCudaSelector, ResolvedLinuxSelector, ValidatedEndpoint, ValidationIssue, diff --git a/xprobe/protocol/src/triage.rs b/xprobe/protocol/src/triage.rs new file mode 100644 index 0000000..2bc3378 --- /dev/null +++ b/xprobe/protocol/src/triage.rs @@ -0,0 +1,85 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// Per-thread time split over the triage window from procfs `schedstat`. +/// +/// `blocked_ns` is window time neither running nor runnable. It includes +/// voluntary sleep and idle waiting, so read it per thread rather than as a +/// process total. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ThreadTimeBreakdown { + pub window_ns: u64, + /// Threads alive for the whole window; totals cover only these threads. + pub threads_observed: u64, + pub threads_started: u64, + pub threads_exited: u64, + pub on_cpu_ns: u64, + pub runqueue_ns: u64, + pub blocked_ns: u64, + /// Busiest whole-window threads by on-CPU plus runqueue time. + pub top_threads: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ThreadTime { + pub tid: u32, + pub name: String, + pub on_cpu_ns: u64, + pub runqueue_ns: u64, + pub blocked_ns: u64, + pub on_cpu_share: f64, + pub runqueue_share: f64, + pub blocked_share: f64, +} + +/// CFS bandwidth throttling of the target's cgroup over the triage window. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CgroupCpuThrottling { + pub version: CgroupVersion, + pub cgroup_path: String, + /// CPU quota divided by period; `null` means no CPU limit. + pub quota_cpus: Option, + pub periods: u64, + pub throttled_periods: u64, + pub throttled_ns: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CgroupVersion { + V1, + V2, +} + +/// NVML device health sampled through the triage window. +/// +/// Utilization is the driver's sampled share of time with at least one kernel +/// running; it is an estimate, not an exact busy-interval union. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct GpuDeviceHealth { + pub index: u32, + pub uuid: String, + pub samples: u64, + pub utilization_mean_pct: Option, + pub utilization_max_pct: Option, + pub memory_utilization_mean_pct: Option, + /// Union of active NVML clock event reasons across samples. + pub clock_event_reasons: Vec, + pub temperature_max_c: Option, + pub power_max_w: Option, + pub pcie: Option, +} + +/// Narrowest `PCIe` link observed against the link maximum. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct PcieLink { + pub generation_min: u32, + pub generation_max: u32, + pub width_min: u32, + pub width_max: u32, +} From cb2f45559782db2bc0cbb962146747a2a97cc162 Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 14 Nov 2026 05:31:04 +0800 Subject: [PATCH 05/15] =?UTF-8?q?=E2=9C=A8=20feat:=20rank=20triage=20findi?= =?UTF-8?q?ngs=20from=20window=20evidence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the schema 3.0 TriageResult contract and deterministic rules that turn thread time, cgroup throttling, CPU hotspots, syscall totals, and GPU health into ranked findings with an estimated window share and a ready MeasurementSpec or environment action. --- xprobe/core/src/lib.rs | 1 + xprobe/core/src/triage.rs | 665 ++++++++++++++++++++++++++++++++++ xprobe/protocol/src/lib.rs | 2 + xprobe/protocol/src/triage.rs | 86 +++++ 4 files changed, 754 insertions(+) create mode 100644 xprobe/core/src/triage.rs diff --git a/xprobe/core/src/lib.rs b/xprobe/core/src/lib.rs index 64ebd12..ddf66bd 100644 --- a/xprobe/core/src/lib.rs +++ b/xprobe/core/src/lib.rs @@ -13,5 +13,6 @@ pub mod inspect; pub mod resolve; pub mod syscall_aggregate; pub mod thread_time; +pub mod triage; mod usdt; pub mod validate; diff --git a/xprobe/core/src/triage.rs b/xprobe/core/src/triage.rs new file mode 100644 index 0000000..a51ac17 --- /dev/null +++ b/xprobe/core/src/triage.rs @@ -0,0 +1,665 @@ +//! Deterministic triage findings from one bounded window of evidence. +//! +//! Every rule reads already-collected summaries, estimates the share of the +//! window it accounts for, and proposes the next exact measurement or an +//! environment action. Shares are estimates; findings never claim causality. + +use xprobe_protocol::{ + AggregateActivity, AggregateGroup, CgroupCpuThrottling, CpuHotspot, GpuDeviceHealth, + MatchPolicy, MeasurementMode, MeasurementSpec, SchemaVersion, SyscallAggregateGroup, + TargetIdentity, ThreadTimeBreakdown, TriageFinding, TriageNextKind, TriageNextStep, +}; + +/// Throttled time at or above this share of the window is reported. +pub const THROTTLED_SHARE: f64 = 0.02; +/// Runnable-but-waiting share of the busiest thread that is reported. +pub const RUNQUEUE_SHARE: f64 = 0.10; +/// On-CPU share of the busiest thread before hotspots are attributed. +pub const ON_CPU_SHARE: f64 = 0.30; +/// Minimum exclusive sample share for a hotspot finding. +pub const HOTSPOT_SHARE: f64 = 0.05; +/// Blocked share of the busiest thread that is reported. +pub const BLOCKED_SHARE: f64 = 0.50; +/// Summed syscall duration share of the window that is reported. +pub const SYSCALL_SHARE: f64 = 0.10; +/// Mean NVML utilization below which a GPU-using target is underutilized. +pub const GPU_BUSY_PCT: f64 = 50.0; +/// Samples requested by proposed exact measurements. +pub const NEXT_SAMPLES: u64 = 100; + +const SLOWDOWN_REASONS: [&str; 5] = [ + "sw_power_cap", + "hw_slowdown", + "sw_thermal_slowdown", + "hw_thermal_slowdown", + "hw_power_brake_slowdown", +]; + +/// Collected triage evidence. Absent sources are `None` or empty. +#[derive(Debug, Clone, Copy)] +pub struct TriageEvidence<'a> { + pub target: &'a TargetIdentity, + pub threads: &'a ThreadTimeBreakdown, + pub cgroup: Option<&'a CgroupCpuThrottling>, + pub cpu_hotspots: &'a [CpuHotspot], + pub syscalls: &'a [SyscallAggregateGroup], + pub gpu_devices: &'a [GpuDeviceHealth], + /// Present when CUDA activity was collected for the target. + pub gpu_activity: Option<&'a [AggregateGroup]>, +} + +/// Evaluate every rule and rank findings by estimated impact. +#[must_use] +pub fn findings(evidence: &TriageEvidence<'_>) -> Vec { + let mut found = Vec::new(); + cgroup_throttled(evidence, &mut found); + thread_rules(evidence, &mut found); + syscall_time(evidence, &mut found); + gpu_rules(evidence, &mut found); + found.sort_by(|left, right| { + right + .impact_share + .total_cmp(&left.impact_share) + .then_with(|| left.code.cmp(&right.code)) + }); + for (rank, finding) in found.iter_mut().enumerate() { + finding.rank = u32::try_from(rank + 1).unwrap_or(u32::MAX); + } + found +} + +#[allow(clippy::cast_precision_loss)] +fn window_share(value_ns: u64, window_ns: u64) -> f64 { + if window_ns == 0 { + 0.0 + } else { + (value_ns as f64 / window_ns as f64).min(1.0) + } +} + +fn percent(share: f64) -> String { + format!("{:.0}%", share * 100.0) +} + +fn finding( + code: &str, + impact_share: f64, + summary: String, + next: Option, +) -> TriageFinding { + TriageFinding { + rank: 0, + code: code.to_owned(), + impact_share: impact_share.clamp(0.0, 1.0), + summary, + next, + } +} + +fn environment(description: String) -> TriageNextStep { + TriageNextStep { + kind: TriageNextKind::Environment, + description, + spec: None, + } +} + +fn measure( + target: &TargetIdentity, + name: &str, + from: &str, + to: &str, + policy: MatchPolicy, + description: String, +) -> TriageNextStep { + TriageNextStep { + kind: TriageNextKind::Measure, + description, + spec: Some(MeasurementSpec { + schema_version: SchemaVersion::current(), + name: Some(name.to_owned()), + target: target.clone(), + start_selector: from.to_owned(), + end_selector: to.to_owned(), + match_policy: policy, + samples: Some(NEXT_SAMPLES), + duration_ms: None, + timeout_ms: 30_000, + max_events: Some(NEXT_SAMPLES * 4), + measurement_mode: MeasurementMode::Exact, + max_groups: None, + }), + } +} + +fn cgroup_throttled(evidence: &TriageEvidence<'_>, found: &mut Vec) { + let Some(cgroup) = evidence.cgroup else { + return; + }; + let share = window_share(cgroup.throttled_ns, evidence.threads.window_ns); + if cgroup.throttled_periods == 0 || share < THROTTLED_SHARE { + return; + } + let quota = cgroup.quota_cpus.map_or_else( + || "its CPU quota".to_owned(), + |cpus| format!("a {cpus:.2}-CPU quota"), + ); + found.push(finding( + "CGROUP_THROTTLED", + share, + format!( + "cgroup {} was throttled by {quota} for {} of the window ({} periods)", + cgroup.cgroup_path, + percent(share), + cgroup.throttled_periods + ), + Some(environment( + "raise the CPU limit or reduce parallelism before profiling code".to_owned(), + )), + )); +} + +fn thread_rules(evidence: &TriageEvidence<'_>, found: &mut Vec) { + let Some(busiest) = evidence.threads.top_threads.first() else { + return; + }; + if busiest.runqueue_share >= RUNQUEUE_SHARE { + found.push(finding( + "RUNQUEUE_WAIT", + busiest.runqueue_share, + format!( + "busiest thread {} ({}) was runnable but waiting for a CPU {} of the window", + busiest.tid, + busiest.name, + percent(busiest.runqueue_share) + ), + Some(environment( + "check CPU oversubscription, affinity, and competing workloads on the host" + .to_owned(), + )), + )); + } + if busiest.on_cpu_share >= ON_CPU_SHARE { + let hotspot = evidence + .cpu_hotspots + .iter() + .filter(|hotspot| hotspot.exclusive_proportion >= HOTSPOT_SHARE) + .max_by(|left, right| { + left.exclusive_proportion + .total_cmp(&right.exclusive_proportion) + }); + if let Some(hotspot) = hotspot { + let symbol = hotspot + .frame + .symbol + .as_deref() + .unwrap_or("an unresolved frame"); + let impact = busiest.on_cpu_share * hotspot.exclusive_proportion; + let next = match (&hotspot.entry_selector_hint, &hotspot.return_selector_hint) { + (Some(entry), Some(exit)) => Some(measure( + evidence.target, + "hotspot_duration", + entry, + exit, + MatchPolicy::StackNested, + format!("measure each call of {symbol}"), + )), + _ => None, + }; + found.push(finding( + "CPU_HOTSPOT", + impact, + format!( + "busiest thread {} was on CPU {} of the window; {symbol} holds {} of sampled stacks exclusively (sampled estimate)", + busiest.tid, + percent(busiest.on_cpu_share), + percent(hotspot.exclusive_proportion) + ), + next, + )); + } + } + if busiest.blocked_share >= BLOCKED_SHARE { + let next = top_syscall(evidence).and_then(|(group, share)| { + (share >= SYSCALL_SHARE) + .then(|| syscall_next(evidence, group)) + .flatten() + }); + found.push(finding( + "THREAD_BLOCKED", + busiest.blocked_share, + format!( + "busiest thread {} ({}) was blocked {} of the window; where it waits needs off-CPU stacks, which this triage does not collect", + busiest.tid, + busiest.name, + percent(busiest.blocked_share) + ), + next, + )); + } +} + +fn top_syscall<'a>(evidence: &TriageEvidence<'a>) -> Option<(&'a SyscallAggregateGroup, f64)> { + evidence + .syscalls + .iter() + .max_by_key(|group| group.duration_ns.total) + .map(|group| { + ( + group, + window_share(group.duration_ns.total, evidence.threads.window_ns), + ) + }) +} + +fn syscall_next( + evidence: &TriageEvidence<'_>, + group: &SyscallAggregateGroup, +) -> Option { + let (Some(entry), Some(exit)) = (&group.entry_selector_hint, &group.exit_selector_hint) else { + return None; + }; + let name = group + .syscall_name + .clone() + .unwrap_or_else(|| format!("syscall {}", group.syscall_number)); + Some(measure( + evidence.target, + "syscall_duration", + entry, + exit, + MatchPolicy::Exact, + format!("measure each {name} call"), + )) +} + +fn syscall_time(evidence: &TriageEvidence<'_>, found: &mut Vec) { + let Some((group, share)) = top_syscall(evidence) else { + return; + }; + if share < SYSCALL_SHARE { + return; + } + let name = group + .syscall_name + .clone() + .unwrap_or_else(|| format!("syscall {}", group.syscall_number)); + found.push(finding( + "SYSCALL_TIME", + share, + format!( + "{name} accumulated {} of the window across {} calls ({} errors), summed over threads", + percent(share), + group.count, + group.errors + ), + syscall_next(evidence, group), + )); +} + +#[allow(clippy::cast_precision_loss)] +fn gpu_rules(evidence: &TriageEvidence<'_>, found: &mut Vec) { + let activity = evidence.gpu_activity; + for device in evidence.gpu_devices { + let Some(mean) = device.utilization_mean_pct else { + continue; + }; + let loaded = device.utilization_max_pct.is_some_and(|max| max > 0); + let uses_gpu = activity.is_some_and(|groups| !groups.is_empty()); + if uses_gpu && mean < GPU_BUSY_PCT { + found.push(finding( + "GPU_UNDERUTILIZED", + 1.0 - mean / 100.0, + format!( + "GPU {} was busy {mean:.0}% of sampled time (device-wide NVML estimate) while the target submitted work; CPU-side or launch-bound time likely limits it", + device.index + ), + Some(measure( + evidence.target, + "launch_to_kernel", + "cuda:runtime_api:cudaLaunchKernel:exit", + "cuda:kernel_start", + MatchPolicy::Exact, + "measure launch-to-kernel latency for each launch".to_owned(), + )), + )); + } + if uses_gpu && mean >= GPU_BUSY_PCT { + kernel_dominant(evidence, device, mean, activity.unwrap_or_default(), found); + } + let slowdowns = device + .clock_event_reasons + .iter() + .filter(|reason| SLOWDOWN_REASONS.contains(&reason.as_str())) + .cloned() + .collect::>(); + if loaded && !slowdowns.is_empty() { + found.push(finding( + "GPU_CLOCK_LIMITED", + mean / 100.0, + format!( + "GPU {} reported clock slowdowns under load: {}", + device.index, + slowdowns.join(", ") + ), + Some(environment( + "check cooling, power limits, and clock policy".to_owned(), + )), + )); + } + if let Some(link) = device.pcie.as_ref().filter(|link| { + loaded && (link.width_min < link.width_max || link.generation_min < link.generation_max) + }) { + found.push(finding( + "PCIE_LINK_REDUCED", + 0.0, + format!( + "GPU {} PCIe link ran as low as gen {} x{} of gen {} x{}; idle power saving also lowers it, so compare against transfer measurements", + device.index, + link.generation_min, + link.width_min, + link.generation_max, + link.width_max + ), + None, + )); + } + } +} + +#[allow(clippy::cast_precision_loss)] +fn kernel_dominant( + evidence: &TriageEvidence<'_>, + device: &GpuDeviceHealth, + mean: f64, + groups: &[AggregateGroup], + found: &mut Vec, +) { + let kernels = groups + .iter() + .filter(|group| group.activity == AggregateActivity::Kernel) + .collect::>(); + let total = kernels + .iter() + .map(|group| group.duration_ns.total) + .sum::(); + let Some(top) = kernels.iter().max_by_key(|group| group.duration_ns.total) else { + return; + }; + if total == 0 { + return; + } + let kernel_share = top.duration_ns.total as f64 / total as f64; + let name = top.name.as_deref().unwrap_or("an unnamed kernel"); + found.push(finding( + "GPU_KERNEL_DOMINANT", + mean / 100.0 * kernel_share, + format!( + "GPU {} was busy {mean:.0}% of sampled time; {name} is {} of summed kernel time over {} launches", + device.index, + percent(kernel_share), + top.count + ), + Some(measure( + evidence.target, + "kernel_duration", + &top.start_selector_hint, + &top.end_selector_hint, + MatchPolicy::Exact, + format!("measure each {name} execution"), + )), + )); +} + +#[cfg(test)] +mod tests { + use xprobe_protocol::{ + AggregateActivity, AggregateDuration, AggregateGroup, CgroupCpuThrottling, CgroupVersion, + CpuFrameLanguage, CpuHotspot, CpuStackFrame, GpuDeviceHealth, MatchPolicy, PcieLink, + SyscallAggregateGroup, TargetIdentity, ThreadTime, ThreadTimeBreakdown, TriageNextKind, + }; + + use super::{TriageEvidence, findings}; + + const WINDOW: u64 = 1_000_000_000; + + fn thread(on_cpu: f64, runqueue: f64) -> ThreadTime { + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let ns = |share: f64| (share * 1e9) as u64; + ThreadTime { + tid: 7, + name: "worker".to_owned(), + on_cpu_ns: ns(on_cpu), + runqueue_ns: ns(runqueue), + blocked_ns: ns(1.0 - on_cpu - runqueue), + on_cpu_share: on_cpu, + runqueue_share: runqueue, + blocked_share: 1.0 - on_cpu - runqueue, + } + } + + fn threads(busiest: ThreadTime) -> ThreadTimeBreakdown { + ThreadTimeBreakdown { + window_ns: WINDOW, + threads_observed: 1, + threads_started: 0, + threads_exited: 0, + on_cpu_ns: busiest.on_cpu_ns, + runqueue_ns: busiest.runqueue_ns, + blocked_ns: busiest.blocked_ns, + top_threads: vec![busiest], + } + } + + fn hotspot(symbol: &str, exclusive: f64) -> CpuHotspot { + CpuHotspot { + frame: CpuStackFrame { + address: 0x1000, + module_path: Some("/srv/app".to_owned()), + build_id: None, + file_offset: Some(0x1000), + symbol: Some(symbol.to_owned()), + symbol_offset: Some(0), + language: CpuFrameLanguage::Native, + source_path: None, + line: None, + }, + inclusive_samples: 10, + exclusive_samples: 10, + inclusive_proportion: exclusive, + exclusive_proportion: exclusive, + entry_selector_hint: Some(format!("uprobe:/srv/app:{symbol}:entry")), + return_selector_hint: Some(format!("uprobe:/srv/app:{symbol}:return")), + } + } + + fn gpu(mean: f64, reasons: &[&str]) -> GpuDeviceHealth { + GpuDeviceHealth { + index: 0, + uuid: "GPU-a".to_owned(), + samples: 4, + utilization_mean_pct: Some(mean), + utilization_max_pct: Some(90), + memory_utilization_mean_pct: Some(10.0), + clock_event_reasons: reasons.iter().map(|reason| (*reason).to_owned()).collect(), + temperature_max_c: Some(70), + power_max_w: Some(80.0), + pcie: Some(PcieLink { + generation_min: 4, + generation_max: 4, + width_min: 16, + width_max: 16, + }), + } + } + + fn kernel(name: &str, total: u64) -> AggregateGroup { + AggregateGroup { + activity: AggregateActivity::Kernel, + name: Some(name.to_owned()), + name_complete: Some(true), + device_id: Some(0), + memcpy_kind: None, + start_selector_hint: format!("cuda:kernel_start:name~^{name}$"), + end_selector_hint: format!("cuda:kernel_end:name~^{name}$"), + count: 10, + duration_ns: AggregateDuration { + min: 1, + mean: 1.0, + max: 1, + total, + }, + total_bytes: None, + } + } + + fn target() -> TargetIdentity { + TargetIdentity { + pid: 42, + process_start_time: 9, + } + } + + #[test] + fn cpu_bound_target_points_at_its_hotspot() { + let target = target(); + let threads = threads(thread(0.9, 0.02)); + let hotspots = [hotspot("parse", 0.4), hotspot("hash", 0.03)]; + let evidence = TriageEvidence { + target: &target, + threads: &threads, + cgroup: None, + cpu_hotspots: &hotspots, + syscalls: &[], + gpu_devices: &[], + gpu_activity: None, + }; + let found = findings(&evidence); + assert_eq!(found.len(), 1); + assert_eq!(found[0].code, "CPU_HOTSPOT"); + assert_eq!(found[0].rank, 1); + assert!((found[0].impact_share - 0.36).abs() < 1e-9); + let spec = found[0].next.as_ref().unwrap().spec.as_ref().unwrap(); + assert_eq!(spec.start_selector, "uprobe:/srv/app:parse:entry"); + assert_eq!(spec.match_policy, MatchPolicy::StackNested); + assert_eq!(spec.target, target); + } + + #[test] + fn ranks_throttling_runqueue_and_blocked_syscalls() { + let target = target(); + let threads = threads(thread(0.05, 0.15)); + let cgroup = CgroupCpuThrottling { + version: CgroupVersion::V2, + cgroup_path: "/kubepods/x".to_owned(), + quota_cpus: Some(0.5), + periods: 10, + throttled_periods: 6, + throttled_ns: 300_000_000, + }; + let syscalls = [SyscallAggregateGroup { + syscall_number: 202, + syscall_name: Some("futex".to_owned()), + count: 900, + errors: 3, + duration_ns: AggregateDuration { + min: 1, + mean: 1.0, + max: 1, + total: 600_000_000, + }, + entry_selector_hint: Some("syscall:futex:entry".to_owned()), + exit_selector_hint: Some("syscall:futex:exit".to_owned()), + }]; + let evidence = TriageEvidence { + target: &target, + threads: &threads, + cgroup: Some(&cgroup), + cpu_hotspots: &[], + syscalls: &syscalls, + gpu_devices: &[], + gpu_activity: None, + }; + let codes = findings(&evidence) + .into_iter() + .map(|finding| (finding.code, finding.next.map(|next| next.kind))) + .collect::>(); + assert_eq!( + codes, + [ + ("THREAD_BLOCKED".to_owned(), Some(TriageNextKind::Measure)), + ("SYSCALL_TIME".to_owned(), Some(TriageNextKind::Measure)), + ( + "CGROUP_THROTTLED".to_owned(), + Some(TriageNextKind::Environment) + ), + ( + "RUNQUEUE_WAIT".to_owned(), + Some(TriageNextKind::Environment) + ), + ] + ); + } + + #[test] + fn gpu_rules_need_target_activity_and_load() { + let target = target(); + let threads = threads(thread(0.1, 0.0)); + let idle_gpu = [gpu(20.0, &["gpu_idle"])]; + let kernels = [kernel("gemm", 900), kernel("relu", 100)]; + let base = TriageEvidence { + target: &target, + threads: &threads, + cgroup: None, + cpu_hotspots: &[], + syscalls: &[], + gpu_devices: &idle_gpu, + gpu_activity: None, + }; + assert!( + findings(&base) + .iter() + .all(|finding| !finding.code.starts_with("GPU")) + ); + + let underfed = TriageEvidence { + gpu_activity: Some(&kernels), + ..base + }; + let found = findings(&underfed); + assert!( + found + .iter() + .any(|finding| finding.code == "GPU_UNDERUTILIZED") + ); + + let busy_gpu = [gpu(90.0, &["sw_thermal_slowdown"])]; + let busy = TriageEvidence { + gpu_devices: &busy_gpu, + gpu_activity: Some(&kernels), + ..base + }; + let found = findings(&busy); + let dominant = found + .iter() + .find(|finding| finding.code == "GPU_KERNEL_DOMINANT") + .unwrap(); + assert!((dominant.impact_share - 0.81).abs() < 1e-9); + assert_eq!( + dominant + .next + .as_ref() + .unwrap() + .spec + .as_ref() + .unwrap() + .start_selector, + "cuda:kernel_start:name~^gemm$" + ); + assert!( + found + .iter() + .any(|finding| finding.code == "GPU_CLOCK_LIMITED") + ); + } +} diff --git a/xprobe/protocol/src/lib.rs b/xprobe/protocol/src/lib.rs index 59884bd..c18bc0d 100644 --- a/xprobe/protocol/src/lib.rs +++ b/xprobe/protocol/src/lib.rs @@ -51,6 +51,8 @@ pub use syscall_aggregate::{ }; pub use triage::{ CgroupCpuThrottling, CgroupVersion, GpuDeviceHealth, PcieLink, ThreadTime, ThreadTimeBreakdown, + TriageFinding, TriageNextKind, TriageNextStep, TriageResult, TriageSource, TriageSourceKind, + TriageSourceStatus, }; pub use validate::{ AgentActivation, EndpointSource, PolicyRecommendation, PolicyRecommendationReason, diff --git a/xprobe/protocol/src/triage.rs b/xprobe/protocol/src/triage.rs index 2bc3378..bb5fdc6 100644 --- a/xprobe/protocol/src/triage.rs +++ b/xprobe/protocol/src/triage.rs @@ -83,3 +83,89 @@ pub struct PcieLink { pub width_min: u32, pub width_max: u32, } + +/// One bounded triage window over a live process: where time went, and the +/// next exact measurement to run. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct TriageResult { + pub schema_version: crate::SchemaVersion3, + pub ok: bool, + pub session_id: String, + pub status: crate::SessionStatus, + pub target: crate::TargetIdentity, + pub window_ms: u64, + /// Ranked by estimated share of the window each finding accounts for. + pub findings: Vec, + pub threads: ThreadTimeBreakdown, + pub cgroup: Option, + pub cpu_hotspots: Vec, + pub syscalls: Vec, + pub gpu_devices: Vec, + pub gpu_activity: Vec, + pub sources: Vec, + /// Questions this triage cannot answer; do not infer them from silence. + pub not_covered: Vec, + #[serde(default)] + pub warnings: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct TriageFinding { + pub rank: u32, + pub code: String, + /// Estimated share of the window this finding accounts for, in [0, 1]. + pub impact_share: f64, + pub summary: String, + pub next: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct TriageNextStep { + pub kind: TriageNextKind, + pub description: String, + /// Ready to write to a file and run with `measure --spec`. + pub spec: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum TriageNextKind { + Measure, + Environment, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct TriageSource { + pub source: TriageSourceKind, + pub status: TriageSourceStatus, + pub reason: Option, + /// Full inventory result for this source, when one was written. + pub artifact: Option, +} + +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum TriageSourceKind { + ThreadTime, + Cgroup, + CpuSample, + SyscallAggregate, + GpuHealth, + GpuActivity, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum TriageSourceStatus { + Collected, + /// The capability is absent or not permitted on this host or target. + Unavailable, + /// Not applicable to this target, such as GPU activity without CUDA. + Skipped, +} From 170340a08c2e7b80ca9a0e4d7767d6f1e551ce2c Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 14 Nov 2026 06:02:02 +0800 Subject: [PATCH 06/15] =?UTF-8?q?=E2=9A=A1=20perf:=20symbolize=20only=20sa?= =?UTF-8?q?mpled=20CPU=20frames?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CPU sample analysis read every executable mapping and demangled every text symbol before resolving a single frame. On a PyTorch CUDA process that maps gigabytes of libtorch, cuBLAS, and cuDNN, a 2 s sample took 159 s to analyze. Resolve only modules that contain retained sampled addresses, and demangle only the symbols that contain those offsets. Lookup still uses the full text-symbol table, so zero-size symbols keep their next-symbol bound. The same 2 s sample now completes in 4.7 s. --- xprobe/core/src/cpu_sampling_analysis.rs | 115 ++++++++++++++++++----- 1 file changed, 94 insertions(+), 21 deletions(-) diff --git a/xprobe/core/src/cpu_sampling_analysis.rs b/xprobe/core/src/cpu_sampling_analysis.rs index 13758c7..515b108 100644 --- a/xprobe/core/src/cpu_sampling_analysis.rs +++ b/xprobe/core/src/cpu_sampling_analysis.rs @@ -137,17 +137,26 @@ struct MapRegion { path: PathBuf, } -#[derive(Debug, Clone)] -struct ModuleSymbol { +/// One text symbol borrowed from a parsed ELF image. +#[derive(Debug, Clone, Copy)] +struct RawSymbol<'a> { file_offset: u64, size: u64, + name: &'a str, +} + +/// The symbol containing one sampled file offset, demangled once. +#[derive(Debug, Clone)] +struct ResolvedSymbol { + file_offset: u64, name: String, } +/// A mapped module, resolved only at the file offsets that were sampled. #[derive(Debug, Clone)] struct Module { build_id: Option, - symbols: Vec, + symbols: BTreeMap, } #[derive(Debug, Clone)] @@ -178,9 +187,13 @@ pub fn analyze( capture: &CpuCaptureData, ) -> Result { inspect::verify_target(&report.target)?; - let mut symbolizer = Symbolizer::load(report, capture.python_status)?; let (raw_groups, group_capacity_reached) = aggregate_raw_stacks(&capture.samples, spec.max_groups); + let sampled = raw_groups + .iter() + .flat_map(|(addresses, _)| addresses.iter().copied()) + .collect::>(); + let mut symbolizer = Symbolizer::load(report, capture.python_status, &sampled)?; let grouped_samples = raw_groups.iter().map(|(_, count)| count).sum::(); let sample_denominator = capture.observed_samples.max(1); let stack_groups = raw_groups @@ -274,9 +287,13 @@ pub fn analyze( } impl Symbolizer { + /// Load symbols only for modules and offsets that appear in `sampled`. + /// Large runtimes map gigabytes of libraries; reading and demangling every + /// symbol of every mapping dominated analysis time. fn load( report: &ProcessReport, python_status: PythonSymbolizationStatus, + sampled: &BTreeSet, ) -> Result { let maps_path = PathBuf::from(format!("/proc/{}/maps", report.target.pid)); let maps_text = fs::read_to_string(&maps_path).map_err(|source| CpuAnalysisError::Io { @@ -286,12 +303,20 @@ impl Symbolizer { let maps = parse_executable_maps(&maps_text, &maps_path)?; let mut modules = BTreeMap::new(); let mut warnings = Vec::new(); - for path in maps - .iter() - .map(|region| ®ion.path) - .collect::>() - { - match load_module(path) { + let mut offsets = BTreeMap::<&PathBuf, BTreeSet>::new(); + for address in sampled { + if let Some(mapping) = maps + .iter() + .find(|mapping| *address >= mapping.start && *address < mapping.end) + { + offsets + .entry(&mapping.path) + .or_default() + .insert(mapping.file_offset + (address - mapping.start)); + } + } + for (path, offsets) in offsets { + match load_module(path, &offsets) { Ok(module) => { modules.insert(path.clone(), module); } @@ -334,7 +359,7 @@ impl Symbolizer { }; let file_offset = mapping.file_offset + (address - mapping.start); let module = self.modules.get(&mapping.path); - let symbol = module.and_then(|module| find_module_symbol(&module.symbols, file_offset)); + let symbol = module.and_then(|module| module.symbols.get(&file_offset)); CpuStackFrame { address, module_path: Some(mapping.path.to_string_lossy().into_owned()), @@ -569,14 +594,14 @@ fn parse_hex(value: &str, path: &Path, reason: &str) -> Result Result { +fn load_module(path: &Path, offsets: &BTreeSet) -> Result { let bytes = fs::read(path).map_err(|error| error.to_string())?; let file = object::File::parse(bytes.as_slice()).map_err(|error| error.to_string())?; let build_id = file .build_id() .map_err(|error| error.to_string())? .map(hex_encode); - let mut symbols = BTreeMap::::new(); + let mut table = BTreeMap::>::new(); for symbol in file.dynamic_symbols().chain(file.symbols()) { if !symbol.is_definition() || symbol.kind() != SymbolKind::Text { continue; @@ -587,20 +612,35 @@ fn load_module(path: &Path) -> Result { let Some(file_offset) = virtual_address_to_file_offset(&file, symbol.address()) else { continue; }; - let name = demangle_cpp(name).unwrap_or_else(|| name.to_owned()); - symbols + table .entry(file_offset) .and_modify(|current| current.size = current.size.max(symbol.size())) - .or_insert(ModuleSymbol { + .or_insert(RawSymbol { file_offset, size: symbol.size(), name, }); } - Ok(Module { - build_id, - symbols: symbols.into_values().collect(), - }) + let table = table.into_values().collect::>(); + let mut demangled = BTreeMap::::new(); + let mut symbols = BTreeMap::new(); + for &offset in offsets { + let Some(symbol) = find_module_symbol(&table, offset) else { + continue; + }; + let name = demangled + .entry(symbol.file_offset) + .or_insert_with(|| demangle_cpp(symbol.name).unwrap_or_else(|| symbol.name.to_owned())) + .clone(); + symbols.insert( + offset, + ResolvedSymbol { + file_offset: symbol.file_offset, + name, + }, + ); + } + Ok(Module { build_id, symbols }) } fn virtual_address_to_file_offset(file: &object::File<'_>, address: u64) -> Option { @@ -611,7 +651,10 @@ fn virtual_address_to_file_offset(file: &object::File<'_>, address: u64) -> Opti }) } -fn find_module_symbol(symbols: &[ModuleSymbol], offset: u64) -> Option<&ModuleSymbol> { +fn find_module_symbol<'s, 'a>( + symbols: &'s [RawSymbol<'a>], + offset: u64, +) -> Option<&'s RawSymbol<'a>> { let index = symbols.partition_point(|symbol| symbol.file_offset <= offset); let symbol = symbols.get(index.checked_sub(1)?)?; let next_start = symbols.get(index).map(|next| next.file_offset); @@ -760,6 +803,36 @@ fn ratio(numerator: u64, denominator: u64) -> f64 { mod tests { use super::*; + #[inline(never)] + fn xprobe_lazy_symbol_marker() -> u64 { + std::hint::black_box(7) + } + + #[test] + fn resolves_only_sampled_offsets_against_the_full_symbol_table() { + let marker: fn() -> u64 = xprobe_lazy_symbol_marker; + let address = marker as usize as u64; + let maps_path = PathBuf::from("/proc/self/maps"); + let maps = parse_executable_maps(&fs::read_to_string(&maps_path).unwrap(), &maps_path) + .expect("own maps parse"); + let mapping = maps + .iter() + .find(|mapping| address >= mapping.start && address < mapping.end) + .expect("marker is mapped executable code"); + let offset = mapping.file_offset + (address - mapping.start); + let module = load_module(&mapping.path, &BTreeSet::from([offset, offset + 1])) + .expect("test binary loads"); + assert_eq!(module.symbols.len(), 2); + let symbol = &module.symbols[&offset]; + assert!( + symbol.name.contains("xprobe_lazy_symbol_marker"), + "{}", + symbol.name + ); + assert_eq!(symbol.file_offset, offset); + assert_eq!(module.symbols[&(offset + 1)].file_offset, offset); + } + #[test] fn keeps_the_heaviest_stack_groups_deterministically() { let samples = vec![ From 8e050f261ceedd18e51495b39e406a900be60d7e Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 14 Nov 2026 06:02:02 +0800 Subject: [PATCH 07/15] =?UTF-8?q?=E2=9C=A8=20feat:=20triage=20a=20live=20p?= =?UTF-8?q?rocess=20in=20one=20bounded=20window?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `measure --pid P --triage --duration-ms N` answers where time went before any selector is known. In one window it concurrently collects per-thread on-CPU, runqueue, and blocked time; cgroup throttling; CPU samples; syscall totals; NVML device health; and, for CUDA targets, a kernel aggregate. Deterministic rules rank findings by an estimated window share and attach an environment action or a ready MeasurementSpec whose record capacity is sized from the observed rate. Full inventories stay in private per-source artifacts. Sources that are not permitted, absent, or over capacity are reported as unavailable with a reason; identity, cleanup, and internal failures still fail the command. `not_covered` states what triage cannot answer. `just test-triage-live` runs both commands against a PyTorch CUDA workload: triage, then the first proposed spec. --- Cargo.lock | 1 + docs/architecture.md | 12 +- docs/cli-contract.md | 26 + justfile | 4 + schemas/triage-result.schema.json | 963 +++++++++++++++++++++++++++++ tests/integration/test_triage.py | 246 ++++++++ xprobe/cli/Cargo.toml | 1 + xprobe/cli/src/args.rs | 20 +- xprobe/cli/src/inventory.rs | 107 +++- xprobe/cli/src/main.rs | 24 +- xprobe/cli/src/render.rs | 56 ++ xprobe/cli/src/triage.rs | 459 ++++++++++++++ xprobe/core/src/triage.rs | 240 ++++++- xprobe/protocol/src/schema.rs | 6 +- xprobe/protocol/src/triage.rs | 7 +- xprobe/protocol/tests/contracts.rs | 95 ++- 16 files changed, 2212 insertions(+), 55 deletions(-) create mode 100644 schemas/triage-result.schema.json create mode 100644 tests/integration/test_triage.py create mode 100644 xprobe/cli/src/triage.rs diff --git a/Cargo.lock b/Cargo.lock index b846c10..f3a45d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -735,6 +735,7 @@ name = "xprobe-cli" version = "0.5.2" dependencies = [ "clap", + "serde", "serde_json", "xprobe-collector", "xprobe-core", diff --git a/docs/architecture.md b/docs/architecture.md index 00c694f..940d2bd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -59,7 +59,10 @@ the resolver checks exported dynamic symbols first, falls back to the complete table, and returns both the attachable mangled name and readable signature. CPU stack analysis resolves sampled instruction pointers through the target's -mapped ELF load segments and symbol tables. CPython frames are resolved +mapped ELF load segments and symbol tables. Only modules containing retained +sampled addresses are read, and only the symbols containing those offsets are +demangled; lookup still uses each module's complete text-symbol table, so +zero-size symbols keep their next-symbol bound. CPython frames are resolved independently from `/tmp/perf-.map` when the interpreter exposes an active perf trampoline. Native coverage remains usable when Python semantic symbols are inactive or unsupported. CPython GC resolution parses `.note.stapsdt` from @@ -149,6 +152,13 @@ syscall aggregation, and GPU aggregation have separate result schemas because their quality and capacity meanings differ. Their groups are hypotheses for a later validated exact capture, not event timelines that can be re-correlated. +Triage runs these inventories concurrently in one foreground `measure`, between +two procfs `schedstat` and cgroup `cpu.stat` snapshots, while sampling NVML +through `nvidia-smi`. Deterministic rules in core turn the summaries into ranked +findings with a proposed exact `MeasurementSpec`. Each full inventory stays in +its own artifact with its own contract; the triage result references them +instead of merging them into one timeline. + Broad GPU inventory uses aggregate mode. The Agent updates a `--max-groups`-bounded table for matching kernel, memcpy, or memset activity and returns only final count/duration/byte summaries. Exact diff --git a/docs/cli-contract.md b/docs/cli-contract.md index 347d1b1..90e6b3d 100644 --- a/docs/cli-contract.md +++ b/docs/cli-contract.md @@ -227,6 +227,12 @@ xprobe measure --spec measurement.json \ --json --non-interactive --no-color ``` +Same-window triage of an unknown live workload: + +```bash +xprobe measure --pid 4242 --triage --duration-ms 2000 +``` + Broad GPU activity inventory: ```bash @@ -252,6 +258,26 @@ ELF and `/tmp/perf-.map` Python symbols are resolved independently. from being mislabeled as Python semantic coverage. Lost samples, truncated stacks, thread coverage, and capacity pressure remain visible. +`--triage` is live-only, requires `--duration-ms`, and emits +`schemas/triage-result.schema.json` (schema `3.0`). Within one window it +concurrently collects procfs per-thread on-CPU, runqueue, and blocked time; +cgroup CPU throttling; a CPU sample inventory; a syscall aggregate; NVML device +health (utilization, clock event reasons, temperature, power, PCIe link); and, +when the target has loaded libcuda, a CUDA kernel aggregate, which may inject +the CUPTI Agent under the same rules as `measure`. The result keeps the top +five hotspots, syscall groups, and kernel groups inline and writes each full +inventory to a private artifact referenced from `sources`. `findings` are +deterministic and ranked by an estimated share of the window; a finding whose +impact cannot be bounded has `impact_share: null` and ranks last. Each finding +carries an environment action or a ready `MeasurementSpec` for `measure +--spec`, with record capacity sized from the observed rate. A source that is not +permitted, absent, or exceeds its bounded capacity is `unavailable` with a +reason and a `TRIAGE_SOURCE_UNAVAILABLE` warning; target exit or reuse, cleanup, +and internal failures fail the command. `not_covered` lists what triage cannot +answer, including where blocked threads wait. Blocked time includes idle +waiting, so read it per thread. NVML utilization is a device-wide sampled +estimate. + `--syscall-aggregate` is live-only and requires `--duration-ms`. PID-filtered raw syscall entry/exit programs retain at most `--max-inflight` thread starts and `--max-groups` syscall groups in BPF maps, then emit one bounded summary. diff --git a/justfile b/justfile index 526555c..3c1f98d 100644 --- a/justfile +++ b/justfile @@ -85,6 +85,9 @@ test-pytorch-live: build test-pytorch-cuda-live: build if [[ -n "${PYTORCH_ENV:-}" ]]; then python3 tests/integration/test_pytorch_cuda.py --image "{{cuda12_devel_image}}" --pytorch-env "${PYTORCH_ENV}"; else python3 tests/integration/test_pytorch_cuda.py --image "{{pytorch_image}}"; fi +test-triage-live: build + if [[ -n "${PYTORCH_ENV:-}" ]]; then python3 tests/integration/test_triage.py --image "{{cuda12_devel_image}}" --pytorch-env "${PYTORCH_ENV}"; else python3 tests/integration/test_triage.py --image "{{pytorch_image}}"; fi + test-nvtx-live: build python3 tests/integration/test_nvtx.py --image "{{cuda13_devel_image}}" @@ -106,6 +109,7 @@ test-release-live: just test-multisource-live just test-pytorch-live just test-pytorch-cuda-live + just test-triage-live just benchmark-gpu just benchmark-aggregate just benchmark-multiprocess diff --git a/schemas/triage-result.schema.json b/schemas/triage-result.schema.json new file mode 100644 index 0000000..ab78fc7 --- /dev/null +++ b/schemas/triage-result.schema.json @@ -0,0 +1,963 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "TriageResult", + "description": "One bounded triage window over a live process: where time went, and the\nnext exact measurement to run.", + "type": "object", + "properties": { + "cgroup": { + "anyOf": [ + { + "$ref": "#/$defs/CgroupCpuThrottling" + }, + { + "type": "null" + } + ] + }, + "cpu_hotspots": { + "type": "array", + "items": { + "$ref": "#/$defs/CpuHotspot" + } + }, + "findings": { + "description": "Ranked by estimated window share; findings without an estimate last.", + "type": "array", + "items": { + "$ref": "#/$defs/TriageFinding" + } + }, + "gpu_activity": { + "type": "array", + "items": { + "$ref": "#/$defs/AggregateGroup" + } + }, + "gpu_devices": { + "type": "array", + "items": { + "$ref": "#/$defs/GpuDeviceHealth" + } + }, + "not_covered": { + "description": "Questions this triage cannot answer; do not infer them from silence.", + "type": "array", + "items": { + "type": "string" + } + }, + "ok": { + "type": "boolean" + }, + "schema_version": { + "$ref": "#/$defs/SchemaVersion3" + }, + "session_id": { + "type": "string" + }, + "sources": { + "type": "array", + "items": { + "$ref": "#/$defs/TriageSource" + } + }, + "status": { + "$ref": "#/$defs/SessionStatus" + }, + "syscalls": { + "type": "array", + "items": { + "$ref": "#/$defs/SyscallAggregateGroup" + } + }, + "target": { + "$ref": "#/$defs/TargetIdentity" + }, + "threads": { + "$ref": "#/$defs/ThreadTimeBreakdown" + }, + "warnings": { + "type": "array", + "default": [], + "items": { + "$ref": "#/$defs/Warning" + } + }, + "window_ms": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "schema_version", + "ok", + "session_id", + "status", + "target", + "window_ms", + "findings", + "threads", + "cpu_hotspots", + "syscalls", + "gpu_devices", + "gpu_activity", + "sources", + "not_covered" + ], + "$defs": { + "AggregateActivity": { + "type": "string", + "enum": [ + "kernel", + "memcpy", + "memset" + ] + }, + "AggregateDuration": { + "type": "object", + "properties": { + "max": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "mean": { + "type": "number", + "format": "double" + }, + "min": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "total": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "min", + "mean", + "max", + "total" + ] + }, + "AggregateGroup": { + "type": "object", + "properties": { + "activity": { + "$ref": "#/$defs/AggregateActivity" + }, + "count": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "device_id": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0 + }, + "duration_ns": { + "$ref": "#/$defs/AggregateDuration" + }, + "end_selector_hint": { + "type": "string" + }, + "memcpy_kind": { + "anyOf": [ + { + "$ref": "#/$defs/MemcpyKind" + }, + { + "type": "null" + } + ] + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "name_complete": { + "type": [ + "boolean", + "null" + ] + }, + "start_selector_hint": { + "type": "string" + }, + "total_bytes": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "activity", + "start_selector_hint", + "end_selector_hint", + "count", + "duration_ns" + ] + }, + "CgroupCpuThrottling": { + "description": "CFS bandwidth throttling of the target's cgroup over the triage window.", + "type": "object", + "properties": { + "cgroup_path": { + "type": "string" + }, + "periods": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "quota_cpus": { + "description": "CPU quota divided by period; `null` means no CPU limit.", + "type": [ + "number", + "null" + ], + "format": "double" + }, + "throttled_ns": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "throttled_periods": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "version": { + "$ref": "#/$defs/CgroupVersion" + } + }, + "additionalProperties": false, + "required": [ + "version", + "cgroup_path", + "periods", + "throttled_periods", + "throttled_ns" + ] + }, + "CgroupVersion": { + "type": "string", + "enum": [ + "v1", + "v2" + ] + }, + "CpuFrameLanguage": { + "type": "string", + "enum": [ + "native", + "python", + "unknown" + ] + }, + "CpuHotspot": { + "type": "object", + "properties": { + "entry_selector_hint": { + "type": [ + "string", + "null" + ] + }, + "exclusive_proportion": { + "type": "number", + "format": "double" + }, + "exclusive_samples": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "frame": { + "$ref": "#/$defs/CpuStackFrame" + }, + "inclusive_proportion": { + "type": "number", + "format": "double" + }, + "inclusive_samples": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "return_selector_hint": { + "type": [ + "string", + "null" + ] + } + }, + "additionalProperties": false, + "required": [ + "frame", + "inclusive_samples", + "exclusive_samples", + "inclusive_proportion", + "exclusive_proportion" + ] + }, + "CpuStackFrame": { + "type": "object", + "properties": { + "address": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "build_id": { + "type": [ + "string", + "null" + ] + }, + "file_offset": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "language": { + "$ref": "#/$defs/CpuFrameLanguage" + }, + "line": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0 + }, + "module_path": { + "type": [ + "string", + "null" + ] + }, + "source_path": { + "type": [ + "string", + "null" + ] + }, + "symbol": { + "type": [ + "string", + "null" + ] + }, + "symbol_offset": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "address", + "language" + ] + }, + "GpuDeviceHealth": { + "description": "NVML device health sampled through the triage window.\n\nUtilization is the driver's sampled share of time with at least one kernel\nrunning; it is an estimate, not an exact busy-interval union.", + "type": "object", + "properties": { + "clock_event_reasons": { + "description": "Union of active NVML clock event reasons across samples.", + "type": "array", + "items": { + "type": "string" + } + }, + "index": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "memory_utilization_mean_pct": { + "type": [ + "number", + "null" + ], + "format": "double" + }, + "pcie": { + "anyOf": [ + { + "$ref": "#/$defs/PcieLink" + }, + { + "type": "null" + } + ] + }, + "power_max_w": { + "type": [ + "number", + "null" + ], + "format": "double" + }, + "samples": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "temperature_max_c": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0 + }, + "utilization_max_pct": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0 + }, + "utilization_mean_pct": { + "type": [ + "number", + "null" + ], + "format": "double" + }, + "uuid": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "index", + "uuid", + "samples", + "clock_event_reasons" + ] + }, + "MatchPolicy": { + "type": "string", + "enum": [ + "exact", + "first_after", + "nearest", + "stack_nested", + "stream_order" + ] + }, + "MeasurementMode": { + "type": "string", + "enum": [ + "exact", + "aggregate" + ] + }, + "MeasurementSpec": { + "type": "object", + "properties": { + "duration_ms": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "end_selector": { + "type": "string" + }, + "match_policy": { + "$ref": "#/$defs/MatchPolicy" + }, + "max_events": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "default": null, + "minimum": 0 + }, + "max_groups": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "default": null, + "minimum": 0 + }, + "measurement_mode": { + "$ref": "#/$defs/MeasurementMode", + "default": "exact" + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "samples": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "schema_version": { + "$ref": "#/$defs/SchemaVersion" + }, + "start_selector": { + "type": "string" + }, + "target": { + "$ref": "#/$defs/TargetIdentity" + }, + "timeout_ms": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "schema_version", + "target", + "start_selector", + "end_selector", + "match_policy", + "timeout_ms" + ] + }, + "MemcpyKind": { + "type": "string", + "enum": [ + "HtoD", + "DtoH", + "DtoD", + "HtoH", + "PtoP", + "unknown" + ] + }, + "PcieLink": { + "description": "Narrowest `PCIe` link observed against the link maximum.", + "type": "object", + "properties": { + "generation_max": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "generation_min": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "width_max": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "width_min": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "generation_min", + "generation_max", + "width_min", + "width_max" + ] + }, + "SchemaVersion": { + "type": "string", + "enum": [ + "2.0" + ] + }, + "SchemaVersion3": { + "description": "Version of records whose shape changed in xprobe 0.6.\n\nRecords keep their own version so unchanged contracts, such as Event JSONL\nartifacts, remain readable at `2.0`.", + "type": "string", + "enum": [ + "3.0" + ] + }, + "SessionStatus": { + "type": "string", + "enum": [ + "completed", + "timed_out", + "cancelled", + "failed" + ] + }, + "SyscallAggregateGroup": { + "type": "object", + "properties": { + "count": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "duration_ns": { + "$ref": "#/$defs/AggregateDuration" + }, + "entry_selector_hint": { + "type": [ + "string", + "null" + ] + }, + "errors": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "exit_selector_hint": { + "type": [ + "string", + "null" + ] + }, + "syscall_name": { + "type": [ + "string", + "null" + ] + }, + "syscall_number": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "syscall_number", + "count", + "errors", + "duration_ns" + ] + }, + "TargetIdentity": { + "type": "object", + "properties": { + "pid": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "process_start_time": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "pid", + "process_start_time" + ] + }, + "ThreadTime": { + "type": "object", + "properties": { + "blocked_ns": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "blocked_share": { + "type": "number", + "format": "double" + }, + "name": { + "type": "string" + }, + "on_cpu_ns": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "on_cpu_share": { + "type": "number", + "format": "double" + }, + "runqueue_ns": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "runqueue_share": { + "type": "number", + "format": "double" + }, + "tid": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "tid", + "name", + "on_cpu_ns", + "runqueue_ns", + "blocked_ns", + "on_cpu_share", + "runqueue_share", + "blocked_share" + ] + }, + "ThreadTimeBreakdown": { + "description": "Per-thread time split over the triage window from procfs `schedstat`.\n\n`blocked_ns` is window time neither running nor runnable. It includes\nvoluntary sleep and idle waiting, so read it per thread rather than as a\nprocess total.", + "type": "object", + "properties": { + "blocked_ns": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "on_cpu_ns": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "runqueue_ns": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "threads_exited": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "threads_observed": { + "description": "Threads alive for the whole window; totals cover only these threads.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "threads_started": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "top_threads": { + "description": "Busiest whole-window threads by on-CPU plus runqueue time.", + "type": "array", + "items": { + "$ref": "#/$defs/ThreadTime" + } + }, + "window_ns": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "window_ns", + "threads_observed", + "threads_started", + "threads_exited", + "on_cpu_ns", + "runqueue_ns", + "blocked_ns", + "top_threads" + ] + }, + "TriageFinding": { + "type": "object", + "properties": { + "code": { + "type": "string" + }, + "impact_share": { + "description": "Estimated share of the window this finding accounts for, in [0, 1];\n`null` when the evidence cannot bound the impact.", + "type": [ + "number", + "null" + ], + "format": "double" + }, + "next": { + "anyOf": [ + { + "$ref": "#/$defs/TriageNextStep" + }, + { + "type": "null" + } + ] + }, + "rank": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "summary": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "rank", + "code", + "summary" + ] + }, + "TriageNextKind": { + "type": "string", + "enum": [ + "measure", + "environment" + ] + }, + "TriageNextStep": { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "kind": { + "$ref": "#/$defs/TriageNextKind" + }, + "spec": { + "description": "Ready to write to a file and run with `measure --spec`.", + "anyOf": [ + { + "$ref": "#/$defs/MeasurementSpec" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "required": [ + "kind", + "description" + ] + }, + "TriageSource": { + "type": "object", + "properties": { + "artifact": { + "description": "Full inventory result for this source, when one was written.", + "type": [ + "string", + "null" + ] + }, + "reason": { + "type": [ + "string", + "null" + ] + }, + "source": { + "$ref": "#/$defs/TriageSourceKind" + }, + "status": { + "$ref": "#/$defs/TriageSourceStatus" + } + }, + "additionalProperties": false, + "required": [ + "source", + "status" + ] + }, + "TriageSourceKind": { + "type": "string", + "enum": [ + "thread_time", + "cgroup", + "cpu_sample", + "syscall_aggregate", + "gpu_health", + "gpu_activity" + ] + }, + "TriageSourceStatus": { + "oneOf": [ + { + "type": "string", + "enum": [ + "collected" + ] + }, + { + "description": "The capability is absent or not permitted on this host or target.", + "type": "string", + "const": "unavailable" + }, + { + "description": "Not applicable to this target, such as GPU activity without CUDA.", + "type": "string", + "const": "skipped" + } + ] + }, + "Warning": { + "type": "object", + "properties": { + "code": { + "type": "string" + }, + "details": { + "type": "object", + "additionalProperties": true, + "default": {} + }, + "message": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "code", + "message" + ] + } + } +} diff --git a/tests/integration/test_triage.py b/tests/integration/test_triage.py new file mode 100644 index 0000000..c72f209 --- /dev/null +++ b/tests/integration/test_triage.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Live two-command flow: triage a PyTorch CUDA workload, then run its next spec.""" + +import argparse +import json +import os +import pathlib +import subprocess +import sys +import tempfile + +EXPECTED_SOURCES = { + "thread_time", + "cgroup", + "cpu_sample", + "syscall_aggregate", + "gpu_health", + "gpu_activity", +} + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--image") + parser.add_argument("--pytorch-env") + parser.add_argument("--inner", action="store_true") + parser.add_argument("--xprobe", default="/workspace/target/debug/xprobe") + args = parser.parse_args() + if args.inner: + run_inner(pathlib.Path(args.xprobe)) + else: + run_container(args) + + +def run_container(args: argparse.Namespace) -> None: + if not args.image: + raise SystemExit("--image is required") + workspace = pathlib.Path(__file__).resolve().parents[2] + python = "python3" + environment_arguments = [] + if args.pytorch_env: + python = "/opt/xprobe-pytorch/bin/python" + environment_arguments = [ + "--volume", + f"{pathlib.Path(args.pytorch_env).resolve()}:/opt/xprobe-pytorch:ro", + ] + completed = subprocess.run( + [ + "docker", + "run", + "--rm", + "--gpus", + "all", + "--cap-add", + "BPF", + "--cap-add", + "PERFMON", + "--cap-add", + "SYS_ADMIN", + "--cap-add", + "SYS_RESOURCE", + "--cap-add", + "SYS_PTRACE", + "--security-opt", + "seccomp=unconfined", + "--volume", + f"{workspace}:/workspace:ro", + *environment_arguments, + "--workdir", + "/workspace", + args.image, + python, + "/workspace/tests/integration/test_triage.py", + "--inner", + ], + check=False, + capture_output=True, + text=True, + ) + sys.stdout.write(completed.stdout) + if completed.returncode != 0: + sys.stderr.write(completed.stderr) + raise SystemExit(completed.returncode) + + +def run_inner(xprobe: pathlib.Path) -> None: + workspace = pathlib.Path("/workspace") + with tempfile.TemporaryDirectory(prefix="xprobe-triage-") as temp_dir: + temp = pathlib.Path(temp_dir) + agent = temp / "libxprobe-cupti.so" + build_agent(workspace, agent) + environment = os.environ.copy() + environment["NVTX_INJECTION64_PATH"] = str(agent) + workload = subprocess.Popen( + [sys.executable, "-u", "-c", WORKLOAD], + env=environment, + stdout=subprocess.PIPE, + text=True, + ) + try: + assert workload.stdout is not None + if not workload.stdout.readline(): + raise AssertionError("PyTorch workload exited before readiness") + triage = run_xprobe( + xprobe, + [ + "measure", + "--pid", + str(workload.pid), + "--agent", + str(agent), + "--triage", + "--duration-ms", + "2000", + ], + ) + next_spec = first_measure_spec(triage) + spec_path = temp / "next.json" + spec_path.write_text(json.dumps(next_spec)) + try: + measured = run_xprobe( + xprobe, + ["measure", "--spec", str(spec_path), "--agent", str(agent)], + ) + except AssertionError as error: + raise AssertionError( + f"next spec failed\nfindings: {json.dumps(triage['findings'], indent=1)}\n{error}" + ) from None + finally: + workload.terminate() + workload.wait(timeout=10) + + assert_triage(triage, workload.pid) + assert measured["schema_version"] == "3.0", measured + assert measured["ok"] is True, measured + assert measured["measurement"]["samples"]["matched"] > 0, measured + assert measured["evidence"]["artifact"] is not None, measured + print( + json.dumps( + { + "findings": [finding["code"] for finding in triage["findings"]], + "next_selector": next_spec["start_selector"], + "next_matched": measured["measurement"]["samples"]["matched"], + "next_max_events": next_spec["max_events"], + "next_quality": measured["quality"]["verdict"], + "ok": True, + "triage_bytes": len(json.dumps(triage)), + "window_ms": triage["window_ms"], + }, + sort_keys=True, + ) + ) + + +def assert_triage(triage: dict, pid: int) -> None: + assert triage["schema_version"] == "3.0", triage + assert triage["ok"] is True, triage + assert triage["target"]["pid"] == pid, triage + statuses = {source["source"]: source["status"] for source in triage["sources"]} + assert set(statuses) == EXPECTED_SOURCES, statuses + collected = {name for name, status in statuses.items() if status == "collected"} + assert collected >= EXPECTED_SOURCES - {"cgroup"}, statuses + for source in triage["sources"]: + if source["source"] in {"cpu_sample", "syscall_aggregate", "gpu_activity"}: + assert pathlib.Path(source["artifact"]).name.endswith(".json"), source + assert triage["threads"]["threads_observed"] > 0, triage["threads"] + assert triage["gpu_activity"], "kernel activity must be grouped" + assert triage["gpu_devices"], "NVML health must be sampled" + assert triage["findings"], "a busy CUDA workload must produce findings" + ranks = [finding["rank"] for finding in triage["findings"]] + assert ranks == list(range(1, len(ranks) + 1)), ranks + assert triage["not_covered"], triage + + +def first_measure_spec(triage: dict) -> dict: + for finding in triage["findings"]: + step = finding["next"] + if step and step["kind"] == "measure" and step["spec"]: + return step["spec"] + raise AssertionError(f"no runnable next step: {json.dumps(triage['findings'])}") + + +def build_agent(workspace: pathlib.Path, output: pathlib.Path) -> None: + subprocess.run( + [ + "gcc", + "-std=c11", + "-D_GNU_SOURCE", + "-DXPROBE_HAS_CUPTI=1", + "-fPIC", + "-shared", + "-pthread", + "-O2", + "-Wall", + "-Wextra", + "-Wpedantic", + "-Werror", + f"-I{workspace / 'cupti/include'}", + "-isystem", + "/usr/local/cuda/include", + str(workspace / "cupti/src/cupti_agent.c"), + "-L/usr/local/cuda/lib64", + "-Wl,-rpath,/usr/local/cuda/lib64", + "-lcupti", + "-o", + str(output), + ], + check=True, + ) + + +def run_xprobe(xprobe: pathlib.Path, arguments: list[str]) -> dict: + completed = subprocess.run( + [str(xprobe), *arguments], + check=False, + capture_output=True, + text=True, + ) + if completed.returncode != 0: + raise AssertionError(f"xprobe failed:\n{completed.stdout}\n{completed.stderr}") + return json.loads(completed.stdout) + + +WORKLOAD = r""" +import json +import time + +import torch + +device = torch.device("cuda") +a = torch.randn(1024, 1024, device=device) +b = torch.randn(1024, 1024, device=device) +host = torch.randn(256, 256) +torch.cuda.synchronize() +print(json.dumps({"pytorch": torch.__version__}), flush=True) +while True: + for _ in range(8): + torch.mm(a, b) + torch.cuda.synchronize() + host @ host + time.sleep(0.001) +""" + + +if __name__ == "__main__": + main() diff --git a/xprobe/cli/Cargo.toml b/xprobe/cli/Cargo.toml index 2e92b77..ac87133 100644 --- a/xprobe/cli/Cargo.toml +++ b/xprobe/cli/Cargo.toml @@ -12,6 +12,7 @@ path = "src/main.rs" [dependencies] clap.workspace = true +serde.workspace = true serde_json.workspace = true xprobe-collector.workspace = true xprobe-core.workspace = true diff --git a/xprobe/cli/src/args.rs b/xprobe/cli/src/args.rs index d98c30a..6dfb57b 100644 --- a/xprobe/cli/src/args.rs +++ b/xprobe/cli/src/args.rs @@ -288,6 +288,10 @@ pub(crate) struct ValidateArgs { } #[derive(Debug, Args)] +#[allow( + clippy::struct_excessive_bools, + reason = "mutually exclusive clap mode flags with declared conflicts" +)] pub(crate) struct MeasureArgs { /// Versioned measurement JSON for a live target. #[arg(long, conflicts_with_all = ["input", "pid", "from", "to", "match_policy", "cpu_sample", "syscall_aggregate", "samples", "duration_ms", "timeout_ms", "max_events", "max_samples", "frequency_hz", "stack_depth", "max_threads", "max_inflight", "aggregate", "max_groups", "name"])] @@ -314,13 +318,27 @@ pub(crate) struct MeasureArgs { #[arg(long)] pub(crate) events_out: Option, + /// Triage one bounded window: thread time, cgroup throttling, CPU samples, + /// syscall totals, GPU health, and CUDA kernel activity, with ranked + /// findings and the next exact measurement. Requires --pid and + /// --duration-ms. + #[arg( + long, + requires_all = ["pid", "duration_ms"], + conflicts_with_all = [ + "spec", "input", "from", "to", "match_policy", "samples", "cpu_sample", + "syscall_aggregate", "aggregate", "events_out", "format" + ] + )] + pub(crate) triage: bool, + /// Inline matched pairs: `summary` keeps rank exemplars and the artifact /// path; `full` also inlines every pair. #[arg( long, value_enum, default_value_t = EvidenceArg::Summary, - conflicts_with_all = ["cpu_sample", "syscall_aggregate", "aggregate"] + conflicts_with_all = ["cpu_sample", "syscall_aggregate", "aggregate", "triage"] )] pub(crate) evidence: EvidenceArg, diff --git a/xprobe/cli/src/inventory.rs b/xprobe/cli/src/inventory.rs index 696f21b..7fec2e6 100644 --- a/xprobe/cli/src/inventory.rs +++ b/xprobe/cli/src/inventory.rs @@ -63,19 +63,41 @@ pub(crate) fn run_cpu_sampling_spec( inspected: Option, json: bool, ) -> ExitCode { + match cpu_inventory(spec, inspected) { + Ok(result) => emit_cpu_inventory(&result, json), + Err(error) => emit_command_failure(error, json), + } +} + +pub(crate) fn cpu_inventory( + spec: &CpuSamplingSpec, + inspected: Option, +) -> Result { let report = match inspected.map_or_else(|| inspect::run(spec.target.pid), Ok) { Ok(report) => report, Err(error) => { - return emit_error(error.code(), error.to_string(), error.recoverable(), json); + return Err(CommandFailure::new( + error.code(), + error.to_string(), + error.recoverable(), + )); } }; if let Err(error) = inspect::verify_target(&spec.target) { - return emit_error(error.code(), error.to_string(), error.recoverable(), json); + return Err(CommandFailure::new( + error.code(), + error.to_string(), + error.recoverable(), + )); } let validation = match cpu_sampling::validate(&report) { Ok(validation) => validation, Err(error) => { - return emit_error(error.code(), error.to_string(), error.recoverable(), json); + return Err(CommandFailure::new( + error.code(), + error.to_string(), + error.recoverable(), + )); } }; if !validation.valid { @@ -83,23 +105,23 @@ pub(crate) fn run_cpu_sampling_spec( || "CPU sampling requirements are not satisfied".to_owned(), |issue| issue.message.clone(), ); - return emit_error(ErrorCode::PermissionDenied, message, true, json); + return Err(CommandFailure::new( + ErrorCode::PermissionDenied, + message, + true, + )); } - let request = match cpu_collector_request(spec) { - Ok(request) => request, - Err(error) => return emit_command_failure(error, json), - }; + let request = cpu_collector_request(spec)?; let capture = match cpu_collector::collect(&request) { Ok(capture) => capture, - Err(error) => return emit_command_failure(cpu_collector_failure(&error), json), + Err(error) => return Err(cpu_collector_failure(&error)), }; if capture.observed_samples == 0 { - return emit_error( + return Err(CommandFailure::new( ErrorCode::NoMatchedSamples, "CPU sampling completed without observing an on-CPU sample".to_owned(), true, - json, - ); + )); } let capture = CpuCaptureData { samples: capture @@ -128,10 +150,14 @@ pub(crate) fn run_cpu_sampling_spec( ) { Ok(result) => result, Err(error) => { - return emit_error(error.code(), error.to_string(), error.recoverable(), json); + return Err(CommandFailure::new( + error.code(), + error.to_string(), + error.recoverable(), + )); } }; - emit_cpu_inventory(&result, json) + Ok(result) } pub(crate) fn cpu_collector_request( @@ -275,19 +301,41 @@ pub(crate) fn run_syscall_aggregate_spec( inspected: Option, json: bool, ) -> ExitCode { + match syscall_inventory(spec, inspected) { + Ok(result) => emit_syscall_inventory(&result, json), + Err(error) => emit_command_failure(error, json), + } +} + +pub(crate) fn syscall_inventory( + spec: &SyscallAggregateSpec, + inspected: Option, +) -> Result { let report = match inspected.map_or_else(|| inspect::run(spec.target.pid), Ok) { Ok(report) => report, Err(error) => { - return emit_error(error.code(), error.to_string(), error.recoverable(), json); + return Err(CommandFailure::new( + error.code(), + error.to_string(), + error.recoverable(), + )); } }; if let Err(error) = inspect::verify_target(&spec.target) { - return emit_error(error.code(), error.to_string(), error.recoverable(), json); + return Err(CommandFailure::new( + error.code(), + error.to_string(), + error.recoverable(), + )); } let validation = match syscall_aggregate::validate(&report) { Ok(validation) => validation, Err(error) => { - return emit_error(error.code(), error.to_string(), error.recoverable(), json); + return Err(CommandFailure::new( + error.code(), + error.to_string(), + error.recoverable(), + )); } }; if !validation.valid { @@ -295,16 +343,21 @@ pub(crate) fn run_syscall_aggregate_spec( || "syscall aggregate requirements are not satisfied".to_owned(), |issue| issue.message.clone(), ); - return emit_error(ErrorCode::PermissionDenied, message, true, json); + return Err(CommandFailure::new( + ErrorCode::PermissionDenied, + message, + true, + )); } - let request = match syscall_collector_request(spec) { - Ok(request) => request, - Err(error) => return emit_command_failure(error, json), - }; + let request = syscall_collector_request(spec)?; let capture = match syscall_collector::collect(&request) { Ok(capture) => capture, Err(error) => { - return emit_error(error.code(), error.to_string(), error.recoverable(), json); + return Err(CommandFailure::new( + error.code(), + error.to_string(), + error.recoverable(), + )); } }; let capture = syscall_aggregate::SyscallCaptureData { @@ -334,10 +387,14 @@ pub(crate) fn run_syscall_aggregate_spec( ) { Ok(result) => result, Err(error) => { - return emit_error(error.code(), error.to_string(), error.recoverable(), json); + return Err(CommandFailure::new( + error.code(), + error.to_string(), + error.recoverable(), + )); } }; - emit_syscall_inventory(&result, json) + Ok(result) } pub(crate) fn syscall_collector_request( diff --git a/xprobe/cli/src/main.rs b/xprobe/cli/src/main.rs index 27d569c..6a93b3e 100644 --- a/xprobe/cli/src/main.rs +++ b/xprobe/cli/src/main.rs @@ -27,6 +27,7 @@ mod failure; mod inventory; mod live; mod render; +mod triage; use args::{ Cli, Command, CommonOutputArgs, DiscoverArgs, DoctorArgs, ExportFormatArg, MeasureArgs, @@ -76,6 +77,7 @@ fn run_measure(args: MeasureArgs) -> ExitCode { cupti_socket, agent, events_out, + triage, evidence, format, timeout_ms, @@ -107,6 +109,21 @@ fn run_measure(args: MeasureArgs) -> ExitCode { format, mode: evidence.mode(), }; + if triage { + let (Some(pid), Some(duration_ms)) = (pid, duration_ms) else { + unreachable!("clap requires --pid and --duration-ms with --triage"); + }; + return triage::run_triage( + &triage::TriageRequest { + pid, + duration_ms, + timeout_ms, + cupti_socket, + agent_path: agent, + }, + json, + ); + } if let Some(spec) = spec { return run_measure_spec(&spec, cupti_socket, agent, output, json); } @@ -548,6 +565,11 @@ impl EvidenceOutput<'_> { } fn default_artifact_path() -> Result { + Ok(private_artifact_dir()?.join("events.jsonl")) +} + +/// Create a new mode `0700` directory for one command's artifacts. +fn private_artifact_dir() -> Result { let nanos = SystemTime::now() .duration_since(UNIX_EPOCH) .map_or(0, |elapsed| elapsed.as_nanos()); @@ -566,7 +588,7 @@ fn default_artifact_path() -> Result { ) .with_hint("pass --events-out PATH to choose a writable artifact location") })?; - Ok(directory.join("events.jsonl")) + Ok(directory) } fn finish_measurement( diff --git a/xprobe/cli/src/render.rs b/xprobe/cli/src/render.rs index 5c54d58..cb648d9 100644 --- a/xprobe/cli/src/render.rs +++ b/xprobe/cli/src/render.rs @@ -6,6 +6,7 @@ use xprobe_protocol::MeasurementResult; use xprobe_protocol::ProcessReport; use xprobe_protocol::ResolvedProbe; use xprobe_protocol::SchemaVersion; +use xprobe_protocol::TriageResult; use xprobe_protocol::ValidationResult; pub(crate) fn print_human_report(report: &CapabilityReport) { @@ -234,3 +235,58 @@ pub(crate) const fn schema_version(version: SchemaVersion) -> &'static str { SchemaVersion::V2 => "2.0", } } + +pub(crate) fn print_triage_result(result: &TriageResult) { + println!( + "Triage of PID {} over {} ms", + result.target.pid, result.window_ms + ); + if result.findings.is_empty() { + println!("No finding crossed its threshold."); + } + for finding in &result.findings { + let impact = finding.impact_share.map_or_else( + || " ?".to_owned(), + |share| format!("{:>3.0}%", share * 100.0), + ); + println!( + "{}. [{}] {impact} {}", + finding.rank, finding.code, finding.summary + ); + if let Some(next) = &finding.next { + println!(" next: {}", next.description); + } + } + let threads = &result.threads; + println!( + "Threads: {} observed, {} started, {} exited", + threads.threads_observed, threads.threads_started, threads.threads_exited + ); + for thread in &threads.top_threads { + println!( + " {} {:<16} on-CPU {:>5.1}% runqueue {:>5.1}% blocked {:>5.1}%", + thread.tid, + thread.name, + thread.on_cpu_share * 100.0, + thread.runqueue_share * 100.0, + thread.blocked_share * 100.0 + ); + } + for source in &result.sources { + println!( + "Source {:?}: {:?}{}", + source.source, + source.status, + source + .reason + .as_deref() + .map_or_else(String::new, |reason| format!(" ({reason})")) + ); + } + for item in &result.not_covered { + println!("Not covered: {item}"); + } + for warning in &result.warnings { + println!("Warning: {}: {}", warning.code, warning.message); + } +} diff --git a/xprobe/cli/src/triage.rs b/xprobe/cli/src/triage.rs new file mode 100644 index 0000000..08f916a --- /dev/null +++ b/xprobe/cli/src/triage.rs @@ -0,0 +1,459 @@ +//! `measure --triage`: one bounded window of concurrent inventories. + +use std::{ + path::{Path, PathBuf}, + process::ExitCode, + thread, + time::{Duration, Instant}, +}; + +use serde::Serialize; +use xprobe_core::{ + cgroup_cpu, gpu_health, inspect, thread_time, + triage::{TriageEvidence, findings}, +}; +use xprobe_correlator::MeasureOptions; +use xprobe_protocol::{ + AggregateInventoryResult, CpuSampleEvent, CpuSamplingSpec, ErrorCode, MatchPolicy, + MeasurementMode, ProcessReport, SchemaVersion, SchemaVersion3, SessionStatus, + SyscallAggregateSpec, TriageResult, TriageSource, TriageSourceKind, TriageSourceStatus, + Warning, +}; + +use crate::{ + dev::write_export_file, + failure::{CommandFailure, emit_command_failure}, + inventory::{cpu_inventory, syscall_inventory}, + live::{LiveMeasureRequest, collect_live_aggregate}, + private_artifact_dir, + render::print_triage_result, +}; + +/// Entries kept inline per summary list; full results go to artifacts. +const TOP: usize = 5; +const CPU_FREQUENCY_HZ: u64 = 99; +const CPU_MAX_GROUPS: u64 = 256; +const CPU_STACK_DEPTH: u32 = 64; +const CPU_MAX_THREADS: u32 = 1024; +/// Concurrently running threads the CPU sample capacity is sized for. +const CPU_SAMPLED_THREAD_BUDGET: u64 = 64; +const SYSCALL_MAX_GROUPS: u64 = 256; +const SYSCALL_MAX_INFLIGHT: u64 = 1024; +const GPU_MAX_GROUPS: usize = 4096; +const GPU_HEALTH_INTERVAL: Duration = Duration::from_millis(250); + +const NOT_COVERED: [&str; 4] = [ + "where blocked threads wait (off-CPU stacks, lock and I/O attribution)", + "exact GPU busy intervals; NVML utilization is a device-wide sampled estimate", + "memcpy and memset activity; GPU activity covers kernels only", + "pageable versus pinned host memory in transfers", +]; + +#[derive(Debug)] +pub(crate) struct TriageRequest { + pub(crate) pid: u32, + pub(crate) duration_ms: u64, + pub(crate) timeout_ms: u64, + pub(crate) cupti_socket: Option, + pub(crate) agent_path: Option, +} + +pub(crate) fn run_triage(request: &TriageRequest, json: bool) -> ExitCode { + match triage(request) { + Ok(result) => { + if json { + println!( + "{}", + serde_json::to_string_pretty(&result).expect("triage result must serialize") + ); + } else { + print_triage_result(&result); + } + ExitCode::SUCCESS + } + Err(error) => emit_command_failure(error, json), + } +} + +fn inspect_failure(error: &inspect::InspectError) -> CommandFailure { + CommandFailure::new(error.code(), error.to_string(), error.recoverable()) +} + +/// A source either produced a value or is unavailable for a stated reason. +enum Outcome { + Collected(T), + Unavailable(String), + Skipped(String), +} + +/// Capability absence and bounded-capacity failures make one source +/// unavailable. Target identity, cleanup, and internal failures abort triage. +fn classify(result: Result) -> Result, CommandFailure> { + match result { + Ok(value) => Ok(Outcome::Collected(value)), + Err(error) => match error.code { + ErrorCode::PermissionDenied + | ErrorCode::CuptiNotAvailable + | ErrorCode::CuptiAgentNotLoaded + | ErrorCode::CudaContextNotFound + | ErrorCode::UnsupportedCudaVersion + | ErrorCode::EventRateTooHigh + | ErrorCode::EventsDropped + | ErrorCode::NoMatchedSamples + | ErrorCode::HighUnmatchedRate => Ok(Outcome::Unavailable(format!( + "{}: {}", + error.code, error.message + ))), + _ => Err(error), + }, + } +} + +#[allow(clippy::too_many_lines)] +fn triage(request: &TriageRequest) -> Result { + if request.duration_ms == 0 { + return Err(CommandFailure::new( + ErrorCode::SessionLimitExceeded, + "--triage requires a positive --duration-ms", + true, + )); + } + let report = inspect::run(request.pid).map_err(|error| inspect_failure(&error))?; + let target = report.target.clone(); + let artifacts = private_artifact_dir()?; + let cgroup = cgroup_cpu::locate(request.pid).map_err(|error| inspect_failure(&error))?; + let quota = match &cgroup { + Some(cgroup) => cgroup + .quota_cpus() + .map_err(|error| inspect_failure(&error))?, + None => None, + }; + let cgroup_before = match &cgroup { + Some(cgroup) => Some(cgroup.counters().map_err(|error| inspect_failure(&error))?), + None => None, + }; + let threads_before = + thread_time::snapshot(request.pid).map_err(|error| inspect_failure(&error))?; + let started = Instant::now(); + let window = Duration::from_millis(request.duration_ms); + + let (cpu, syscalls, gpu_activity, gpu_rounds) = thread::scope(|scope| { + let cpu = scope.spawn(|| cpu_inventory(&cpu_spec(&report, request), Some(report.clone()))); + let syscalls = scope + .spawn(|| syscall_inventory(&syscall_spec(&report, request), Some(report.clone()))); + let gpu_activity = report + .cuda + .libcuda_loaded + .then(|| scope.spawn(|| collect_live_aggregate(&gpu_request(request)))); + let gpu_rounds = scope.spawn(|| sample_gpu_health(window)); + ( + cpu.join(), + syscalls.join(), + gpu_activity.map(thread::ScopedJoinHandle::join), + gpu_rounds.join(), + ) + }); + let elapsed = started.elapsed(); + let threads_after = + thread_time::snapshot(request.pid).map_err(|error| inspect_failure(&error))?; + let cgroup_after = match &cgroup { + Some(cgroup) => Some(cgroup.counters().map_err(|error| inspect_failure(&error))?), + None => None, + }; + inspect::verify_target(&target).map_err(|error| inspect_failure(&error))?; + + let panicked = |source: &str| { + CommandFailure::new( + ErrorCode::Internal, + format!("{source} collector panicked"), + false, + ) + }; + let cpu = classify(cpu.map_err(|_| panicked("CPU sampling"))?)?; + let syscalls = classify(syscalls.map_err(|_| panicked("syscall aggregate"))?)?; + let gpu_activity = match gpu_activity { + Some(joined) => classify(joined.map_err(|_| panicked("GPU activity"))?)?, + None => Outcome::Skipped("the target has not loaded libcuda".to_owned()), + }; + let gpu_rounds = gpu_rounds.map_err(|_| panicked("GPU health"))?; + + let window_ns = u64::try_from(elapsed.as_nanos()).unwrap_or(u64::MAX); + let threads = thread_time::breakdown(&threads_before, &threads_after, window_ns, TOP); + let cgroup_throttling = cgroup + .as_ref() + .zip(cgroup_before.zip(cgroup_after)) + .map(|(cgroup, (before, after))| cgroup_cpu::throttling(cgroup, quota, before, after)); + + let mut sources = vec![ + source( + TriageSourceKind::ThreadTime, + TriageSourceStatus::Collected, + None, + None, + ), + match &cgroup_throttling { + Some(_) => source( + TriageSourceKind::Cgroup, + TriageSourceStatus::Collected, + None, + None, + ), + None => source( + TriageSourceKind::Cgroup, + TriageSourceStatus::Unavailable, + Some("no CPU cgroup controller is mounted for the target".to_owned()), + None, + ), + }, + ]; + let mut warnings = Vec::new(); + + let mut cpu_hotspots = Vec::new(); + let mut python_status = None; + match cpu { + Outcome::Collected(result) => { + python_status = Some(result.symbolization.python_status); + let path = write_artifact(&artifacts, "cpu-sample.json", &result)?; + cpu_hotspots = result.inventory.hotspots; + cpu_hotspots.sort_by(|left, right| { + right + .exclusive_proportion + .total_cmp(&left.exclusive_proportion) + }); + cpu_hotspots.truncate(TOP); + warnings.extend(result.warnings); + sources.push(source( + TriageSourceKind::CpuSample, + TriageSourceStatus::Collected, + None, + Some(path), + )); + } + outcome => sources.push(not_collected(TriageSourceKind::CpuSample, outcome)), + } + + let mut syscall_groups = Vec::new(); + match syscalls { + Outcome::Collected(result) => { + let path = write_artifact(&artifacts, "syscall-aggregate.json", &result)?; + syscall_groups = result.inventory.groups; + syscall_groups.sort_by_key(|group| std::cmp::Reverse(group.duration_ns.total)); + syscall_groups.truncate(TOP); + warnings.extend(result.warnings); + sources.push(source( + TriageSourceKind::SyscallAggregate, + TriageSourceStatus::Collected, + None, + Some(path), + )); + } + outcome => sources.push(not_collected(TriageSourceKind::SyscallAggregate, outcome)), + } + + let mut activity_groups = None; + match gpu_activity { + Outcome::Collected(result) => { + let path = write_artifact(&artifacts, "gpu-activity.json", &result)?; + let AggregateInventoryResult { + inventory, + warnings: activity_warnings, + .. + } = result; + let mut groups = inventory.groups; + groups.sort_by_key(|group| std::cmp::Reverse(group.duration_ns.total)); + groups.truncate(TOP); + activity_groups = Some(groups); + warnings.extend(activity_warnings); + sources.push(source( + TriageSourceKind::GpuActivity, + TriageSourceStatus::Collected, + None, + Some(path), + )); + } + outcome => sources.push(not_collected(TriageSourceKind::GpuActivity, outcome)), + } + + let gpu_devices = match gpu_rounds { + Ok(rounds) => { + sources.push(source( + TriageSourceKind::GpuHealth, + TriageSourceStatus::Collected, + None, + None, + )); + gpu_health::summarize(&rounds) + } + Err(reason) => { + sources.push(source( + TriageSourceKind::GpuHealth, + TriageSourceStatus::Unavailable, + Some(reason), + None, + )); + Vec::new() + } + }; + for unavailable in sources + .iter() + .filter(|source| source.status == TriageSourceStatus::Unavailable) + { + warnings.push(Warning { + code: "TRIAGE_SOURCE_UNAVAILABLE".to_owned(), + message: format!( + "{:?} evidence is missing: {}", + unavailable.source, + unavailable.reason.as_deref().unwrap_or("unknown reason") + ), + details: std::collections::BTreeMap::new(), + }); + } + sources.sort_by_key(|source| source.source); + + let found = findings(&TriageEvidence { + target: &target, + threads: &threads, + collection_window_ns: request.duration_ms.saturating_mul(1_000_000), + cgroup: cgroup_throttling.as_ref(), + cpu_hotspots: &cpu_hotspots, + python_status, + syscalls: &syscall_groups, + gpu_devices: &gpu_devices, + gpu_activity: activity_groups.as_deref(), + }); + Ok(TriageResult { + schema_version: SchemaVersion3::current(), + ok: true, + session_id: format!("xp_triage_{}", std::process::id()), + status: SessionStatus::Completed, + target, + window_ms: u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX), + findings: found, + threads, + cgroup: cgroup_throttling, + cpu_hotspots, + syscalls: syscall_groups, + gpu_devices, + gpu_activity: activity_groups.unwrap_or_default(), + sources, + not_covered: NOT_COVERED.iter().map(|item| (*item).to_owned()).collect(), + warnings, + }) +} + +fn source( + source: TriageSourceKind, + status: TriageSourceStatus, + reason: Option, + artifact: Option, +) -> TriageSource { + TriageSource { + source, + status, + reason, + artifact, + } +} + +fn not_collected(kind: TriageSourceKind, outcome: Outcome) -> TriageSource { + match outcome { + Outcome::Collected(_) => unreachable!("collected outcomes are handled by the caller"), + Outcome::Unavailable(reason) => { + source(kind, TriageSourceStatus::Unavailable, Some(reason), None) + } + Outcome::Skipped(reason) => source(kind, TriageSourceStatus::Skipped, Some(reason), None), + } +} + +fn write_artifact( + directory: &Path, + name: &str, + value: &T, +) -> Result { + let path = directory.join(name); + let bytes = serde_json::to_vec_pretty(value).map_err(|error| { + CommandFailure::new( + ErrorCode::Internal, + format!("failed to encode {name}: {error}"), + false, + ) + })?; + write_export_file(&path, &bytes).map_err(|error| { + CommandFailure::new( + ErrorCode::TraceExportFailed, + format!("failed to write {}: {error}", path.display()), + false, + ) + })?; + Ok(path.display().to_string()) +} + +fn cpu_spec(report: &ProcessReport, request: &TriageRequest) -> CpuSamplingSpec { + CpuSamplingSpec { + schema_version: SchemaVersion::current(), + name: Some("triage".to_owned()), + target: report.target.clone(), + sample_event: CpuSampleEvent::CpuClock, + frequency_hz: CPU_FREQUENCY_HZ, + duration_ms: request.duration_ms, + timeout_ms: request.timeout_ms, + max_samples: (CPU_FREQUENCY_HZ * request.duration_ms / 1000 + 1) + * CPU_SAMPLED_THREAD_BUDGET, + max_groups: CPU_MAX_GROUPS, + stack_depth: CPU_STACK_DEPTH, + max_threads: CPU_MAX_THREADS, + } +} + +fn syscall_spec(report: &ProcessReport, request: &TriageRequest) -> SyscallAggregateSpec { + SyscallAggregateSpec { + schema_version: SchemaVersion::current(), + name: Some("triage".to_owned()), + target: report.target.clone(), + duration_ms: request.duration_ms, + timeout_ms: request.timeout_ms, + max_groups: SYSCALL_MAX_GROUPS, + max_inflight: SYSCALL_MAX_INFLIGHT, + } +} + +fn gpu_request(request: &TriageRequest) -> LiveMeasureRequest { + LiveMeasureRequest { + pid: request.pid, + expected_target: None, + cupti_socket: request.cupti_socket.clone(), + agent_path: request.agent_path.clone(), + timeout: Duration::from_millis(request.timeout_ms), + match_policy_text: "exact", + mode: MeasurementMode::Aggregate, + options: MeasureOptions { + session_id: format!("xp_triage_gpu_{}", std::process::id()), + name: Some("triage".to_owned()), + start_selector: "cuda:kernel_start".to_owned(), + end_selector: "cuda:kernel_end".to_owned(), + match_policy: MatchPolicy::Exact, + samples: None, + duration: Some(Duration::from_millis(request.duration_ms)), + max_events: GPU_MAX_GROUPS, + dropped_events: 0, + }, + } +} + +/// Query NVML until the window ends. Any failed query makes the source +/// unavailable rather than silently summarizing a shorter window. +fn sample_gpu_health(window: Duration) -> Result>, String> { + let deadline = Instant::now() + window; + let mut rounds = Vec::new(); + loop { + let round = gpu_health::query() + .map_err(|error| format!("NVML query {} failed: {error}", rounds.len() + 1))?; + rounds.push(round); + let now = Instant::now(); + if now >= deadline { + break; + } + thread::sleep(GPU_HEALTH_INTERVAL.min(deadline - now)); + } + Ok(rounds) +} diff --git a/xprobe/core/src/triage.rs b/xprobe/core/src/triage.rs index a51ac17..5c7862f 100644 --- a/xprobe/core/src/triage.rs +++ b/xprobe/core/src/triage.rs @@ -6,8 +6,9 @@ use xprobe_protocol::{ AggregateActivity, AggregateGroup, CgroupCpuThrottling, CpuHotspot, GpuDeviceHealth, - MatchPolicy, MeasurementMode, MeasurementSpec, SchemaVersion, SyscallAggregateGroup, - TargetIdentity, ThreadTimeBreakdown, TriageFinding, TriageNextKind, TriageNextStep, + MatchPolicy, MeasurementMode, MeasurementSpec, PythonSymbolizationStatus, SchemaVersion, + SyscallAggregateGroup, TargetIdentity, ThreadTimeBreakdown, TriageFinding, TriageNextKind, + TriageNextStep, }; /// Throttled time at or above this share of the window is reported. @@ -26,6 +27,8 @@ pub const SYSCALL_SHARE: f64 = 0.10; pub const GPU_BUSY_PCT: f64 = 50.0; /// Samples requested by proposed exact measurements. pub const NEXT_SAMPLES: u64 = 100; +/// Upper bound for proposed record capacity; the CLI default `--max-events`. +pub const MAX_NEXT_EVENTS: u64 = 100_000; const SLOWDOWN_REASONS: [&str; 5] = [ "sw_power_cap", @@ -40,8 +43,14 @@ const SLOWDOWN_REASONS: [&str; 5] = [ pub struct TriageEvidence<'a> { pub target: &'a TargetIdentity, pub threads: &'a ThreadTimeBreakdown, + /// Configured duration of the syscall and GPU activity captures. Their + /// counts and summed durations are relative to this window, not to the + /// longer triage wall time that includes attach and teardown. + pub collection_window_ns: u64, pub cgroup: Option<&'a CgroupCpuThrottling>, pub cpu_hotspots: &'a [CpuHotspot], + /// Python perf-map status of the CPU sample, when one was collected. + pub python_status: Option, pub syscalls: &'a [SyscallAggregateGroup], pub gpu_devices: &'a [GpuDeviceHealth], /// Present when CUDA activity was collected for the target. @@ -57,10 +66,13 @@ pub fn findings(evidence: &TriageEvidence<'_>) -> Vec { syscall_time(evidence, &mut found); gpu_rules(evidence, &mut found); found.sort_by(|left, right| { - right - .impact_share - .total_cmp(&left.impact_share) - .then_with(|| left.code.cmp(&right.code)) + match (left.impact_share, right.impact_share) { + (Some(left), Some(right)) => right.total_cmp(&left), + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => std::cmp::Ordering::Equal, + } + .then_with(|| left.code.cmp(&right.code)) }); for (rank, finding) in found.iter_mut().enumerate() { finding.rank = u32::try_from(rank + 1).unwrap_or(u32::MAX); @@ -90,7 +102,18 @@ fn finding( TriageFinding { rank: 0, code: code.to_owned(), - impact_share: impact_share.clamp(0.0, 1.0), + impact_share: Some(impact_share.clamp(0.0, 1.0)), + summary, + next, + } +} + +/// A finding whose share of the window the evidence cannot bound. +fn unestimated(code: &str, summary: String, next: Option) -> TriageFinding { + TriageFinding { + rank: 0, + code: code.to_owned(), + impact_share: None, summary, next, } @@ -110,6 +133,7 @@ fn measure( from: &str, to: &str, policy: MatchPolicy, + max_events: u64, description: String, ) -> TriageNextStep { TriageNextStep { @@ -125,13 +149,43 @@ fn measure( samples: Some(NEXT_SAMPLES), duration_ms: None, timeout_ms: 30_000, - max_events: Some(NEXT_SAMPLES * 4), + max_events: Some(max_events), measurement_mode: MeasurementMode::Exact, max_groups: None, }), } } +/// Record capacity for an exact capture of an operation observed `count` +/// times in the triage window: two records per occurrence, with one second of +/// headroom because live completion is checked from periodic snapshots. +fn sized_capacity(count: u64, window_ns: u64) -> u64 { + let floor = NEXT_SAMPLES * 4; + if window_ns == 0 { + return floor; + } + let per_second = u128::from(count) * 2 * 1_000_000_000 / u128::from(window_ns); + u64::try_from(per_second) + .unwrap_or(u64::MAX) + .clamp(floor, MAX_NEXT_EVENTS) +} + +/// A readable frame name: symbol, else module and file offset. +fn frame_label(frame: &xprobe_protocol::CpuStackFrame) -> String { + if let Some(symbol) = &frame.symbol { + return symbol.clone(); + } + let module = frame + .module_path + .as_deref() + .map(|path| path.rsplit('/').next().unwrap_or(path)); + match (module, frame.file_offset) { + (Some(module), Some(offset)) => format!("unresolved {module}+{offset:#x}"), + (Some(module), None) => format!("unresolved frame in {module}"), + _ => format!("unresolved address {:#x}", frame.address), + } +} + fn cgroup_throttled(evidence: &TriageEvidence<'_>, found: &mut Vec) { let Some(cgroup) = evidence.cgroup else { return; @@ -189,11 +243,7 @@ fn thread_rules(evidence: &TriageEvidence<'_>, found: &mut Vec) { .total_cmp(&right.exclusive_proportion) }); if let Some(hotspot) = hotspot { - let symbol = hotspot - .frame - .symbol - .as_deref() - .unwrap_or("an unresolved frame"); + let symbol = frame_label(&hotspot.frame); let impact = busiest.on_cpu_share * hotspot.exclusive_proportion; let next = match (&hotspot.entry_selector_hint, &hotspot.return_selector_hint) { (Some(entry), Some(exit)) => Some(measure( @@ -202,6 +252,7 @@ fn thread_rules(evidence: &TriageEvidence<'_>, found: &mut Vec) { entry, exit, MatchPolicy::StackNested, + NEXT_SAMPLES * 4, format!("measure each call of {symbol}"), )), _ => None, @@ -217,6 +268,8 @@ fn thread_rules(evidence: &TriageEvidence<'_>, found: &mut Vec) { ), next, )); + } else { + found.push(cpu_bound(evidence, busiest.tid, busiest.on_cpu_share)); } } if busiest.blocked_share >= BLOCKED_SHARE { @@ -239,6 +292,47 @@ fn thread_rules(evidence: &TriageEvidence<'_>, found: &mut Vec) { } } +/// The busiest thread is on CPU but no single frame crosses the hotspot share. +fn cpu_bound(evidence: &TriageEvidence<'_>, tid: u32, on_cpu_share: f64) -> TriageFinding { + let frames = evidence + .cpu_hotspots + .iter() + .take(3) + .map(|hotspot| { + format!( + "{} {}", + frame_label(&hotspot.frame), + percent(hotspot.exclusive_proportion) + ) + }) + .collect::>(); + let python_inactive = matches!( + evidence.python_status, + Some(PythonSymbolizationStatus::Inactive) + ); + let next = python_inactive.then(|| { + environment( + "restart the interpreter with -X perf (CPython 3.12+) so samples resolve to Python functions" + .to_owned(), + ) + }); + finding( + "CPU_BOUND", + on_cpu_share, + format!( + "busiest thread {tid} was on CPU {} of the window, spread across frames ({}); no single frame holds {} of samples", + percent(on_cpu_share), + if frames.is_empty() { + "none sampled".to_owned() + } else { + frames.join(", ") + }, + percent(HOTSPOT_SHARE) + ), + next, + ) +} + fn top_syscall<'a>(evidence: &TriageEvidence<'a>) -> Option<(&'a SyscallAggregateGroup, f64)> { evidence .syscalls @@ -247,7 +341,7 @@ fn top_syscall<'a>(evidence: &TriageEvidence<'a>) -> Option<(&'a SyscallAggregat .map(|group| { ( group, - window_share(group.duration_ns.total, evidence.threads.window_ns), + window_share(group.duration_ns.total, evidence.collection_window_ns), ) }) } @@ -269,6 +363,7 @@ fn syscall_next( entry, exit, MatchPolicy::Exact, + sized_capacity(group.count, evidence.collection_window_ns), format!("measure each {name} call"), )) } @@ -306,6 +401,12 @@ fn gpu_rules(evidence: &TriageEvidence<'_>, found: &mut Vec) { }; let loaded = device.utilization_max_pct.is_some_and(|max| max > 0); let uses_gpu = activity.is_some_and(|groups| !groups.is_empty()); + let launches = activity + .unwrap_or_default() + .iter() + .filter(|group| group.activity == AggregateActivity::Kernel) + .map(|group| group.count) + .sum::(); if uses_gpu && mean < GPU_BUSY_PCT { found.push(finding( "GPU_UNDERUTILIZED", @@ -320,6 +421,7 @@ fn gpu_rules(evidence: &TriageEvidence<'_>, found: &mut Vec) { "cuda:runtime_api:cudaLaunchKernel:exit", "cuda:kernel_start", MatchPolicy::Exact, + sized_capacity(launches, evidence.collection_window_ns), "measure launch-to-kernel latency for each launch".to_owned(), )), )); @@ -334,9 +436,8 @@ fn gpu_rules(evidence: &TriageEvidence<'_>, found: &mut Vec) { .cloned() .collect::>(); if loaded && !slowdowns.is_empty() { - found.push(finding( + found.push(unestimated( "GPU_CLOCK_LIMITED", - mean / 100.0, format!( "GPU {} reported clock slowdowns under load: {}", device.index, @@ -350,9 +451,8 @@ fn gpu_rules(evidence: &TriageEvidence<'_>, found: &mut Vec) { if let Some(link) = device.pcie.as_ref().filter(|link| { loaded && (link.width_min < link.width_max || link.generation_min < link.generation_max) }) { - found.push(finding( + found.push(unestimated( "PCIE_LINK_REDUCED", - 0.0, format!( "GPU {} PCIe link ran as low as gen {} x{} of gen {} x{}; idle power saving also lowers it, so compare against transfer measurements", device.index, @@ -406,6 +506,7 @@ fn kernel_dominant( &top.start_selector_hint, &top.end_selector_hint, MatchPolicy::Exact, + sized_capacity(top.count, evidence.collection_window_ns), format!("measure each {name} execution"), )), )); @@ -416,7 +517,8 @@ mod tests { use xprobe_protocol::{ AggregateActivity, AggregateDuration, AggregateGroup, CgroupCpuThrottling, CgroupVersion, CpuFrameLanguage, CpuHotspot, CpuStackFrame, GpuDeviceHealth, MatchPolicy, PcieLink, - SyscallAggregateGroup, TargetIdentity, ThreadTime, ThreadTimeBreakdown, TriageNextKind, + PythonSymbolizationStatus, SyscallAggregateGroup, TargetIdentity, ThreadTime, + ThreadTimeBreakdown, TriageNextKind, }; use super::{TriageEvidence, findings}; @@ -528,8 +630,10 @@ mod tests { let evidence = TriageEvidence { target: &target, threads: &threads, + collection_window_ns: WINDOW, cgroup: None, cpu_hotspots: &hotspots, + python_status: None, syscalls: &[], gpu_devices: &[], gpu_activity: None, @@ -538,13 +642,92 @@ mod tests { assert_eq!(found.len(), 1); assert_eq!(found[0].code, "CPU_HOTSPOT"); assert_eq!(found[0].rank, 1); - assert!((found[0].impact_share - 0.36).abs() < 1e-9); + assert!((found[0].impact_share.unwrap() - 0.36).abs() < 1e-9); let spec = found[0].next.as_ref().unwrap().spec.as_ref().unwrap(); assert_eq!(spec.start_selector, "uprobe:/srv/app:parse:entry"); assert_eq!(spec.match_policy, MatchPolicy::StackNested); assert_eq!(spec.target, target); } + #[test] + fn sizes_capacity_from_observed_rate() { + assert_eq!(super::sized_capacity(10, WINDOW), 400); + assert_eq!(super::sized_capacity(3_548, 2 * WINDOW), 3_548); + assert_eq!( + super::sized_capacity(u64::MAX, WINDOW), + super::MAX_NEXT_EVENTS + ); + assert_eq!(super::sized_capacity(10, 0), 400); + } + + #[test] + fn labels_unresolved_frames_by_module_offset() { + let mut frame = hotspot("x", 0.5).frame; + frame.symbol = None; + frame.module_path = Some("/usr/lib/x86_64-linux-gnu/libcuda.so.1".to_owned()); + frame.file_offset = Some(0x1a2b); + assert_eq!(super::frame_label(&frame), "unresolved libcuda.so.1+0x1a2b"); + frame.module_path = None; + assert_eq!(super::frame_label(&frame), "unresolved address 0x1000"); + } + + #[test] + fn unestimated_findings_rank_after_estimated_ones() { + let target = target(); + let threads = threads(thread(0.1, 0.0)); + let kernels = [kernel("gemm", 900)]; + let busy_gpu = [gpu(60.0, &["sw_power_cap"])]; + let evidence = TriageEvidence { + target: &target, + threads: &threads, + collection_window_ns: WINDOW, + cgroup: None, + cpu_hotspots: &[], + python_status: None, + syscalls: &[], + gpu_devices: &busy_gpu, + gpu_activity: Some(&kernels), + }; + let found = findings(&evidence) + .into_iter() + .map(|finding| (finding.code, finding.impact_share.is_some())) + .collect::>(); + assert_eq!( + found, + [ + ("THREAD_BLOCKED".to_owned(), true), + ("GPU_KERNEL_DOMINANT".to_owned(), true), + ("GPU_CLOCK_LIMITED".to_owned(), false), + ] + ); + } + + #[test] + fn spread_cpu_time_is_cpu_bound_with_python_hint() { + let target = target(); + let threads = threads(thread(0.95, 0.0)); + let hotspots = [hotspot("eval", 0.04), hotspot("add", 0.03)]; + let evidence = TriageEvidence { + target: &target, + threads: &threads, + collection_window_ns: WINDOW, + cgroup: None, + cpu_hotspots: &hotspots, + python_status: Some(PythonSymbolizationStatus::Inactive), + syscalls: &[], + gpu_devices: &[], + gpu_activity: None, + }; + let found = findings(&evidence); + assert_eq!(found.len(), 1); + assert_eq!(found[0].code, "CPU_BOUND"); + assert!(found[0].summary.contains("eval 4%")); + assert_eq!( + found[0].next.as_ref().map(|next| next.kind), + Some(TriageNextKind::Environment) + ); + } + #[test] fn ranks_throttling_runqueue_and_blocked_syscalls() { let target = target(); @@ -574,8 +757,10 @@ mod tests { let evidence = TriageEvidence { target: &target, threads: &threads, + collection_window_ns: WINDOW, cgroup: Some(&cgroup), cpu_hotspots: &[], + python_status: None, syscalls: &syscalls, gpu_devices: &[], gpu_activity: None, @@ -610,8 +795,10 @@ mod tests { let base = TriageEvidence { target: &target, threads: &threads, + collection_window_ns: WINDOW, cgroup: None, cpu_hotspots: &[], + python_status: None, syscalls: &[], gpu_devices: &idle_gpu, gpu_activity: None, @@ -644,7 +831,18 @@ mod tests { .iter() .find(|finding| finding.code == "GPU_KERNEL_DOMINANT") .unwrap(); - assert!((dominant.impact_share - 0.81).abs() < 1e-9); + assert!((dominant.impact_share.unwrap() - 0.81).abs() < 1e-9); + assert_eq!( + dominant + .next + .as_ref() + .unwrap() + .spec + .as_ref() + .unwrap() + .max_events, + Some(400) + ); assert_eq!( dominant .next diff --git a/xprobe/protocol/src/schema.rs b/xprobe/protocol/src/schema.rs index 56a7f0c..4b61389 100644 --- a/xprobe/protocol/src/schema.rs +++ b/xprobe/protocol/src/schema.rs @@ -4,11 +4,12 @@ use crate::{ AggregateInventoryResult, CapabilityReport, CpuSampleInventoryResult, CpuSamplingSpec, CpuSamplingValidationResult, DiscoveryResult, ErrorResponse, Event, HostCaptureResult, MeasurementResult, MeasurementSpec, ProcessReport, ResolvedProbe, SyscallAggregateResult, - SyscallAggregateSpec, SyscallAggregateValidationResult, TraceExportResult, ValidationResult, + SyscallAggregateSpec, SyscallAggregateValidationResult, TraceExportResult, TriageResult, + ValidationResult, }; #[must_use] -pub fn generated_schemas() -> [(&'static str, Schema); 18] { +pub fn generated_schemas() -> [(&'static str, Schema); 19] { [ ("event.schema.json", schema_for!(Event)), ("error.schema.json", schema_for!(ErrorResponse)), @@ -52,5 +53,6 @@ pub fn generated_schemas() -> [(&'static str, Schema); 18] { ("resolve.schema.json", schema_for!(ResolvedProbe)), ("validate.schema.json", schema_for!(ValidationResult)), ("trace-export.schema.json", schema_for!(TraceExportResult)), + ("triage-result.schema.json", schema_for!(TriageResult)), ] } diff --git a/xprobe/protocol/src/triage.rs b/xprobe/protocol/src/triage.rs index bb5fdc6..d55e5dc 100644 --- a/xprobe/protocol/src/triage.rs +++ b/xprobe/protocol/src/triage.rs @@ -95,7 +95,7 @@ pub struct TriageResult { pub status: crate::SessionStatus, pub target: crate::TargetIdentity, pub window_ms: u64, - /// Ranked by estimated share of the window each finding accounts for. + /// Ranked by estimated window share; findings without an estimate last. pub findings: Vec, pub threads: ThreadTimeBreakdown, pub cgroup: Option, @@ -115,8 +115,9 @@ pub struct TriageResult { pub struct TriageFinding { pub rank: u32, pub code: String, - /// Estimated share of the window this finding accounts for, in [0, 1]. - pub impact_share: f64, + /// Estimated share of the window this finding accounts for, in [0, 1]; + /// `null` when the evidence cannot bound the impact. + pub impact_share: Option, pub summary: String, pub next: Option, } diff --git a/xprobe/protocol/tests/contracts.rs b/xprobe/protocol/tests/contracts.rs index dc6b625..d641b4e 100644 --- a/xprobe/protocol/tests/contracts.rs +++ b/xprobe/protocol/tests/contracts.rs @@ -7,7 +7,7 @@ use xprobe_protocol::{ CpuSamplingValidationResult, DiscoveryResult, ErrorResponse, Event, HostCaptureResult, MeasurementResult, MeasurementSpec, ProcessReport, ResolvedLinuxSelector, ResolvedProbe, SyscallAggregateResult, SyscallAggregateSpec, SyscallAggregateValidationResult, - TraceExportResult, ValidationResult, schema::generated_schemas, + TraceExportResult, TriageResult, ValidationResult, schema::generated_schemas, }; fn assert_round_trip(fixture: &Value) @@ -722,3 +722,96 @@ fn unsupported_schema_versions_are_rejected() { assert!(result.is_err()); } + +#[test] +fn triage_result_contract_round_trips() { + assert_round_trip::(&json!({ + "schema_version": "3.0", + "ok": true, + "session_id": "xp_triage_1", + "status": "completed", + "target": {"pid": 42, "process_start_time": 9}, + "window_ms": 2100, + "findings": [ + { + "rank": 1, + "code": "CPU_HOTSPOT", + "impact_share": 0.36, + "summary": "busiest thread 7 was on CPU 90% of the window", + "next": { + "kind": "measure", + "description": "measure each call of parse", + "spec": { + "schema_version": "2.0", + "name": "hotspot_duration", + "target": {"pid": 42, "process_start_time": 9}, + "start_selector": "uprobe:/srv/app:parse:entry", + "end_selector": "uprobe:/srv/app:parse:return", + "match_policy": "stack_nested", + "samples": 100, + "duration_ms": null, + "timeout_ms": 30000, + "max_events": 400, + "measurement_mode": "exact", + "max_groups": null + } + } + }, + { + "rank": 2, + "code": "GPU_CLOCK_LIMITED", + "impact_share": null, + "summary": "GPU 0 reported clock slowdowns under load: sw_power_cap", + "next": {"kind": "environment", "description": "check cooling", "spec": null} + } + ], + "threads": { + "window_ns": 2_100_000_000_u64, + "threads_observed": 1, + "threads_started": 0, + "threads_exited": 0, + "on_cpu_ns": 1_890_000_000_u64, + "runqueue_ns": 0, + "blocked_ns": 210_000_000, + "top_threads": [{ + "tid": 7, + "name": "worker", + "on_cpu_ns": 1_890_000_000_u64, + "runqueue_ns": 0, + "blocked_ns": 210_000_000, + "on_cpu_share": 0.9, + "runqueue_share": 0.0, + "blocked_share": 0.1 + }] + }, + "cgroup": { + "version": "v2", + "cgroup_path": "/", + "quota_cpus": 0.5, + "periods": 21, + "throttled_periods": 0, + "throttled_ns": 0 + }, + "cpu_hotspots": [], + "syscalls": [], + "gpu_devices": [{ + "index": 0, + "uuid": "GPU-a", + "samples": 9, + "utilization_mean_pct": 54.0, + "utilization_max_pct": 61, + "memory_utilization_mean_pct": 12.5, + "clock_event_reasons": ["sw_power_cap"], + "temperature_max_c": 71, + "power_max_w": 80.5, + "pcie": {"generation_min": 4, "generation_max": 4, "width_min": 8, "width_max": 16} + }], + "gpu_activity": [], + "sources": [ + {"source": "thread_time", "status": "collected", "reason": null, "artifact": null}, + {"source": "gpu_activity", "status": "skipped", "reason": "the target has not loaded libcuda", "artifact": null} + ], + "not_covered": ["where blocked threads wait"], + "warnings": [] + })); +} From 5ca9b9d8cc4c7bf251fd285fcbbb706e8868685c Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 14 Nov 2026 14:05:08 +0800 Subject: [PATCH 08/15] =?UTF-8?q?=E2=9C=A8=20feat:=20carry=20up=20to=20eig?= =?UTF-8?q?ht=20CUPTI=20endpoint=20filters?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measurement graphs need more than one start/end pair of CUDA endpoints in a single capture. Raise the Agent filter table from 2 to 8 and bump the control protocol to v5: the former reserved field now carries the active filter count on ARM (zero for other commands). Hot paths scan only the active filters, so single-edge captures keep their per-record cost. Aggregate ARM still requires exactly one start/end pair. The capture output ABI is unchanged. --- cupti/include/xprobe/cupti_agent.h | 7 ++-- cupti/src/cupti_agent.c | 36 ++++++++++++++------ cupti/tests/cupti_smoke.c | 3 +- xprobe/collector/src/cupti.rs | 54 ++++++++++++++++++++++++------ 4 files changed, 76 insertions(+), 24 deletions(-) diff --git a/cupti/include/xprobe/cupti_agent.h b/cupti/include/xprobe/cupti_agent.h index d48d031..0a50a89 100644 --- a/cupti/include/xprobe/cupti_agent.h +++ b/cupti/include/xprobe/cupti_agent.h @@ -10,9 +10,9 @@ extern "C" { #define XPROBE_CUPTI_AGENT_ABI_VERSION 4U #define XPROBE_CUPTI_OUTPUT_MAGIC "XPCUPTI" #define XPROBE_CUPTI_CONTROL_MAGIC "XPCTRL\0" -#define XPROBE_CUPTI_CONTROL_VERSION 4U +#define XPROBE_CUPTI_CONTROL_VERSION 5U #define XPROBE_CUPTI_NAME_LENGTH 128U -#define XPROBE_CUPTI_FILTER_COUNT 2U +#define XPROBE_CUPTI_FILTER_COUNT 8U #define XPROBE_CUPTI_VALUE_UNKNOWN UINT32_MAX enum xprobe_cupti_feature { @@ -80,7 +80,8 @@ struct xprobe_cupti_control_request { uint64_t record_capacity; uint64_t record_offset; uint32_t capture_mode; - uint32_t reserved; + /* ARM: active filters in filters[0..filter_count); other commands: 0. */ + uint32_t filter_count; struct xprobe_cupti_filter filters[XPROBE_CUPTI_FILTER_COUNT]; }; diff --git a/cupti/src/cupti_agent.c b/cupti/src/cupti_agent.c index abb3cf3..540bf1f 100644 --- a/cupti/src/cupti_agent.c +++ b/cupti/src/cupti_agent.c @@ -95,7 +95,7 @@ _Static_assert(sizeof(struct xprobe_cupti_aggregate_record) == 200U, "unexpected CUPTI aggregate record layout"); _Static_assert(sizeof(struct xprobe_cupti_filter) == 144U, "unexpected CUPTI filter layout"); -_Static_assert(sizeof(struct xprobe_cupti_control_request) == 328U, +_Static_assert(sizeof(struct xprobe_cupti_control_request) == 1192U, "unexpected CUPTI control request layout"); struct xprobe_cupti_aggregate_slot { @@ -156,6 +156,8 @@ static _Atomic int snapshot_thread_stop; static _Atomic int snapshot_listener = -1; static int snapshot_thread_started; static struct xprobe_cupti_filter capture_filters[XPROBE_CUPTI_FILTER_COUNT]; +/* Written only while capture is inactive; hot paths scan just these slots. */ +static size_t capture_filter_count; static _Atomic int capture_filter_enabled; static _Atomic int nvtx_injection_active; static pthread_once_t injection_agent_once = PTHREAD_ONCE_INIT; @@ -557,7 +559,7 @@ static int values_match_capture(uint32_t first_kind, uint32_t second_kind, if (atomic_load_explicit(&capture_filter_enabled, memory_order_relaxed) == 0) { return 1; } - for (size_t index = 0U; index < XPROBE_CUPTI_FILTER_COUNT; ++index) { + for (size_t index = 0U; index < capture_filter_count; ++index) { const struct xprobe_cupti_filter *filter = &capture_filters[index]; if (filter_matches_values(filter, first_kind, api_domain, memcpy_kind, @@ -576,7 +578,7 @@ static int capture_filter_matches(const struct xprobe_cupti_record *record) if (atomic_load_explicit(&capture_filter_enabled, memory_order_relaxed) == 0) { return 1; } - for (size_t index = 0U; index < XPROBE_CUPTI_FILTER_COUNT; ++index) { + for (size_t index = 0U; index < capture_filter_count; ++index) { if (record_matches_filter(record, &capture_filters[index]) != 0) { return 1; } @@ -1730,11 +1732,15 @@ static int valid_filter(const struct xprobe_cupti_filter *filter) } static int valid_aggregate_filters( - const struct xprobe_cupti_filter filters[XPROBE_CUPTI_FILTER_COUNT]) + const struct xprobe_cupti_filter filters[XPROBE_CUPTI_FILTER_COUNT], + uint32_t filter_count) { uint32_t start = filters[0].record_kind; uint32_t end = filters[1].record_kind; + if (filter_count != 2U) { + return 0; + } if (!((start == XPROBE_CUPTI_GPU_KERNEL_START && end == XPROBE_CUPTI_GPU_KERNEL_END) || (start == XPROBE_CUPTI_GPU_MEMCPY_START && @@ -1755,8 +1761,16 @@ static int arm_capture(const struct xprobe_cupti_control_request *request) int status; int needs_nvtx = 0; + if (request->filter_count == 0U || + request->filter_count > XPROBE_CUPTI_FILTER_COUNT) { + fprintf(stderr, "xprobe CUPTI: invalid capture filter count\n"); + remember_output_error(); + return XPROBE_CUPTI_AGENT_OUTPUT_ERROR; + } for (size_t index = 0U; index < XPROBE_CUPTI_FILTER_COUNT; ++index) { - if (valid_filter(&request->filters[index]) == 0) { + if (valid_filter(&request->filters[index]) == 0 || + (index >= request->filter_count && + request->filters[index].record_kind != 0U)) { fprintf(stderr, "xprobe CUPTI: invalid capture filter\n"); remember_output_error(); return XPROBE_CUPTI_AGENT_OUTPUT_ERROR; @@ -1769,7 +1783,7 @@ static int arm_capture(const struct xprobe_cupti_control_request *request) } } if (request->capture_mode == XPROBE_CUPTI_CAPTURE_AGGREGATE && - valid_aggregate_filters(request->filters) == 0) { + valid_aggregate_filters(request->filters, request->filter_count) == 0) { fprintf(stderr, "xprobe CUPTI: aggregate capture requires one matching " "activity start/end pair\n"); @@ -1790,6 +1804,7 @@ static int arm_capture(const struct xprobe_cupti_control_request *request) } reset_capture(); memcpy(capture_filters, request->filters, sizeof(capture_filters)); + capture_filter_count = request->filter_count; atomic_store_explicit(&capture_filter_enabled, 1, memory_order_release); return activate_capture(); } @@ -1861,7 +1876,8 @@ static void *serve_snapshots(void *unused) (request.command < XPROBE_CUPTI_CONTROL_ARM || request.command > XPROBE_CUPTI_CONTROL_CLOSE) || request.capture_mode > XPROBE_CUPTI_CAPTURE_AGGREGATE || - request.reserved != 0U) { + (request.command != XPROBE_CUPTI_CONTROL_ARM && + request.filter_count != 0U)) { fprintf(stderr, "xprobe CUPTI: invalid snapshot control request\n"); } else if (pthread_mutex_lock(&flush_mutex) != 0) { fprintf(stderr, "xprobe CUPTI: failed to lock snapshot flush mutex\n"); @@ -2084,7 +2100,7 @@ static int capture_needs_record_kind(uint32_t first_kind, uint32_t second_kind) if (atomic_load_explicit(&capture_filter_enabled, memory_order_relaxed) == 0) { return 1; } - for (size_t index = 0U; index < XPROBE_CUPTI_FILTER_COUNT; ++index) { + for (size_t index = 0U; index < capture_filter_count; ++index) { uint32_t kind = capture_filters[index].record_kind; if (kind == first_kind || kind == second_kind) { @@ -2103,7 +2119,7 @@ static int capture_needs_api_domain(uint32_t domain) if (atomic_load_explicit(&capture_filter_enabled, memory_order_relaxed) == 0) { return 1; } - for (size_t index = 0U; index < XPROBE_CUPTI_FILTER_COUNT; ++index) { + for (size_t index = 0U; index < capture_filter_count; ++index) { const struct xprobe_cupti_filter *filter = &capture_filters[index]; if ((filter->record_kind == XPROBE_CUPTI_CUDA_API_ENTRY || @@ -2163,7 +2179,7 @@ static int api_filter_matches_callback(CUpti_CallbackDomain domain, break; } } - for (size_t index = 0U; index < XPROBE_CUPTI_FILTER_COUNT; ++index) { + for (size_t index = 0U; index < capture_filter_count; ++index) { const struct xprobe_cupti_filter *filter = &capture_filters[index]; if ((filter->record_kind == XPROBE_CUPTI_CUDA_API_ENTRY || diff --git a/cupti/tests/cupti_smoke.c b/cupti/tests/cupti_smoke.c index 8f770c4..42e0114 100644 --- a/cupti/tests/cupti_smoke.c +++ b/cupti/tests/cupti_smoke.c @@ -32,12 +32,13 @@ int main(void) return 7; } if (sizeof(struct xprobe_cupti_filter) != 144U || - sizeof(struct xprobe_cupti_control_request) != 328U) { + sizeof(struct xprobe_cupti_control_request) != 1192U) { return 8; } if (offsetof(struct xprobe_cupti_control_request, record_capacity) != 16U || offsetof(struct xprobe_cupti_control_request, record_offset) != 24U || offsetof(struct xprobe_cupti_control_request, capture_mode) != 32U || + offsetof(struct xprobe_cupti_control_request, filter_count) != 36U || offsetof(struct xprobe_cupti_control_request, filters) != 40U || offsetof(struct xprobe_cupti_filter, name) != 16U) { return 9; diff --git a/xprobe/collector/src/cupti.rs b/xprobe/collector/src/cupti.rs index 6ba708f..0e05ed1 100644 --- a/xprobe/collector/src/cupti.rs +++ b/xprobe/collector/src/cupti.rs @@ -22,15 +22,16 @@ pub use filter::{CuptiFilterError, bounded_name_filter, event_filter}; const OUTPUT_MAGIC: &[u8; 8] = b"XPCUPTI\0"; const CONTROL_MAGIC: &[u8; 8] = b"XPCTRL\0\0"; -const CONTROL_VERSION: u32 = 4; +const CONTROL_VERSION: u32 = 5; const ABI_VERSION: u32 = 4; const HEADER_SIZE: usize = 88; const HEADER_SIZE_U32: u32 = 88; const RECORD_SIZE: usize = 200; const RECORD_SIZE_U32: u32 = 200; -const CONTROL_SIZE: usize = 328; +const CONTROL_SIZE: usize = 1192; const FILTER_SIZE: usize = 144; -const FILTER_COUNT: usize = 2; +/// Endpoint filters one ARM can carry; matches `XPROBE_CUPTI_FILTER_COUNT`. +pub const CUPTI_FILTER_COUNT: usize = 8; pub const CUPTI_NAME_CAPACITY: usize = 128; const FEATURE_HOST_MONOTONIC_TIMESTAMPS: u32 = 1 << 0; const FEATURE_TRANSFER_RECORDS: u32 = 1 << 1; @@ -456,9 +457,9 @@ fn encode_control( "CUPTI record capacity must be positive".to_owned(), )); } - if config.filters.is_empty() || config.filters.len() > FILTER_COUNT { + if config.filters.is_empty() || config.filters.len() > CUPTI_FILTER_COUNT { return Err(CuptiSnapshotError::InvalidControl(format!( - "CUPTI capture requires one or two endpoint filters, found {}", + "CUPTI capture requires between one and {CUPTI_FILTER_COUNT} endpoint filters, found {}", config.filters.len() ))); } @@ -469,6 +470,8 @@ fn encode_control( } control[16..24].copy_from_slice(&config.record_capacity.to_ne_bytes()); control[32..36].copy_from_slice(&(config.capture_mode as u32).to_ne_bytes()); + let filter_count = u32::try_from(config.filters.len()).expect("filter count is bounded"); + control[36..40].copy_from_slice(&filter_count.to_ne_bytes()); for (index, filter) in config.filters.iter().enumerate() { let offset = 40 + index * FILTER_SIZE; control[offset..offset + 4].copy_from_slice(&(filter.record_kind as u32).to_ne_bytes()); @@ -974,9 +977,10 @@ mod tests { use std::io::Cursor; use super::{ - CUPTI_NAME_CAPACITY, CuptiApiDomain, CuptiArmConfig, CuptiCaptureMode, CuptiDecodeError, - CuptiEventFilter, CuptiMemcpyKind, CuptiNameFilter, CuptiRecordKind, HEADER_SIZE, - OUTPUT_MAGIC, RECORD_SIZE, decode_capture, encode_control, read_snapshot, + CUPTI_FILTER_COUNT, CUPTI_NAME_CAPACITY, CuptiApiDomain, CuptiArmConfig, CuptiCaptureMode, + CuptiDecodeError, CuptiEventFilter, CuptiMemcpyKind, CuptiNameFilter, CuptiRecordKind, + FILTER_SIZE, HEADER_SIZE, OUTPUT_MAGIC, RECORD_SIZE, decode_capture, encode_control, + read_snapshot, }; use xprobe_protocol::{ClockDomain, EventSource, EventType, MemcpyKind, NvtxRangeKind}; @@ -1402,7 +1406,7 @@ mod tests { ) .expect("ARM request must encode"); - assert_eq!(control.len(), 328); + assert_eq!(control.len(), 1192); assert_eq!(&control[0..8], super::CONTROL_MAGIC); assert_eq!(u32::from_ne_bytes(control[12..16].try_into().unwrap()), 1); assert_eq!( @@ -1411,12 +1415,38 @@ mod tests { ); assert_eq!(u64::from_ne_bytes(control[24..32].try_into().unwrap()), 0); assert_eq!(u32::from_ne_bytes(control[32..36].try_into().unwrap()), 0); + assert_eq!(u32::from_ne_bytes(control[36..40].try_into().unwrap()), 1); assert_eq!(u32::from_ne_bytes(control[40..44].try_into().unwrap()), 3); assert_eq!(u32::from_ne_bytes(control[52..56].try_into().unwrap()), 2); assert_eq!(&control[56..62], b"flash_"); assert!(control[184..].iter().all(|byte| *byte == 0)); } + #[test] + fn encodes_up_to_the_filter_capacity() { + let filter = CuptiEventFilter { + record_kind: CuptiRecordKind::GpuKernelEnd, + api_domain: CuptiApiDomain::Any, + memcpy_kind: CuptiMemcpyKind::Any, + name: CuptiNameFilter::Any, + }; + let config = |count: usize| CuptiArmConfig { + record_capacity: 16, + capture_mode: CuptiCaptureMode::Exact, + filters: vec![filter.clone(); count], + }; + let control = encode_control(1, 0, Some(&config(CUPTI_FILTER_COUNT))).unwrap(); + assert_eq!(u32::from_ne_bytes(control[36..40].try_into().unwrap()), 8); + let last = 40 + 7 * FILTER_SIZE; + assert_eq!( + u32::from_ne_bytes(control[last..last + 4].try_into().unwrap()), + 4 + ); + assert!(encode_control(1, 0, Some(&config(CUPTI_FILTER_COUNT + 1))).is_err()); + let snapshot = encode_control(2, 0, None).unwrap(); + assert_eq!(u32::from_ne_bytes(snapshot[36..40].try_into().unwrap()), 0); + } + #[test] fn encodes_aggregate_arm_mode() { let control = encode_control( @@ -1461,7 +1491,11 @@ mod tests { ) .expect_err("empty filter set must fail"); - assert!(error.to_string().contains("one or two endpoint filters")); + assert!( + error + .to_string() + .contains("between one and 8 endpoint filters") + ); } #[test] From a2d967505f2cf701ea22aff6064cd71b28457952 Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 14 Nov 2026 14:10:12 +0800 Subject: [PATCH 09/15] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20share=20?= =?UTF-8?q?the=20live=20capture=20loop=20across=20evaluators?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exact live measurement now builds a LivePlan of validated endpoints and merged requirements, then runs one generic capture loop that delegates correlation and sample completion to a LiveEvaluator. Attachment, polling, timeout, CUPTI shutdown, cleanup failures, and artifact preservation stay in one place. Identical CUDA filters from shared endpoints are armed once. Single-edge behavior is unchanged. --- xprobe/cli/src/live.rs | 273 +++++++++++++++++++++++++++++------------ 1 file changed, 194 insertions(+), 79 deletions(-) diff --git a/xprobe/cli/src/live.rs b/xprobe/cli/src/live.rs index 4e1fabb..77ec824 100644 --- a/xprobe/cli/src/live.rs +++ b/xprobe/cli/src/live.rs @@ -28,6 +28,7 @@ use xprobe_protocol::AggregateActivity; use xprobe_protocol::AggregateInventoryResult; use xprobe_protocol::CuptiCollectionSummary; use xprobe_protocol::ErrorCode; +use xprobe_protocol::Event; use xprobe_protocol::EventType; use xprobe_protocol::HostCaptureResult; use xprobe_protocol::MatchPolicy; @@ -38,6 +39,8 @@ use xprobe_protocol::ResolvedLinuxSelector; use xprobe_protocol::ResolvedProbe; use xprobe_protocol::SessionStatus; use xprobe_protocol::TargetIdentity; +use xprobe_protocol::ValidatedEndpoint; +use xprobe_protocol::ValidationRequirements; use xprobe_protocol::ValidationResult; use xprobe_protocol::Warning; @@ -52,6 +55,74 @@ pub(crate) struct LiveMeasureRequest { pub(crate) options: MeasureOptions, } +/// Resolved endpoints and merged requirements for one live capture. +pub(crate) struct LivePlan { + pub(crate) report: ProcessReport, + pub(crate) endpoints: Vec, + pub(crate) requirements: ValidationRequirements, +} + +/// A correlated view of the capture that the live loop can finish with. +pub(crate) trait LiveOutcome { + fn set_status(&mut self, status: SessionStatus); + fn push_warning(&mut self, warning: Warning); + fn into_events(self) -> Vec; +} + +/// How a live capture is correlated and when it is complete. +pub(crate) trait LiveEvaluator { + type Outcome: LiveOutcome; + + /// Correlate the capture so far. With `require_complete` false, a + /// `NO_MATCHED_SAMPLES` failure means "keep collecting". + fn evaluate( + &self, + host_captures: &[HostCaptureResult], + collection: &LiveCollection, + require_complete: bool, + ) -> Result; + + /// Whether every requested sample bound has been reached. + fn reached_samples(&self, outcome: &Self::Outcome) -> bool; +} + +impl LiveOutcome for MeasurementExecution { + fn set_status(&mut self, status: SessionStatus) { + self.result.status = status; + } + + fn push_warning(&mut self, warning: Warning) { + self.result.warnings.push(warning); + } + + fn into_events(self) -> Vec { + self.events + } +} + +/// The single start/end edge of `measure --from/--to` and `MeasurementSpec`. +struct SingleEdge<'a>(&'a MeasureOptions); + +impl LiveEvaluator for SingleEdge<'_> { + type Outcome = MeasurementExecution; + + fn evaluate( + &self, + host_captures: &[HostCaptureResult], + collection: &LiveCollection, + require_complete: bool, + ) -> Result { + measure_live_capture(host_captures, collection, self.0, require_complete) + } + + fn reached_samples(&self, outcome: &MeasurementExecution) -> bool { + self.0.samples.is_some_and(|samples| { + outcome.result.measurement.samples.matched + >= u64::try_from(samples).expect("sample limit fits u64") + }) + } +} + type HostCaptureHandle = JoinHandle>; pub(crate) const SNAPSHOT_QUIET_PERIOD: Duration = Duration::from_millis(100); @@ -118,17 +189,30 @@ pub(crate) fn collect_live_measurement( ) -> Result { assert_eq!(request.mode, MeasurementMode::Exact); validate_live_limits(request)?; - let mut collection = prepare_live_collection(request)?; + let plan = single_edge_plan(request)?; + collect_live(plan, request, &SingleEdge(&request.options)) +} + +/// Run one bounded exact capture of `plan` and correlate it with `evaluator`. +/// +/// Attachment, polling, timeout, CUPTI shutdown, cleanup failures, and +/// artifact preservation are shared by every exact live measurement. +pub(crate) fn collect_live( + plan: LivePlan, + request: &LiveMeasureRequest, + evaluator: &E, +) -> Result { + let mut collection = start_live_collection(plan, request)?; let outcome = (|| { loop { refresh_live_sources(&mut collection, request)?; let now = Instant::now(); - if let Some(result) = evaluate_live_result(&collection, request, now)? { + if let Some(result) = evaluate_live_result(&collection, evaluator, now)? { break Ok(result); } if now >= collection.deadline { - break finish_live_timeout(&mut collection, request); + break finish_live_timeout(&mut collection, evaluator); } if collection.socket.is_none() { thread::sleep(Duration::from_millis(10)); @@ -140,23 +224,21 @@ pub(crate) fn collect_live_measurement( let mut execution = match (outcome, stop_result) { (Ok(mut execution), Ok(())) => { if cuda_capture && collection.host_captures.is_some() { - let status = execution.result.status; - execution = measure_live_capture( + let status = execution.1; + execution.0 = evaluator.evaluate( collection.host_captures.as_deref().unwrap_or(&[]), &collection, - &request.options, true, )?; - execution.result.status = status; + execution.0.set_status(status); } - execution + execution.0 } (Err(error), Ok(())) => { if cuda_capture && collection.host_captures.is_some() { - return match measure_live_capture( + return match evaluator.evaluate( collection.host_captures.as_deref().unwrap_or(&[]), &collection, - &request.options, true, ) { Ok(_) => Err(error), @@ -166,7 +248,7 @@ pub(crate) fn collect_live_measurement( return Err(error); } (Ok(execution), Err(cleanup)) => { - return Err(cleanup.with_artifact_events(execution.events)); + return Err(cleanup.with_artifact_events(execution.0.into_events())); } (Err(mut error), Err(cleanup)) => { let artifact_events = error.artifact_events.take(); @@ -205,7 +287,7 @@ pub(crate) fn collect_live_measurement( serde_json::Value::from(path.display().to_string()), ); } - execution.result.warnings.push(Warning { + execution.push_warning(Warning { code: "CUPTI_AGENT_INJECTED".to_owned(), message: "xprobe injected the CUPTI agent and left the shared object mapped".to_owned(), details, @@ -317,6 +399,11 @@ pub(crate) fn validate_live_limits(request: &LiveMeasureRequest) -> Result<(), C pub(crate) fn prepare_live_collection( request: &LiveMeasureRequest, ) -> Result { + start_live_collection(single_edge_plan(request)?, request) +} + +/// Validate the request's start/end pair into a capture plan. +fn single_edge_plan(request: &LiveMeasureRequest) -> Result { let report = inspect::run(request.pid).map_err(|error| { CommandFailure::new(error.code(), error.to_string(), error.recoverable()) })?; @@ -354,13 +441,35 @@ pub(crate) fn prepare_live_collection( if request.mode == MeasurementMode::Aggregate { validate_aggregate_selection(&validation)?; } + Ok(LivePlan { + report, + endpoints: vec![validation.start, validation.end], + requirements: validation.requirements, + }) +} + +/// Arm CUPTI and attach host collectors for a validated plan. +pub(crate) fn start_live_collection( + plan: LivePlan, + request: &LiveMeasureRequest, +) -> Result { + let LivePlan { + report, + endpoints, + requirements, + } = plan; - let activation = prepare_cupti_activation(&report, &validation, request)?; - let arm_config = cupti_arm_config(&validation, request.options.max_events, request.mode)?; + let activation = prepare_cupti_activation(&report, &requirements, request)?; + let arm_config = cupti_arm_config( + &endpoints, + &requirements, + request.options.max_events, + request.mode, + )?; let baseline = arm_cupti_capture( &activation, arm_config.as_ref(), - validation.requirements.needs_nvtx, + requirements.needs_nvtx, request, )?; let socket = activation.socket; @@ -389,7 +498,7 @@ pub(crate) fn prepare_live_collection( .options .duration .and_then(|duration| started.checked_add(duration)); - let collectors = start_host_collectors(&report, &validation, request)?; + let collectors = start_host_collectors(&report, &endpoints, request)?; let host_captures = collectors.handles.is_empty().then(Vec::new); Ok(LiveCollection { @@ -530,15 +639,16 @@ pub(crate) fn cleanup_failed_arm( } pub(crate) fn cupti_arm_config( - validation: &ValidationResult, + endpoints: &[ValidatedEndpoint], + requirements: &ValidationRequirements, max_events: usize, mode: MeasurementMode, ) -> Result, CommandFailure> { - if !validation.requirements.needs_cupti { + if !requirements.needs_cupti { return Ok(None); } - let filters = [&validation.start, &validation.end] - .into_iter() + let mut filters = endpoints + .iter() .filter_map(|endpoint| endpoint.cuda.as_ref()) .map(|selector| { cupti::event_filter(selector).map_err(|error| { @@ -546,6 +656,15 @@ pub(crate) fn cupti_arm_config( }) }) .collect::, _>>()?; + if mode == MeasurementMode::Exact { + let mut unique = Vec::with_capacity(filters.len()); + for filter in filters { + if !unique.contains(&filter) { + unique.push(filter); + } + } + filters = unique; + } let record_capacity = u64::try_from(max_events).map_err(|error| { CommandFailure::new( ErrorCode::SessionLimitExceeded, @@ -580,10 +699,10 @@ pub(crate) struct AgentSelection { pub(crate) fn prepare_cupti_activation( report: &ProcessReport, - validation: &ValidationResult, + requirements: &ValidationRequirements, request: &LiveMeasureRequest, ) -> Result { - if !validation.requirements.needs_cupti { + if !requirements.needs_cupti { if request.cupti_socket.is_some() { return Err(CommandFailure::new( ErrorCode::InvalidEventSelector, @@ -791,7 +910,7 @@ impl Drop for LiveCollection { pub(crate) fn start_host_collectors( report: &ProcessReport, - validation: &ValidationResult, + endpoints: &[ValidatedEndpoint], request: &LiveMeasureRequest, ) -> Result { inspect::verify_target(&report.target).map_err(|error| { @@ -800,7 +919,7 @@ pub(crate) fn start_host_collectors( let cancelled = Arc::new(AtomicBool::new(false)); let mut probes = Vec::new(); let mut linux_probes = Vec::new(); - for endpoint in [&validation.start, &validation.end] { + for endpoint in endpoints { if let Some(probe) = endpoint.host.as_ref() && !probes .iter() @@ -843,7 +962,7 @@ pub(crate) fn start_host_collectors( symbol_demangled: probe.symbol_demangled, offset, probe_kind: probe.probe_kind, - probe_id: u32::try_from(index + 1).expect("two endpoints fit u32"), + probe_id: u32::try_from(index + 1).expect("bounded endpoints fit u32"), samples: host_samples, timeout: host_timeout, cancelled: Arc::clone(&cancelled), @@ -1148,87 +1267,83 @@ pub(crate) fn injection_warning(collection: &LiveCollection) -> Warning { } } -pub(crate) fn evaluate_live_result( +pub(crate) fn evaluate_live_result( collection: &LiveCollection, - request: &LiveMeasureRequest, + evaluator: &E, now: Instant, -) -> Result, CommandFailure> { +) -> Result, CommandFailure> { let Some(host) = collection.host_captures.as_ref() else { return Ok(None); }; if collection.socket.is_some() && collection.latest_cuda.is_none() { return Ok(None); } - let mut execution = match measure_live_capture(host, collection, &request.options, false) { - Ok(execution) => execution, + let mut outcome = match evaluator.evaluate(host, collection, false) { + Ok(outcome) => outcome, Err(error) if error.code == ErrorCode::NoMatchedSamples => return Ok(None), Err(error) => return Err(error), }; - let reached_samples = request.options.samples.is_some_and(|samples| { - execution.result.measurement.samples.matched - >= u64::try_from(samples).expect("sample limit fits u64") - }); let reached_duration = collection.collection_end.is_some_and(|end| now >= end); - if reached_samples || reached_duration { + if evaluator.reached_samples(&outcome) || reached_duration { inspect::verify_target(&collection.report.target).map_err(|error| { CommandFailure::new(error.code(), error.to_string(), error.recoverable()) })?; - return Ok(Some(execution)); + return Ok(Some((outcome, SessionStatus::Completed))); } if now >= collection.deadline { - execution.result.status = SessionStatus::TimedOut; - return Ok(Some(execution)); + outcome.set_status(SessionStatus::TimedOut); + return Ok(Some((outcome, SessionStatus::TimedOut))); } Ok(None) } -pub(crate) fn finish_live_timeout( +pub(crate) fn finish_live_timeout( collection: &mut LiveCollection, - request: &LiveMeasureRequest, -) -> Result { + evaluator: &E, +) -> Result<(E::Outcome, SessionStatus), CommandFailure> { collection.collectors.cancel(); if collection.host_captures.is_none() { collection.host_captures = Some(collection.collectors.join()?); } - let mut execution = measure_live_capture( - collection - .host_captures - .as_ref() - .expect("host captures were assigned"), - collection, - &request.options, - true, - ) - .map_err(|mut error| { - if error.code == ErrorCode::NoMatchedSamples { - let host_events = collection.host_captures.as_ref().map_or(0, |captures| { - captures.iter().map(|capture| capture.events.len()).sum() - }); - let cuda_events = collection.latest_cuda.as_ref().map_or(0, |capture| { - capture - .events - .iter() - .filter(|event| { - collection - .baseline_timestamp - .is_none_or(|start| event.timestamp_ns > start) - }) - .count() - }); - error.message = format!( - "no event pairs matched before the live measurement timeout \ + let mut outcome = evaluator + .evaluate( + collection + .host_captures + .as_ref() + .expect("host captures were assigned"), + collection, + true, + ) + .map_err(|mut error| { + if error.code == ErrorCode::NoMatchedSamples { + let host_events = collection.host_captures.as_ref().map_or(0, |captures| { + captures.iter().map(|capture| capture.events.len()).sum() + }); + let cuda_events = collection.latest_cuda.as_ref().map_or(0, |capture| { + capture + .events + .iter() + .filter(|event| { + collection + .baseline_timestamp + .is_none_or(|start| event.timestamp_ns > start) + }) + .count() + }); + error.message = format!( + "no event pairs matched before the live measurement timeout \ (host events: {host_events}, CUDA events: {cuda_events})" - ); - error - .with_detail("timed_out", true) - .with_detail("host_events", host_events as u64) - .with_detail("cuda_events", cuda_events as u64) - } else { - error - } - })?; - execution.result.status = SessionStatus::TimedOut; - Ok(execution) + ); + error + .with_detail("timed_out", true) + .with_detail("host_events", host_events as u64) + .with_detail("cuda_events", cuda_events as u64) + } else { + error + } + })?; + outcome.set_status(SessionStatus::TimedOut); + Ok((outcome, SessionStatus::TimedOut)) } pub(crate) fn measure_live_capture( From 466733c4ee647a4a9606ede7ecddd7c763b9c4b2 Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 14 Nov 2026 14:10:12 +0800 Subject: [PATCH 10/15] =?UTF-8?q?=E2=9C=A8=20feat:=20define=20and=20valida?= =?UTF-8?q?te=20measurement=20graphs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add schema 3.0 MeasurementGraphSpec, validation, and result contracts. Graph validation resolves each edge with the single-edge rules, merges requirements, and reports graph, endpoint, and edge-scoped issues for structure, bounds, unknown or unused endpoints, and the two shared Linux collector slots. --- schemas/measurement-graph-result.schema.json | 1083 +++++++++++++++++ schemas/measurement-graph-spec.schema.json | 135 ++ .../measurement-graph-validate.schema.json | 635 ++++++++++ xprobe/core/src/graph.rs | 425 +++++++ xprobe/core/src/lib.rs | 1 + xprobe/protocol/src/graph.rs | 137 +++ xprobe/protocol/src/lib.rs | 7 + xprobe/protocol/src/schema.rs | 15 +- 8 files changed, 2437 insertions(+), 1 deletion(-) create mode 100644 schemas/measurement-graph-result.schema.json create mode 100644 schemas/measurement-graph-spec.schema.json create mode 100644 schemas/measurement-graph-validate.schema.json create mode 100644 xprobe/core/src/graph.rs create mode 100644 xprobe/protocol/src/graph.rs diff --git a/schemas/measurement-graph-result.schema.json b/schemas/measurement-graph-result.schema.json new file mode 100644 index 0000000..c43b6c9 --- /dev/null +++ b/schemas/measurement-graph-result.schema.json @@ -0,0 +1,1083 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "MeasurementGraphResult", + "description": "Per-edge results from one shared bounded capture.", + "type": "object", + "properties": { + "artifact": { + "anyOf": [ + { + "$ref": "#/$defs/EvidenceArtifact" + }, + { + "type": "null" + } + ] + }, + "collection": { + "description": "The shared capture every edge was correlated from.", + "$ref": "#/$defs/CollectionSummary" + }, + "edges": { + "type": "array", + "items": { + "$ref": "#/$defs/GraphEdgeResult" + } + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "ok": { + "type": "boolean" + }, + "schema_version": { + "$ref": "#/$defs/SchemaVersion3" + }, + "session_id": { + "type": "string" + }, + "status": { + "$ref": "#/$defs/SessionStatus" + }, + "target": { + "$ref": "#/$defs/TargetIdentity" + }, + "warnings": { + "type": "array", + "default": [], + "items": { + "$ref": "#/$defs/Warning" + } + } + }, + "additionalProperties": false, + "required": [ + "schema_version", + "ok", + "session_id", + "status", + "target", + "edges", + "collection" + ], + "$defs": { + "ArgumentValue": { + "type": "object", + "properties": { + "abi_type": { + "type": "string" + }, + "index": { + "type": "integer", + "format": "uint16", + "maximum": 65535, + "minimum": 0 + }, + "read_error": { + "type": [ + "string", + "null" + ] + }, + "value": true + }, + "additionalProperties": false, + "required": [ + "index", + "abi_type" + ] + }, + "CaptureCompleteness": { + "type": "string", + "enum": [ + "complete" + ] + }, + "ClockDomain": { + "type": "string", + "enum": [ + "host_monotonic", + "cupti", + "cupti_normalized_to_host_monotonic" + ] + }, + "ClockQuality": { + "type": "object", + "properties": { + "alignment": { + "type": "string" + }, + "estimated_error_ns": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "alignment" + ] + }, + "CollectionSummary": { + "type": "object", + "properties": { + "completeness": { + "$ref": "#/$defs/CaptureCompleteness" + }, + "cuda_events": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "cupti": { + "anyOf": [ + { + "$ref": "#/$defs/CuptiCollectionSummary" + }, + { + "type": "null" + } + ] + }, + "dropped_events": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "host_events": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "completeness", + "host_events", + "cuda_events", + "dropped_events" + ] + }, + "CorrelationConfidence": { + "type": "string", + "enum": [ + "exact", + "high", + "heuristic", + "ambiguous" + ] + }, + "CorrelationSummary": { + "type": "object", + "properties": { + "confidence": { + "$ref": "#/$defs/CorrelationConfidence" + }, + "method": { + "type": "string" + }, + "score": { + "type": "number", + "format": "double" + } + }, + "additionalProperties": false, + "required": [ + "method", + "confidence", + "score" + ] + }, + "CudaEvent": { + "type": "object", + "properties": { + "block": { + "anyOf": [ + { + "$ref": "#/$defs/Dim3" + }, + { + "type": "null" + } + ] + }, + "bytes": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "callback_domain": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0 + }, + "callback_id": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0 + }, + "context_id": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0 + }, + "correlation_id": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0 + }, + "device_id": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0 + }, + "end_ns": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "grid": { + "anyOf": [ + { + "$ref": "#/$defs/Dim3" + }, + { + "type": "null" + } + ] + }, + "kernel_name": { + "type": [ + "string", + "null" + ] + }, + "kernel_name_mangled": { + "type": [ + "string", + "null" + ] + }, + "memcpy_kind": { + "anyOf": [ + { + "$ref": "#/$defs/MemcpyKind" + }, + { + "type": "null" + } + ] + }, + "runtime_correlation_id": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0 + }, + "start_ns": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "stream_id": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false + }, + "CuptiCollectionSummary": { + "type": "object", + "properties": { + "buffer_utilization": { + "type": "number", + "format": "double" + }, + "dropped_records": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "observed_records": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "record_capacity": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "retained_records": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "record_capacity", + "observed_records", + "retained_records", + "dropped_records", + "buffer_utilization" + ] + }, + "Dim3": { + "type": "object", + "properties": { + "x": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "y": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "z": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "x", + "y", + "z" + ] + }, + "Event": { + "type": "object", + "properties": { + "attributes": { + "type": "object", + "additionalProperties": true, + "default": {} + }, + "clock_domain": { + "$ref": "#/$defs/ClockDomain" + }, + "cpu": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0 + }, + "cuda": { + "anyOf": [ + { + "$ref": "#/$defs/CudaEvent" + }, + { + "type": "null" + } + ] + }, + "event_id": { + "type": "string" + }, + "event_type": { + "$ref": "#/$defs/EventType" + }, + "host": { + "anyOf": [ + { + "$ref": "#/$defs/HostEvent" + }, + { + "type": "null" + } + ] + }, + "nvtx": { + "anyOf": [ + { + "$ref": "#/$defs/NvtxEvent" + }, + { + "type": "null" + } + ], + "default": null + }, + "pid": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "process_start_time": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "schema_version": { + "$ref": "#/$defs/SchemaVersion" + }, + "sequence": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "session_id": { + "type": "string" + }, + "source": { + "$ref": "#/$defs/EventSource" + }, + "tid": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "timestamp_error_ns": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "timestamp_ns": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "timestamp_raw": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "schema_version", + "session_id", + "event_id", + "sequence", + "source", + "event_type", + "pid", + "tid", + "timestamp_raw", + "timestamp_ns", + "clock_domain" + ] + }, + "EventSource": { + "type": "string", + "enum": [ + "ebpf", + "cupti_callback", + "cupti_activity", + "marker" + ] + }, + "EventType": { + "type": "string", + "enum": [ + "host_function_entry", + "host_function_exit", + "kernel_function_entry", + "kernel_function_exit", + "tracepoint", + "syscall_entry", + "syscall_exit", + "cuda_api_entry", + "cuda_api_exit", + "gpu_kernel_start", + "gpu_kernel_end", + "gpu_memcpy_start", + "gpu_memcpy_end", + "gpu_memset_start", + "gpu_memset_end", + "nvtx_range_start", + "nvtx_range_end", + "python_gc_start", + "python_gc_end", + "marker" + ] + }, + "Evidence": { + "description": "Matched pair evidence. `summary` keeps representative exemplars inline and\nreferences the bounded capture artifact; `full` also inlines every pair.", + "type": "object", + "properties": { + "artifact": { + "anyOf": [ + { + "$ref": "#/$defs/EvidenceArtifact" + }, + { + "type": "null" + } + ] + }, + "exemplars": { + "type": "array", + "items": { + "$ref": "#/$defs/EvidenceExemplar" + } + }, + "mode": { + "$ref": "#/$defs/EvidenceMode" + }, + "pairs": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/$defs/MatchedEventPair" + } + } + }, + "additionalProperties": false, + "required": [ + "mode", + "exemplars" + ] + }, + "EvidenceArtifact": { + "description": "Bounded capture written by `measure`; re-correlate it with `measure --input`.", + "type": "object", + "properties": { + "events": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "format": { + "$ref": "#/$defs/ExportFormat" + }, + "path": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "path", + "format", + "events" + ] + }, + "EvidenceExemplar": { + "description": "One matched pair selected for a latency rank. Ranks use the same\nnearest-rank indices as `measurement.latency_ns`, so an exemplar's latency\nequals the reported statistic.", + "type": "object", + "properties": { + "pair": { + "$ref": "#/$defs/MatchedEventPair" + }, + "ranks": { + "type": "array", + "items": { + "$ref": "#/$defs/LatencyRank" + } + } + }, + "additionalProperties": false, + "required": [ + "ranks", + "pair" + ] + }, + "EvidenceMode": { + "type": "string", + "enum": [ + "summary", + "full" + ] + }, + "ExportFormat": { + "type": "string", + "enum": [ + "jsonl", + "chrome" + ] + }, + "GraphEdgeResult": { + "type": "object", + "properties": { + "clock": { + "$ref": "#/$defs/ClockQuality" + }, + "correlation": { + "$ref": "#/$defs/CorrelationSummary" + }, + "end_selector": { + "type": "string" + }, + "evidence": { + "description": "Exemplars and optional full pairs; the artifact is graph-level.", + "$ref": "#/$defs/Evidence" + }, + "from": { + "type": "string" + }, + "measurement": { + "$ref": "#/$defs/Measurement" + }, + "name": { + "type": "string" + }, + "quality": { + "$ref": "#/$defs/ResultQuality" + }, + "start_selector": { + "type": "string" + }, + "to": { + "type": "string" + }, + "warnings": { + "type": "array", + "default": [], + "items": { + "$ref": "#/$defs/Warning" + } + } + }, + "additionalProperties": false, + "required": [ + "name", + "from", + "to", + "start_selector", + "end_selector", + "measurement", + "quality", + "correlation", + "clock", + "evidence" + ] + }, + "HostEvent": { + "type": "object", + "properties": { + "arguments": { + "type": "array", + "default": [], + "items": { + "$ref": "#/$defs/ArgumentValue" + } + }, + "binary_path": { + "type": [ + "string", + "null" + ] + }, + "build_id": { + "type": [ + "string", + "null" + ] + }, + "offset": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "probe_kind": { + "$ref": "#/$defs/HostProbeKind" + }, + "return_value": { + "type": [ + "integer", + "null" + ], + "format": "int64" + }, + "symbol": { + "type": [ + "string", + "null" + ] + }, + "symbol_demangled": { + "type": [ + "string", + "null" + ], + "default": null + } + }, + "additionalProperties": false, + "required": [ + "probe_kind" + ] + }, + "HostProbeKind": { + "type": "string", + "enum": [ + "uprobe", + "uretprobe", + "kprobe", + "kretprobe", + "tracepoint", + "syscall", + "usdt" + ] + }, + "LatencyRank": { + "type": "string", + "enum": [ + "min", + "p50", + "p90", + "p99", + "max" + ] + }, + "LatencyStatistics": { + "type": "object", + "properties": { + "max": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "mean": { + "type": "number", + "format": "double" + }, + "min": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "p50": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "p90": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "p95": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "p99": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "min", + "mean", + "p50", + "p90", + "p95", + "p99", + "max" + ] + }, + "MatchedEventPair": { + "type": "object", + "properties": { + "end": { + "$ref": "#/$defs/Event" + }, + "latency_ns": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "start": { + "$ref": "#/$defs/Event" + } + }, + "additionalProperties": false, + "required": [ + "start", + "end", + "latency_ns" + ] + }, + "Measurement": { + "type": "object", + "properties": { + "latency_ns": { + "$ref": "#/$defs/LatencyStatistics" + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "samples": { + "$ref": "#/$defs/SampleSummary" + } + }, + "additionalProperties": false, + "required": [ + "samples", + "latency_ns" + ] + }, + "MemcpyKind": { + "type": "string", + "enum": [ + "HtoD", + "DtoH", + "DtoD", + "HtoH", + "PtoP", + "unknown" + ] + }, + "NvtxEvent": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "name_complete": { + "type": "boolean" + }, + "range_id": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "range_kind": { + "$ref": "#/$defs/NvtxRangeKind" + }, + "start_tid": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "name", + "name_complete", + "range_kind", + "range_id", + "start_tid" + ] + }, + "NvtxRangeKind": { + "type": "string", + "enum": [ + "thread", + "process" + ] + }, + "QualityIssue": { + "type": "object", + "properties": { + "code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "severity": { + "$ref": "#/$defs/QualitySeverity" + } + }, + "additionalProperties": false, + "required": [ + "code", + "severity", + "message" + ] + }, + "QualitySeverity": { + "type": "string", + "enum": [ + "info", + "caution" + ] + }, + "QualityVerdict": { + "oneOf": [ + { + "description": "No caution-level issue; info issues may still be listed.", + "type": "string", + "const": "good" + }, + { + "description": "At least one issue limits how the latency may be interpreted.", + "type": "string", + "const": "caution" + } + ] + }, + "ResultQuality": { + "description": "Deterministic interpretation of the result's quality fields.", + "type": "object", + "properties": { + "issues": { + "type": "array", + "items": { + "$ref": "#/$defs/QualityIssue" + } + }, + "verdict": { + "$ref": "#/$defs/QualityVerdict" + } + }, + "additionalProperties": false, + "required": [ + "verdict", + "issues" + ] + }, + "SampleSummary": { + "type": "object", + "properties": { + "ambiguous": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "dropped": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "matched": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "unmatched_end": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "unmatched_start": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "matched", + "unmatched_start", + "unmatched_end", + "ambiguous", + "dropped" + ] + }, + "SchemaVersion": { + "type": "string", + "enum": [ + "2.0" + ] + }, + "SchemaVersion3": { + "description": "Version of records whose shape changed in xprobe 0.6.\n\nRecords keep their own version so unchanged contracts, such as Event JSONL\nartifacts, remain readable at `2.0`.", + "type": "string", + "enum": [ + "3.0" + ] + }, + "SessionStatus": { + "type": "string", + "enum": [ + "completed", + "timed_out", + "cancelled", + "failed" + ] + }, + "TargetIdentity": { + "type": "object", + "properties": { + "pid": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "process_start_time": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "pid", + "process_start_time" + ] + }, + "Warning": { + "type": "object", + "properties": { + "code": { + "type": "string" + }, + "details": { + "type": "object", + "additionalProperties": true, + "default": {} + }, + "message": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "code", + "message" + ] + } + } +} diff --git a/schemas/measurement-graph-spec.schema.json b/schemas/measurement-graph-spec.schema.json new file mode 100644 index 0000000..e42de36 --- /dev/null +++ b/schemas/measurement-graph-spec.schema.json @@ -0,0 +1,135 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "MeasurementGraphSpec", + "description": "Several start/end edges measured from one bounded capture of one process.\n\nEndpoints are named once and shared by edges. Each edge is correlated\nindependently; a graph never joins edges into per-request paths.", + "type": "object", + "properties": { + "duration_ms": { + "description": "Stop collection after this window; required unless every edge sets\n`samples`.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "edges": { + "type": "array", + "items": { + "$ref": "#/$defs/GraphEdgeSpec" + } + }, + "endpoints": { + "description": "Endpoint name to event selector.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "max_events": { + "description": "Record capacity of the whole shared capture.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "schema_version": { + "$ref": "#/$defs/SchemaVersion3" + }, + "target": { + "$ref": "#/$defs/TargetIdentity" + }, + "timeout_ms": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "schema_version", + "target", + "endpoints", + "edges", + "timeout_ms" + ], + "$defs": { + "GraphEdgeSpec": { + "type": "object", + "properties": { + "from": { + "type": "string" + }, + "match_policy": { + "$ref": "#/$defs/MatchPolicy" + }, + "name": { + "type": "string" + }, + "samples": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "to": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "name", + "from", + "to", + "match_policy" + ] + }, + "MatchPolicy": { + "type": "string", + "enum": [ + "exact", + "first_after", + "nearest", + "stack_nested", + "stream_order" + ] + }, + "SchemaVersion3": { + "description": "Version of records whose shape changed in xprobe 0.6.\n\nRecords keep their own version so unchanged contracts, such as Event JSONL\nartifacts, remain readable at `2.0`.", + "type": "string", + "enum": [ + "3.0" + ] + }, + "TargetIdentity": { + "type": "object", + "properties": { + "pid": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "process_start_time": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "pid", + "process_start_time" + ] + } + } +} diff --git a/schemas/measurement-graph-validate.schema.json b/schemas/measurement-graph-validate.schema.json new file mode 100644 index 0000000..50b2bdd --- /dev/null +++ b/schemas/measurement-graph-validate.schema.json @@ -0,0 +1,635 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "MeasurementGraphValidationResult", + "description": "Read-only validation of every endpoint and edge in a graph.", + "type": "object", + "properties": { + "edges": { + "type": "array", + "items": { + "$ref": "#/$defs/GraphEdgeValidation" + } + }, + "endpoints": { + "type": "array", + "items": { + "$ref": "#/$defs/GraphEndpointValidation" + } + }, + "issues": { + "type": "array", + "items": { + "$ref": "#/$defs/GraphValidationIssue" + } + }, + "ok": { + "type": "boolean" + }, + "requirements": { + "description": "Requirements merged across every edge.", + "$ref": "#/$defs/ValidationRequirements" + }, + "schema_version": { + "$ref": "#/$defs/SchemaVersion3" + }, + "target": { + "$ref": "#/$defs/TargetIdentity" + }, + "valid": { + "description": "True only when every endpoint and edge is valid.", + "type": "boolean" + }, + "warnings": { + "type": "array", + "default": [], + "items": { + "$ref": "#/$defs/Warning" + } + } + }, + "additionalProperties": false, + "required": [ + "schema_version", + "ok", + "valid", + "target", + "endpoints", + "edges", + "requirements", + "issues" + ], + "$defs": { + "AgentActivation": { + "type": "string", + "enum": [ + "not_required", + "already_loaded", + "injection_required", + "startup_required" + ] + }, + "ElfObjectKind": { + "type": "string", + "enum": [ + "executable", + "position_independent_executable", + "shared_library" + ] + }, + "EndpointSource": { + "type": "string", + "enum": [ + "host", + "cuda" + ] + }, + "ErrorCode": { + "type": "string", + "enum": [ + "PERMISSION_DENIED", + "TARGET_NOT_FOUND", + "TARGET_EXITED", + "TARGET_REUSED", + "AMBIGUOUS_TARGET", + "SYMBOL_NOT_FOUND", + "BINARY_NOT_MAPPED", + "INVALID_EVENT_SELECTOR", + "INVALID_CORRELATION_POLICY", + "CUPTI_NOT_AVAILABLE", + "CUPTI_AGENT_NOT_LOADED", + "CUDA_CONTEXT_NOT_FOUND", + "UNSUPPORTED_CUDA_VERSION", + "EVENT_RATE_TOO_HIGH", + "SESSION_LIMIT_EXCEEDED", + "NO_MATCHED_SAMPLES", + "HIGH_UNMATCHED_RATE", + "EVENTS_DROPPED", + "CLOCK_ALIGNMENT_FAILED", + "TRACE_EXPORT_FAILED", + "CLEANUP_FAILED", + "INTERNAL" + ] + }, + "EventType": { + "type": "string", + "enum": [ + "host_function_entry", + "host_function_exit", + "kernel_function_entry", + "kernel_function_exit", + "tracepoint", + "syscall_entry", + "syscall_exit", + "cuda_api_entry", + "cuda_api_exit", + "gpu_kernel_start", + "gpu_kernel_end", + "gpu_memcpy_start", + "gpu_memcpy_end", + "gpu_memset_start", + "gpu_memset_end", + "nvtx_range_start", + "nvtx_range_end", + "python_gc_start", + "python_gc_end", + "marker" + ] + }, + "GraphEdgeValidation": { + "type": "object", + "properties": { + "from": { + "type": "string" + }, + "match_policy": { + "$ref": "#/$defs/MatchPolicy" + }, + "name": { + "type": "string" + }, + "policy_recommendation": { + "$ref": "#/$defs/PolicyRecommendation" + }, + "to": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "name", + "from", + "to", + "match_policy", + "policy_recommendation" + ] + }, + "GraphEndpointValidation": { + "type": "object", + "properties": { + "endpoint": { + "$ref": "#/$defs/ValidatedEndpoint" + }, + "name": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "name", + "endpoint" + ] + }, + "GraphIssueScope": { + "type": "string", + "enum": [ + "graph", + "endpoint", + "edge" + ] + }, + "GraphValidationIssue": { + "description": "An issue attached to the graph, one endpoint, or one edge.", + "type": "object", + "properties": { + "code": { + "$ref": "#/$defs/ErrorCode" + }, + "message": { + "type": "string" + }, + "name": { + "description": "Endpoint or edge name; `null` for graph-level issues.", + "type": [ + "string", + "null" + ] + }, + "scope": { + "$ref": "#/$defs/GraphIssueScope" + } + }, + "additionalProperties": false, + "required": [ + "scope", + "code", + "message" + ] + }, + "HostProbeKind": { + "type": "string", + "enum": [ + "uprobe", + "uretprobe", + "kprobe", + "kretprobe", + "tracepoint", + "syscall", + "usdt" + ] + }, + "MatchPolicy": { + "type": "string", + "enum": [ + "exact", + "first_after", + "nearest", + "stack_nested", + "stream_order" + ] + }, + "MemcpyKind": { + "type": "string", + "enum": [ + "HtoD", + "DtoH", + "DtoD", + "HtoH", + "PtoP", + "unknown" + ] + }, + "PolicyRecommendation": { + "type": "object", + "properties": { + "compatible_policies": { + "type": "array", + "items": { + "$ref": "#/$defs/MatchPolicy" + } + }, + "policy": { + "$ref": "#/$defs/MatchPolicy" + }, + "reason": { + "$ref": "#/$defs/PolicyRecommendationReason" + } + }, + "additionalProperties": false, + "required": [ + "policy", + "reason", + "compatible_policies" + ] + }, + "PolicyRecommendationReason": { + "type": "string", + "enum": [ + "deterministic_correlation_key", + "host_call_frame", + "cuda_stream_order", + "temporal_order_only" + ] + }, + "ProcessMapping": { + "type": "object", + "properties": { + "end_address": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "file_offset": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "start_address": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "start_address", + "end_address", + "file_offset" + ] + }, + "ResolvedCudaSelector": { + "type": "object", + "properties": { + "api_domain": { + "type": [ + "string", + "null" + ] + }, + "api_name": { + "type": [ + "string", + "null" + ] + }, + "event_type": { + "$ref": "#/$defs/EventType" + }, + "kernel_name_regex": { + "type": [ + "string", + "null" + ] + }, + "memcpy_kind": { + "anyOf": [ + { + "$ref": "#/$defs/MemcpyKind" + }, + { + "type": "null" + } + ] + }, + "nvtx_name_regex": { + "type": [ + "string", + "null" + ], + "default": null + } + }, + "additionalProperties": false, + "required": [ + "event_type" + ] + }, + "ResolvedLinuxSelector": { + "type": "object", + "properties": { + "binary_path": { + "type": [ + "string", + "null" + ] + }, + "category": { + "type": "string" + }, + "event_type": { + "$ref": "#/$defs/EventType" + }, + "name": { + "type": "string" + }, + "probe_kind": { + "$ref": "#/$defs/HostProbeKind" + }, + "provider": { + "type": [ + "string", + "null" + ] + }, + "syscall_number": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "event_type", + "probe_kind", + "category", + "name" + ] + }, + "ResolvedProbe": { + "type": "object", + "properties": { + "binary_path": { + "type": "string" + }, + "build_id": { + "type": [ + "string", + "null" + ] + }, + "file_offset": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "mapping": { + "$ref": "#/$defs/ProcessMapping" + }, + "object_kind": { + "$ref": "#/$defs/ElfObjectKind" + }, + "ok": { + "type": "boolean" + }, + "probe_kind": { + "$ref": "#/$defs/HostProbeKind" + }, + "runtime_address": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "schema_version": { + "$ref": "#/$defs/SchemaVersion" + }, + "selector": { + "type": "string" + }, + "symbol": { + "type": [ + "string", + "null" + ] + }, + "symbol_demangled": { + "type": [ + "string", + "null" + ] + }, + "symbol_size": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "symbol_virtual_address": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "target": { + "$ref": "#/$defs/TargetIdentity" + } + }, + "additionalProperties": false, + "required": [ + "schema_version", + "ok", + "target", + "selector", + "binary_path", + "object_kind", + "probe_kind", + "file_offset", + "runtime_address", + "mapping" + ] + }, + "SchemaVersion": { + "type": "string", + "enum": [ + "2.0" + ] + }, + "SchemaVersion3": { + "description": "Version of records whose shape changed in xprobe 0.6.\n\nRecords keep their own version so unchanged contracts, such as Event JSONL\nartifacts, remain readable at `2.0`.", + "type": "string", + "enum": [ + "3.0" + ] + }, + "TargetIdentity": { + "type": "object", + "properties": { + "pid": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "process_start_time": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "pid", + "process_start_time" + ] + }, + "ValidatedEndpoint": { + "type": "object", + "properties": { + "collectable": { + "type": "boolean" + }, + "cuda": { + "anyOf": [ + { + "$ref": "#/$defs/ResolvedCudaSelector" + }, + { + "type": "null" + } + ] + }, + "event_type": { + "$ref": "#/$defs/EventType" + }, + "host": { + "anyOf": [ + { + "$ref": "#/$defs/ResolvedProbe" + }, + { + "type": "null" + } + ] + }, + "linux": { + "anyOf": [ + { + "$ref": "#/$defs/ResolvedLinuxSelector" + }, + { + "type": "null" + } + ] + }, + "selector": { + "type": "string" + }, + "source": { + "$ref": "#/$defs/EndpointSource" + } + }, + "additionalProperties": false, + "required": [ + "selector", + "source", + "event_type", + "collectable" + ] + }, + "ValidationRequirements": { + "type": "object", + "properties": { + "agent_activation": { + "$ref": "#/$defs/AgentActivation" + }, + "needs_clock_alignment": { + "type": "boolean" + }, + "needs_cupti": { + "type": "boolean" + }, + "needs_cupti_activity": { + "type": "boolean" + }, + "needs_cupti_callback": { + "type": "boolean" + }, + "needs_ebpf": { + "type": "boolean" + }, + "needs_nvtx": { + "type": "boolean", + "default": false + }, + "target_mutation": { + "type": "boolean" + } + }, + "additionalProperties": false, + "required": [ + "needs_ebpf", + "needs_cupti", + "needs_cupti_callback", + "needs_cupti_activity", + "needs_clock_alignment", + "agent_activation", + "target_mutation" + ] + }, + "Warning": { + "type": "object", + "properties": { + "code": { + "type": "string" + }, + "details": { + "type": "object", + "additionalProperties": true, + "default": {} + }, + "message": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "code", + "message" + ] + } + } +} diff --git a/xprobe/core/src/graph.rs b/xprobe/core/src/graph.rs new file mode 100644 index 0000000..95a0bf7 --- /dev/null +++ b/xprobe/core/src/graph.rs @@ -0,0 +1,425 @@ +//! Read-only validation of multi-edge measurement graphs. + +use std::{collections::BTreeSet, error::Error, fmt}; + +use xprobe_protocol::{ + AgentActivation, ErrorCode, GRAPH_MAX_EDGES, GRAPH_MAX_ENDPOINTS, GRAPH_MAX_LINUX_ENDPOINTS, + GraphEdgeValidation, GraphEndpointValidation, GraphIssueScope, GraphValidationIssue, + MatchPolicy, MeasurementGraphSpec, MeasurementGraphValidationResult, ProcessReport, + SchemaVersion3, ValidatedEndpoint, ValidationRequirements, +}; + +use crate::{ + inspect::{self, InspectError}, + validate::{self, ValidateError}, +}; + +#[derive(Debug)] +pub enum GraphValidateError { + Inspect(InspectError), + Edge { edge: String, source: ValidateError }, +} + +impl GraphValidateError { + #[must_use] + pub fn code(&self) -> ErrorCode { + match self { + Self::Inspect(error) => error.code(), + Self::Edge { source, .. } => source.code(), + } + } + + #[must_use] + pub fn recoverable(&self) -> bool { + match self { + Self::Inspect(error) => error.recoverable(), + Self::Edge { source, .. } => source.recoverable(), + } + } + + #[must_use] + pub fn edge(&self) -> Option<&str> { + match self { + Self::Inspect(_) => None, + Self::Edge { edge, .. } => Some(edge), + } + } +} + +impl fmt::Display for GraphValidateError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Inspect(error) => error.fmt(formatter), + Self::Edge { edge, source } => write!(formatter, "edge {edge:?}: {source}"), + } + } +} + +impl Error for GraphValidateError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Inspect(error) => Some(error), + Self::Edge { source, .. } => Some(source), + } + } +} + +impl From for GraphValidateError { + fn from(error: InspectError) -> Self { + Self::Inspect(error) + } +} + +/// Text accepted by single-edge validation for a policy. +#[must_use] +pub const fn policy_text(policy: MatchPolicy) -> &'static str { + match policy { + MatchPolicy::Exact => "exact", + MatchPolicy::FirstAfter => "first-after", + MatchPolicy::Nearest => "nearest", + MatchPolicy::StackNested => "stack-nested", + MatchPolicy::StreamOrder => "stream-order", + } +} + +/// Validate a graph without attaching. +/// +/// Structural problems, unknown endpoint names, collector slot limits, and +/// every single-edge issue are returned as scoped issues. +/// +/// # Errors +/// +/// Returns [`GraphValidateError`] when target identity changes or an edge's +/// selector cannot be resolved at all, naming the edge. +pub fn run( + report: &ProcessReport, + spec: &MeasurementGraphSpec, +) -> Result { + inspect::verify_target(&report.target)?; + let mut issues = structural_issues(report, spec); + let mut endpoints: Vec = Vec::new(); + let mut edges = Vec::new(); + let mut requirements: Option = None; + let mut warnings = Vec::new(); + let resolvable = issues + .iter() + .all(|issue| issue.scope != GraphIssueScope::Graph); + for edge in spec.edges.iter().filter(|_| resolvable) { + let (Some(from), Some(to)) = (spec.endpoints.get(&edge.from), spec.endpoints.get(&edge.to)) + else { + continue; + }; + let result = + validate::run(report, from, to, policy_text(edge.match_policy)).map_err(|source| { + GraphValidateError::Edge { + edge: edge.name.clone(), + source, + } + })?; + for (name, endpoint) in [(&edge.from, &result.start), (&edge.to, &result.end)] { + if !endpoints.iter().any(|existing| &existing.name == name) { + endpoints.push(GraphEndpointValidation { + name: name.clone(), + endpoint: endpoint.clone(), + }); + } + } + issues.extend(result.issues.into_iter().map(|issue| GraphValidationIssue { + scope: GraphIssueScope::Edge, + name: Some(edge.name.clone()), + code: issue.code, + message: issue.message, + })); + warnings.extend(result.warnings); + requirements = Some(match requirements { + None => result.requirements, + Some(merged) => merge_requirements(&merged, &result.requirements), + }); + edges.push(GraphEdgeValidation { + name: edge.name.clone(), + from: edge.from.clone(), + to: edge.to.clone(), + match_policy: edge.match_policy, + policy_recommendation: result.policy_recommendation, + }); + } + endpoints.sort_by(|left, right| left.name.cmp(&right.name)); + let public = endpoints + .iter() + .map(|endpoint| &endpoint.endpoint) + .collect::>(); + issues.extend(slot_issues(&public)); + warnings.dedup_by(|left, right| left.code == right.code && left.message == right.message); + inspect::verify_target(&report.target)?; + Ok(MeasurementGraphValidationResult { + schema_version: SchemaVersion3::current(), + ok: true, + valid: issues.is_empty(), + target: report.target.clone(), + endpoints, + edges, + requirements: requirements.unwrap_or_else(empty_requirements), + issues, + warnings, + }) +} + +fn graph_issue(message: String) -> GraphValidationIssue { + GraphValidationIssue { + scope: GraphIssueScope::Graph, + name: None, + code: ErrorCode::InvalidEventSelector, + message, + } +} + +fn structural_issues( + report: &ProcessReport, + spec: &MeasurementGraphSpec, +) -> Vec { + let mut issues = Vec::new(); + if spec.target != report.target { + issues.push(GraphValidationIssue { + scope: GraphIssueScope::Graph, + name: None, + code: ErrorCode::TargetReused, + message: "graph target identity does not match the process".to_owned(), + }); + } + if spec.edges.is_empty() || spec.edges.len() > GRAPH_MAX_EDGES { + issues.push(graph_issue(format!( + "a graph needs between 1 and {GRAPH_MAX_EDGES} edges, found {}", + spec.edges.len() + ))); + } + if spec.endpoints.is_empty() || spec.endpoints.len() > GRAPH_MAX_ENDPOINTS { + issues.push(graph_issue(format!( + "a graph needs between 1 and {GRAPH_MAX_ENDPOINTS} endpoints, found {}", + spec.endpoints.len() + ))); + } + if spec.timeout_ms == 0 + || spec.duration_ms == Some(0) + || spec.max_events == Some(0) + || spec.edges.iter().any(|edge| edge.samples == Some(0)) + { + issues.push(GraphValidationIssue { + scope: GraphIssueScope::Graph, + name: None, + code: ErrorCode::SessionLimitExceeded, + message: "timeout, duration, max_events, and samples must be positive".to_owned(), + }); + } + if spec.duration_ms.is_none() && spec.edges.iter().any(|edge| edge.samples.is_none()) { + issues.push(GraphValidationIssue { + scope: GraphIssueScope::Graph, + name: None, + code: ErrorCode::SessionLimitExceeded, + message: "set duration_ms or samples on every edge so the capture is bounded" + .to_owned(), + }); + } + let mut names = BTreeSet::new(); + let mut used = BTreeSet::new(); + for edge in &spec.edges { + if edge.name.is_empty() || !names.insert(edge.name.as_str()) { + issues.push(GraphValidationIssue { + scope: GraphIssueScope::Edge, + name: Some(edge.name.clone()), + code: ErrorCode::InvalidEventSelector, + message: "edge names must be non-empty and unique".to_owned(), + }); + } + for endpoint in [&edge.from, &edge.to] { + used.insert(endpoint.as_str()); + if !spec.endpoints.contains_key(endpoint) { + issues.push(GraphValidationIssue { + scope: GraphIssueScope::Edge, + name: Some(edge.name.clone()), + code: ErrorCode::InvalidEventSelector, + message: format!("edge references unknown endpoint {endpoint:?}"), + }); + } + } + } + for name in spec + .endpoints + .keys() + .filter(|name| !used.contains(name.as_str())) + { + issues.push(GraphValidationIssue { + scope: GraphIssueScope::Endpoint, + name: Some(name.clone()), + code: ErrorCode::InvalidEventSelector, + message: "endpoint is not used by any edge".to_owned(), + }); + } + issues +} + +/// Syscall, tracepoint, and USDT endpoints share the Linux collector's slots. +fn slot_issues(endpoints: &[&ValidatedEndpoint]) -> Vec { + let linux = endpoints + .iter() + .filter(|endpoint| endpoint.linux.is_some()) + .map(|endpoint| endpoint.selector.as_str()) + .collect::>(); + if linux.len() <= GRAPH_MAX_LINUX_ENDPOINTS { + return Vec::new(); + } + vec![GraphValidationIssue { + scope: GraphIssueScope::Graph, + name: None, + code: ErrorCode::SessionLimitExceeded, + message: format!( + "a graph supports at most {GRAPH_MAX_LINUX_ENDPOINTS} syscall, tracepoint, or USDT endpoints, found {}", + linux.len() + ), + }] +} + +const fn activation_rank(activation: AgentActivation) -> u8 { + match activation { + AgentActivation::NotRequired => 0, + AgentActivation::AlreadyLoaded => 1, + AgentActivation::InjectionRequired => 2, + AgentActivation::StartupRequired => 3, + } +} + +fn merge_requirements( + left: &ValidationRequirements, + right: &ValidationRequirements, +) -> ValidationRequirements { + let agent_activation = + if activation_rank(right.agent_activation) > activation_rank(left.agent_activation) { + right.agent_activation + } else { + left.agent_activation + }; + ValidationRequirements { + needs_ebpf: left.needs_ebpf || right.needs_ebpf, + needs_cupti: left.needs_cupti || right.needs_cupti, + needs_cupti_callback: left.needs_cupti_callback || right.needs_cupti_callback, + needs_cupti_activity: left.needs_cupti_activity || right.needs_cupti_activity, + needs_nvtx: left.needs_nvtx || right.needs_nvtx, + needs_clock_alignment: left.needs_clock_alignment || right.needs_clock_alignment, + agent_activation, + target_mutation: agent_activation == AgentActivation::InjectionRequired, + } +} + +const fn empty_requirements() -> ValidationRequirements { + ValidationRequirements { + needs_ebpf: false, + needs_cupti: false, + needs_cupti_callback: false, + needs_cupti_activity: false, + needs_nvtx: false, + needs_clock_alignment: false, + agent_activation: AgentActivation::NotRequired, + target_mutation: false, + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use xprobe_protocol::{ + AgentActivation, GraphEdgeSpec, GraphIssueScope, MatchPolicy, MeasurementGraphSpec, + SchemaVersion3, + }; + + use super::{merge_requirements, structural_issues}; + use crate::inspect; + + fn spec(edges: &[(&str, &str, &str, Option)]) -> MeasurementGraphSpec { + let report = inspect::run(std::process::id()).unwrap(); + MeasurementGraphSpec { + schema_version: SchemaVersion3::current(), + name: None, + target: report.target, + endpoints: BTreeMap::from([ + ("a".to_owned(), "cuda:kernel_start".to_owned()), + ("b".to_owned(), "cuda:kernel_end".to_owned()), + ]), + edges: edges + .iter() + .map(|(name, from, to, samples)| GraphEdgeSpec { + name: (*name).to_owned(), + from: (*from).to_owned(), + to: (*to).to_owned(), + match_policy: MatchPolicy::Exact, + samples: *samples, + }) + .collect(), + duration_ms: None, + timeout_ms: 1000, + max_events: None, + } + } + + #[test] + fn accepts_a_bounded_well_formed_graph() { + let report = inspect::run(std::process::id()).unwrap(); + assert_eq!( + structural_issues(&report, &spec(&[("k", "a", "b", Some(10))])), + [] + ); + } + + #[test] + fn reports_structural_problems_with_scope() { + let report = inspect::run(std::process::id()).unwrap(); + let graph = spec(&[("k", "a", "c", None), ("k", "a", "b", Some(1))]); + let issues = structural_issues(&report, &graph) + .into_iter() + .map(|issue| (issue.scope, issue.name, issue.message)) + .collect::>(); + assert!( + issues + .iter() + .any(|(scope, _, message)| *scope == GraphIssueScope::Graph + && message.contains("bounded")) + ); + assert!( + issues + .iter() + .any(|(scope, name, message)| *scope == GraphIssueScope::Edge + && name.as_deref() == Some("k") + && message.contains("unique")) + ); + assert!( + issues + .iter() + .any(|(_, _, message)| message.contains("unknown endpoint \"c\"")) + ); + } + + #[test] + fn merged_requirements_take_the_strongest_activation() { + let base = xprobe_protocol::ValidationRequirements { + needs_ebpf: true, + needs_cupti: false, + needs_cupti_callback: false, + needs_cupti_activity: false, + needs_nvtx: false, + needs_clock_alignment: false, + agent_activation: AgentActivation::NotRequired, + target_mutation: false, + }; + let cuda = xprobe_protocol::ValidationRequirements { + needs_ebpf: false, + needs_cupti: true, + needs_cupti_activity: true, + agent_activation: AgentActivation::InjectionRequired, + target_mutation: true, + ..base.clone() + }; + let merged = merge_requirements(&base, &cuda); + assert!(merged.needs_ebpf && merged.needs_cupti && merged.needs_cupti_activity); + assert_eq!(merged.agent_activation, AgentActivation::InjectionRequired); + assert!(merged.target_mutation); + } +} diff --git a/xprobe/core/src/lib.rs b/xprobe/core/src/lib.rs index ddf66bd..2c691ec 100644 --- a/xprobe/core/src/lib.rs +++ b/xprobe/core/src/lib.rs @@ -8,6 +8,7 @@ pub mod discover; pub mod doctor; pub mod gpu_aggregate; pub mod gpu_health; +pub mod graph; pub mod inject; pub mod inspect; pub mod resolve; diff --git a/xprobe/protocol/src/graph.rs b/xprobe/protocol/src/graph.rs new file mode 100644 index 0000000..9c6018d --- /dev/null +++ b/xprobe/protocol/src/graph.rs @@ -0,0 +1,137 @@ +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::{ + ClockQuality, CollectionSummary, CorrelationSummary, ErrorCode, Evidence, EvidenceArtifact, + MatchPolicy, Measurement, PolicyRecommendation, ResultQuality, SchemaVersion3, SessionStatus, + TargetIdentity, ValidatedEndpoint, ValidationRequirements, Warning, +}; + +/// Endpoints and edges allowed in one bounded graph capture. +pub const GRAPH_MAX_ENDPOINTS: usize = 8; +pub const GRAPH_MAX_EDGES: usize = 8; +/// Linux syscall, tracepoint, and USDT endpoints share two collector slots. +pub const GRAPH_MAX_LINUX_ENDPOINTS: usize = 2; + +/// Several start/end edges measured from one bounded capture of one process. +/// +/// Endpoints are named once and shared by edges. Each edge is correlated +/// independently; a graph never joins edges into per-request paths. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct MeasurementGraphSpec { + pub schema_version: SchemaVersion3, + pub name: Option, + pub target: TargetIdentity, + /// Endpoint name to event selector. + pub endpoints: BTreeMap, + pub edges: Vec, + /// Stop collection after this window; required unless every edge sets + /// `samples`. + pub duration_ms: Option, + pub timeout_ms: u64, + /// Record capacity of the whole shared capture. + pub max_events: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct GraphEdgeSpec { + pub name: String, + pub from: String, + pub to: String, + pub match_policy: MatchPolicy, + pub samples: Option, +} + +/// Read-only validation of every endpoint and edge in a graph. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct MeasurementGraphValidationResult { + pub schema_version: SchemaVersion3, + pub ok: bool, + /// True only when every endpoint and edge is valid. + pub valid: bool, + pub target: TargetIdentity, + pub endpoints: Vec, + pub edges: Vec, + /// Requirements merged across every edge. + pub requirements: ValidationRequirements, + pub issues: Vec, + #[serde(default)] + pub warnings: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct GraphEndpointValidation { + pub name: String, + pub endpoint: ValidatedEndpoint, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct GraphEdgeValidation { + pub name: String, + pub from: String, + pub to: String, + pub match_policy: MatchPolicy, + pub policy_recommendation: PolicyRecommendation, +} + +/// An issue attached to the graph, one endpoint, or one edge. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct GraphValidationIssue { + pub scope: GraphIssueScope, + /// Endpoint or edge name; `null` for graph-level issues. + pub name: Option, + pub code: ErrorCode, + pub message: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum GraphIssueScope { + Graph, + Endpoint, + Edge, +} + +/// Per-edge results from one shared bounded capture. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct MeasurementGraphResult { + pub schema_version: SchemaVersion3, + pub ok: bool, + pub session_id: String, + pub status: SessionStatus, + pub name: Option, + pub target: TargetIdentity, + pub edges: Vec, + /// The shared capture every edge was correlated from. + pub collection: CollectionSummary, + pub artifact: Option, + #[serde(default)] + pub warnings: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct GraphEdgeResult { + pub name: String, + pub from: String, + pub to: String, + pub start_selector: String, + pub end_selector: String, + pub measurement: Measurement, + pub quality: ResultQuality, + pub correlation: CorrelationSummary, + pub clock: ClockQuality, + /// Exemplars and optional full pairs; the artifact is graph-level. + pub evidence: Evidence, + #[serde(default)] + pub warnings: Vec, +} diff --git a/xprobe/protocol/src/lib.rs b/xprobe/protocol/src/lib.rs index c18bc0d..98acfb6 100644 --- a/xprobe/protocol/src/lib.rs +++ b/xprobe/protocol/src/lib.rs @@ -7,6 +7,7 @@ mod discover; mod error; mod event; mod export; +mod graph; mod measurement; mod process; mod resolve; @@ -33,6 +34,12 @@ pub use event::{ HostProbeKind, MemcpyKind, NvtxEvent, NvtxRangeKind, }; pub use export::{ExportFormat, TraceExportResult}; +pub use graph::{ + GRAPH_MAX_EDGES, GRAPH_MAX_ENDPOINTS, GRAPH_MAX_LINUX_ENDPOINTS, GraphEdgeResult, + GraphEdgeSpec, GraphEdgeValidation, GraphEndpointValidation, GraphIssueScope, + GraphValidationIssue, MeasurementGraphResult, MeasurementGraphSpec, + MeasurementGraphValidationResult, +}; pub use measurement::{ AggregateActivity, AggregateCollectionSummary, AggregateDuration, AggregateGroup, AggregateInventory, AggregateInventoryResult, CaptureCompleteness, ClockQuality, diff --git a/xprobe/protocol/src/schema.rs b/xprobe/protocol/src/schema.rs index 4b61389..ff57259 100644 --- a/xprobe/protocol/src/schema.rs +++ b/xprobe/protocol/src/schema.rs @@ -3,13 +3,14 @@ use schemars::{Schema, schema_for}; use crate::{ AggregateInventoryResult, CapabilityReport, CpuSampleInventoryResult, CpuSamplingSpec, CpuSamplingValidationResult, DiscoveryResult, ErrorResponse, Event, HostCaptureResult, + MeasurementGraphResult, MeasurementGraphSpec, MeasurementGraphValidationResult, MeasurementResult, MeasurementSpec, ProcessReport, ResolvedProbe, SyscallAggregateResult, SyscallAggregateSpec, SyscallAggregateValidationResult, TraceExportResult, TriageResult, ValidationResult, }; #[must_use] -pub fn generated_schemas() -> [(&'static str, Schema); 19] { +pub fn generated_schemas() -> [(&'static str, Schema); 22] { [ ("event.schema.json", schema_for!(Event)), ("error.schema.json", schema_for!(ErrorResponse)), @@ -54,5 +55,17 @@ pub fn generated_schemas() -> [(&'static str, Schema); 19] { ("validate.schema.json", schema_for!(ValidationResult)), ("trace-export.schema.json", schema_for!(TraceExportResult)), ("triage-result.schema.json", schema_for!(TriageResult)), + ( + "measurement-graph-spec.schema.json", + schema_for!(MeasurementGraphSpec), + ), + ( + "measurement-graph-validate.schema.json", + schema_for!(MeasurementGraphValidationResult), + ), + ( + "measurement-graph-result.schema.json", + schema_for!(MeasurementGraphResult), + ), ] } From c7861a66194670d5462abe833345868721800c29 Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 14 Nov 2026 14:15:20 +0800 Subject: [PATCH 11/15] =?UTF-8?q?=E2=9C=A8=20feat:=20measure=20several=20e?= =?UTF-8?q?dges=20from=20one=20bounded=20capture?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `measure --spec graph.json` runs a measurement graph: named endpoints shared by up to eight edges, collected once and correlated per edge with the single-edge rules. `validate --spec graph.json` checks it read-only with graph, endpoint, and edge-scoped issues. Any edge failure fails the command with the edge named and keeps the capture artifact for offline re-correlation; edges are never joined into per-request paths. The multisource live test measures request-to-launch, launch-to-kernel, and kernel duration from one capture. --- docs/architecture.md | 11 +- docs/cli-contract.md | 55 +++ tests/integration/run-multisource-live.sh | 25 +- tests/integration/test_multisource.py | 36 ++ xprobe/cli/src/args.rs | 12 +- xprobe/cli/src/graph.rs | 395 ++++++++++++++++++++++ xprobe/cli/src/live.rs | 43 ++- xprobe/cli/src/main.rs | 26 ++ xprobe/cli/tests/validate.rs | 112 ++++++ 9 files changed, 698 insertions(+), 17 deletions(-) create mode 100644 xprobe/cli/src/graph.rs diff --git a/docs/architecture.md b/docs/architecture.md index 940d2bd..5ece33b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -133,7 +133,9 @@ per-CPU values and maps known x86_64 numbers to selector hints. CUDA events use an in-process CUPTI Agent. CUDA 12 and CUDA 13 builds share the same source and capture ABI but link their matching CUPTI SONAME. Loading the Agent creates a control socket; `measure` separately arms one fresh, -`--max-events`-bounded capture with filters for its CUDA endpoints. Runtime and +`--max-events`-bounded capture with up to eight filters for its CUDA endpoints +(control protocol v5 carries the active filter count, and hot paths scan only +active filters). Runtime and Driver callbacks plus kernel, memcpy, and memset activity are filtered before fixed records consume capacity. Callback hot paths do not perform blocking I/O or allocation. @@ -199,6 +201,13 @@ agent because unloading callback code in a live CUDA process is unsafe. ## Correlation and evidence +Every exact live measurement runs one shared capture loop over a plan of +validated endpoints and merged requirements. The loop owns attachment, polling, +timeout, CUPTI shutdown, cleanup, and artifact preservation; an evaluator owns +correlation and sample completion. A single edge and a measurement graph differ +only in their evaluator: a graph validates the merged capture once and +correlates each edge independently. + All sources normalize into the versioned `Event` type. `measure` supports: | Policy | Pairing | Confidence | diff --git a/docs/cli-contract.md b/docs/cli-contract.md index 90e6b3d..f4acfbd 100644 --- a/docs/cli-contract.md +++ b/docs/cli-contract.md @@ -127,6 +127,14 @@ and Python symbolization status. `validate --syscall-aggregate` checks the bounded eBPF aggregate path. Inventory modes do not take event-pair selectors. Results conform to `schemas/validate.schema.json`. +`validate --spec graph.json` validates a measurement graph without `--pid`; the +spec's target selects the process. It emits +`schemas/measurement-graph-validate.schema.json` (schema `3.0`) with each named +endpoint's resolution, each edge's policy recommendation, requirements merged +across edges, and issues scoped to the graph, an endpoint, or an edge. Unknown +or unused endpoints, duplicate edge names, missing bounds, and more than two +syscall, tracepoint, or USDT endpoints are issues, not command errors. + Supported policies are `exact`, `first-after`, `nearest`, `stack-nested`, and `stream-order`. Exact uses deterministic CUPTI correlation IDs, NVTX range kind/ID, one named syscall's per-thread entry/exit lifecycle, or one CPython GC @@ -368,6 +376,53 @@ error are never silently converted to high-quality results. Error `details` and `hints` carry selectors, policies, counters, clock domains, and explicit next actions where applicable; callers must not infer these fields from messages. +## Measurement graphs + +A graph measures several edges from one bounded capture of one process: + +```json +{ + "schema_version": "3.0", + "name": "request_path", + "target": {"pid": 4242, "process_start_time": 123456}, + "endpoints": { + "request": "uprobe:/srv/app/server:handle_request:entry", + "launch_exit": "cuda:runtime_api:cudaLaunchKernel:exit", + "kernel_start": "cuda:kernel_start:name~^flash", + "kernel_end": "cuda:kernel_end:name~^flash" + }, + "edges": [ + {"name": "request_to_launch", "from": "request", "to": "launch_exit", + "match_policy": "first_after", "samples": 100}, + {"name": "launch_to_kernel", "from": "launch_exit", "to": "kernel_start", + "match_policy": "exact", "samples": 100}, + {"name": "kernel_duration", "from": "kernel_start", "to": "kernel_end", + "match_policy": "exact", "samples": 100} + ], + "duration_ms": null, + "timeout_ms": 30000, + "max_events": 20000 +} +``` + +`measure --spec graph.json` accepts this `schemas/measurement-graph-spec.schema.json` +document. A graph has 1–8 named endpoints and 1–8 edges; every edge sets +`samples` or the graph sets `duration_ms`. Collectors attach once per unique +endpoint: uprobes have no further limit, CUDA endpoints share up to eight Agent +filters, and syscall, tracepoint, and USDT endpoints share two Linux collector +slots. `max_events` bounds the whole capture. Collection ends when every edge +reaches its samples or the duration elapses. + +The capture is validated once, then each edge is correlated independently with +the single-edge rules. A graph never joins edges into per-request paths and +never implies causality between edges. Results use +`schemas/measurement-graph-result.schema.json` (schema `3.0`): per-edge +statistics, quality verdict, correlation, clock quality, and evidence, one +shared collection summary, and the shared event artifact. Any invalid edge or +edge correlation failure fails the whole command with the edge name in the +message and `details.edge`; the captured events are still written to the +artifact so the caller can re-correlate edges with `measure --input`. + ## Compatibility Earlier low-level command spellings remain hidden during the pre-1.0 transition so diff --git a/tests/integration/run-multisource-live.sh b/tests/integration/run-multisource-live.sh index 622eb6c..f70cc68 100755 --- a/tests/integration/run-multisource-live.sh +++ b/tests/integration/run-multisource-live.sh @@ -18,6 +18,9 @@ live_capture="$1/live.jsonl" live_measurement="$1/live-measure.json" spec="${build_dir}/measurement-spec.json" spec_measurement="$1/spec-measure.json" +graph="${build_dir}/measurement-graph.json" +graph_validation="$1/graph-validate.json" +graph_measurement="$1/graph-measure.json" gpu_info="$1/gpu.txt" compute_capability=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | sed -n '1p') compute_arch=${compute_capability//./} @@ -87,6 +90,25 @@ printf '{"schema_version":"2.0","name":"spec_host_to_kernel","target":{"pid":%s, --cupti-socket "${snapshot_socket}" \ --json --non-interactive --no-color >"${spec_measurement}" +kernel='cuda:kernel_start:name~xprobe_multisource_kernel.*' +kernel_end='cuda:kernel_end:name~xprobe_multisource_kernel.*' +cat >"${graph}" <"${graph_validation}" +/workspace/target/debug/xprobe measure \ + --spec "${graph}" \ + --cupti-socket "${snapshot_socket}" \ + --events-out "$1/graph-events.jsonl" >"${graph_measurement}" + if [[ $(stat -c '%a' "${snapshot_socket}") != 600 ]]; then echo "CUPTI snapshot socket is not mode 0600" >&2 exit 1 @@ -100,4 +122,5 @@ touch "${stop}" wait "${target_pid}" trap - EXIT chmod 0644 "${live_capture}" \ - "${live_measurement}" "${spec_measurement}" "${gpu_info}" + "${live_measurement}" "${spec_measurement}" "${gpu_info}" \ + "${graph_validation}" "${graph_measurement}" "$1/graph-events.jsonl" diff --git a/tests/integration/test_multisource.py b/tests/integration/test_multisource.py index 9361cff..6b090ee 100755 --- a/tests/integration/test_multisource.py +++ b/tests/integration/test_multisource.py @@ -70,6 +70,16 @@ def main() -> None: assert spec_result["measurement"]["name"] == "spec_host_to_kernel" assert spec_result["measurement"]["samples"]["matched"] == 3 + graph_validation = json.loads( + (pathlib.Path(output_dir) / "graph-validate.json").read_text() + ) + graph_result = json.loads( + (pathlib.Path(output_dir) / "graph-measure.json").read_text() + ) + graph_events = ( + pathlib.Path(output_dir) / "graph-events.jsonl" + ).read_text().splitlines() + trace_path = pathlib.Path(output_dir) / "trace.json" exported = subprocess.run( [ @@ -102,6 +112,28 @@ def main() -> None: assert live_result["correlation"]["confidence"] == "heuristic" assert live_result["clock"]["alignment"] == "cupti_normalized_to_host_monotonic" assert live_result["collection"]["cuda_events"] >= 3 + assert graph_validation["schema_version"] == "3.0", graph_validation + assert graph_validation["valid"] is True, graph_validation["issues"] + assert [edge["name"] for edge in graph_validation["edges"]] == [ + "request_to_launch", + "launch_to_kernel", + "kernel_duration", + ] + assert graph_validation["requirements"]["needs_ebpf"] is True + assert graph_validation["requirements"]["needs_cupti"] is True + assert graph_result["schema_version"] == "3.0", graph_result + assert graph_result["status"] == "completed", graph_result + edges = {edge["name"]: edge for edge in graph_result["edges"]} + assert set(edges) == {"request_to_launch", "launch_to_kernel", "kernel_duration"} + for edge in edges.values(): + assert edge["measurement"]["samples"]["matched"] >= 3, edge + assert edge["measurement"]["latency_ns"]["min"] > 0, edge + assert edge["evidence"]["exemplars"], edge + assert edges["request_to_launch"]["correlation"]["confidence"] == "heuristic" + assert edges["launch_to_kernel"]["correlation"]["confidence"] == "exact" + assert edges["kernel_duration"]["correlation"]["confidence"] == "exact" + assert graph_result["artifact"]["events"] == len(graph_events) + assert graph_result["collection"]["host_events"] >= 3 assert export_result["format"] == "chrome" assert export_result["event_count"] == len(chrome_trace["traceEvents"]) assert export_result["event_count"] == len(live_events) @@ -118,6 +150,10 @@ def main() -> None: "live_events": len(live_events), "live_matched": live_result["measurement"]["samples"]["matched"], "spec_matched": spec_result["measurement"]["samples"]["matched"], + "graph_p50_ns": { + name: edge["measurement"]["latency_ns"]["p50"] + for name, edge in edges.items() + }, "trace_events": export_result["event_count"], "min_ns": live_result["measurement"]["latency_ns"]["min"], "gpu": gpu, diff --git a/xprobe/cli/src/args.rs b/xprobe/cli/src/args.rs index 6dfb57b..77c4f4f 100644 --- a/xprobe/cli/src/args.rs +++ b/xprobe/cli/src/args.rs @@ -260,8 +260,16 @@ pub(crate) struct ResolveArgs { #[derive(Debug, Args)] pub(crate) struct ValidateArgs { /// Target process ID. - #[arg(long)] - pub(crate) pid: u32, + #[arg(long, required_unless_present = "spec")] + pub(crate) pid: Option, + + /// Validate a versioned measurement graph spec; its target selects the + /// process. + #[arg( + long, + conflicts_with_all = ["pid", "from", "to", "match_policy", "cpu_sample", "syscall_aggregate"] + )] + pub(crate) spec: Option, /// Start event selector. #[arg(long)] diff --git a/xprobe/cli/src/graph.rs b/xprobe/cli/src/graph.rs new file mode 100644 index 0000000..cb8f655 --- /dev/null +++ b/xprobe/cli/src/graph.rs @@ -0,0 +1,395 @@ +//! `validate --spec` and `measure --spec` for multi-edge measurement graphs. + +use std::{ + path::{Path, PathBuf}, + process::ExitCode, + time::Duration, +}; + +use xprobe_collector::completed::CompletedCapture; +use xprobe_core::{graph, inspect}; +use xprobe_correlator::{MeasureOptions, finalize, measure}; +use xprobe_protocol::{ + CollectionSummary, ErrorCode, Event, EvidenceArtifact, GraphEdgeResult, GraphIssueScope, + HostCaptureResult, MeasurementGraphResult, MeasurementGraphSpec, + MeasurementGraphValidationResult, MeasurementMode, MeasurementResult, SchemaVersion3, + SessionStatus, Warning, +}; + +use crate::{ + EvidenceOutput, ExportFormatArg, export_measurement_capture, + failure::{CommandFailure, emit_command_failure, measurement_failure}, + finish_measurement_failure, + live::{ + LiveCollection, LiveEvaluator, LiveMeasureRequest, LiveOutcome, LivePlan, + apply_collection_summary, checked_live_capture, collect_live, validate_live_limits, + }, + protocol_export_format, +}; + +/// Parse a graph spec, failing with the spec path on malformed input. +pub(crate) fn parse_spec( + value: serde_json::Value, + path: &Path, +) -> Result { + serde_json::from_value(value).map_err(|error| { + CommandFailure::new( + ErrorCode::InvalidEventSelector, + format!("invalid MeasurementGraphSpec {}: {error}", path.display()), + true, + ) + }) +} + +fn validate_graph( + spec: &MeasurementGraphSpec, +) -> Result { + let report = inspect::run(spec.target.pid).map_err(|error| { + CommandFailure::new(error.code(), error.to_string(), error.recoverable()) + })?; + graph::run(&report, spec).map_err(|error| { + let edge = error.edge().map(str::to_owned); + let failure = CommandFailure::new(error.code(), error.to_string(), error.recoverable()); + match edge { + Some(edge) => failure.with_detail("edge", edge), + None => failure, + } + }) +} + +pub(crate) fn run_validate_graph(spec: &MeasurementGraphSpec, json: bool) -> ExitCode { + match validate_graph(spec) { + Ok(result) => { + if json { + println!( + "{}", + serde_json::to_string_pretty(&result).expect("graph validation must serialize") + ); + } else { + print_graph_validation(&result); + } + ExitCode::SUCCESS + } + Err(error) => emit_command_failure(error, json), + } +} + +fn print_graph_validation(result: &MeasurementGraphValidationResult) { + println!( + "Graph validation for PID {}: {}", + result.target.pid, + if result.valid { "valid" } else { "invalid" } + ); + for edge in &result.edges { + println!( + " edge {}: {} -> {} ({:?}, recommended {:?})", + edge.name, edge.from, edge.to, edge.match_policy, edge.policy_recommendation.policy + ); + } + for issue in &result.issues { + println!( + " issue [{:?} {}] {}: {}", + issue.scope, + issue.name.as_deref().unwrap_or("-"), + issue.code, + issue.message + ); + } +} + +/// One edge of a running graph with its correlator options. +struct EdgePlan { + name: String, + options: MeasureOptions, +} + +struct GraphEdges { + session_id: String, + edges: Vec, +} + +pub(crate) struct GraphExecution { + session_id: String, + results: Vec, + status: SessionStatus, + warnings: Vec, + events: Vec, + /// The shared capture's summary; every edge reports the same one. + collection: CollectionSummary, +} + +impl LiveOutcome for GraphExecution { + fn set_status(&mut self, status: SessionStatus) { + self.status = status; + for result in &mut self.results { + result.status = status; + } + } + + fn push_warning(&mut self, warning: Warning) { + self.warnings.push(warning); + } + + fn into_events(self) -> Vec { + self.events + } +} + +impl LiveEvaluator for GraphEdges { + type Outcome = GraphExecution; + + fn evaluate( + &self, + host_captures: &[HostCaptureResult], + collection: &LiveCollection, + require_complete: bool, + ) -> Result { + let capture = checked_live_capture( + host_captures, + collection, + &self.session_id, + require_complete, + )?; + let results = self + .edges + .iter() + .map(|edge| measure_edge(edge, &capture, require_complete)) + .collect::, _>>()?; + Ok(GraphExecution { + session_id: self.session_id.clone(), + collection: results[0].collection.clone(), + results, + status: SessionStatus::Completed, + warnings: Vec::new(), + events: capture.events, + }) + } + + fn reached_samples(&self, outcome: &GraphExecution) -> bool { + self.edges + .iter() + .zip(&outcome.results) + .all(|(edge, result)| { + edge.options.samples.is_some_and(|samples| { + result.measurement.samples.matched + >= u64::try_from(samples).expect("sample limit fits u64") + }) + }) + } +} + +fn measure_edge( + edge: &EdgePlan, + capture: &CompletedCapture, + require_complete: bool, +) -> Result { + let options = MeasureOptions { + dropped_events: capture.dropped_records, + ..edge.options.clone() + }; + let mut result = measure(&capture.events, &options).map_err(|error| { + let mut failure = measurement_failure(&error, &options, &capture.events, require_complete) + .with_detail("edge", edge.name.clone()); + failure.message = format!("edge {:?}: {}", edge.name, failure.message); + failure + })?; + apply_collection_summary(&mut result, capture); + Ok(result) +} + +pub(crate) fn run_measure_graph( + spec: MeasurementGraphSpec, + cupti_socket: Option, + agent_path: Option, + output: EvidenceOutput<'_>, + json: bool, +) -> ExitCode { + match collect_graph(spec, cupti_socket, agent_path) { + Ok((execution, spec)) => finish_graph(execution, &spec, output, json), + Err(error) => finish_measurement_failure(error, output, true, json), + } +} + +fn collect_graph( + spec: MeasurementGraphSpec, + cupti_socket: Option, + agent_path: Option, +) -> Result<(GraphExecution, MeasurementGraphSpec), CommandFailure> { + let validation = validate_graph(&spec)?; + if let Some(issue) = validation.issues.first() { + let scope = match (issue.scope, issue.name.as_deref()) { + (GraphIssueScope::Graph, _) | (_, None) => "graph".to_owned(), + (GraphIssueScope::Endpoint, Some(name)) => format!("endpoint {name:?}"), + (GraphIssueScope::Edge, Some(name)) => format!("edge {name:?}"), + }; + return Err( + CommandFailure::new(issue.code, format!("{scope}: {}", issue.message), true) + .with_detail("issues", validation.issues.len() as u64) + .with_hint("run validate --spec to see every graph issue"), + ); + } + let report = inspect::run(spec.target.pid).map_err(|error| { + CommandFailure::new(error.code(), error.to_string(), error.recoverable()) + })?; + let session_id = format!("xp_graph_{}", std::process::id()); + let limit = |value: u64, name: &str| { + usize::try_from(value).map_err(|error| { + CommandFailure::new( + ErrorCode::SessionLimitExceeded, + format!("graph {name} exceeds this platform: {error}"), + true, + ) + }) + }; + let max_events = limit(spec.max_events.unwrap_or(100_000), "max_events")?; + let duration = spec.duration_ms.map(Duration::from_millis); + let edges = spec + .edges + .iter() + .map(|edge| { + Ok(EdgePlan { + name: edge.name.clone(), + options: MeasureOptions { + session_id: session_id.clone(), + name: Some(edge.name.clone()), + start_selector: spec.endpoints[&edge.from].clone(), + end_selector: spec.endpoints[&edge.to].clone(), + match_policy: edge.match_policy, + samples: edge + .samples + .map(|samples| limit(samples, "samples")) + .transpose()?, + duration, + max_events, + dropped_events: 0, + }, + }) + }) + .collect::, CommandFailure>>()?; + let first = &edges[0].options; + let all_sampled = edges.iter().all(|edge| edge.options.samples.is_some()); + let request = LiveMeasureRequest { + pid: spec.target.pid, + expected_target: Some(spec.target.clone()), + cupti_socket, + agent_path, + timeout: Duration::from_millis(spec.timeout_ms), + match_policy_text: graph::policy_text(first.match_policy), + mode: MeasurementMode::Exact, + options: MeasureOptions { + session_id: session_id.clone(), + name: spec.name.clone(), + samples: all_sampled + .then(|| edges.iter().filter_map(|edge| edge.options.samples).max()) + .flatten(), + ..first.clone() + }, + }; + validate_live_limits(&request)?; + let plan = LivePlan { + report, + endpoints: validation + .endpoints + .into_iter() + .map(|endpoint| endpoint.endpoint) + .collect(), + requirements: validation.requirements, + }; + let execution = collect_live(plan, &request, &GraphEdges { session_id, edges })?; + Ok((execution, spec)) +} + +fn finish_graph( + execution: GraphExecution, + spec: &MeasurementGraphSpec, + output: EvidenceOutput<'_>, + json: bool, +) -> ExitCode { + let path = match output.artifact_path(true) { + Ok(path) => path, + Err(error) => return emit_command_failure(error, json), + }; + let mut artifact = None; + if let Some(path) = path { + let format = output.format.unwrap_or(ExportFormatArg::Jsonl); + if let Err(error) = export_measurement_capture(&execution.events, &path, format) { + return emit_command_failure(error, json); + } + artifact = Some(EvidenceArtifact { + path: path.display().to_string(), + format: protocol_export_format(format), + events: execution.events.len() as u64, + }); + } + let edges = spec + .edges + .iter() + .zip(execution.results) + .map(|(edge, mut result)| { + finalize(&mut result, output.mode); + GraphEdgeResult { + name: edge.name.clone(), + from: edge.from.clone(), + to: edge.to.clone(), + start_selector: spec.endpoints[&edge.from].clone(), + end_selector: spec.endpoints[&edge.to].clone(), + measurement: result.measurement, + quality: result.quality, + correlation: result.correlation, + clock: result.clock, + evidence: result.evidence, + warnings: result.warnings, + } + }) + .collect(); + let result = MeasurementGraphResult { + schema_version: SchemaVersion3::current(), + ok: true, + session_id: execution.session_id, + status: execution.status, + name: spec.name.clone(), + target: spec.target.clone(), + edges, + collection: execution.collection, + artifact, + warnings: execution.warnings, + }; + if json { + println!( + "{}", + serde_json::to_string_pretty(&result).expect("graph result must serialize") + ); + } else { + print_graph_result(&result); + } + ExitCode::SUCCESS +} + +fn print_graph_result(result: &MeasurementGraphResult) { + println!( + "Graph {} on PID {}: {:?}", + result.name.as_deref().unwrap_or("unnamed"), + result.target.pid, + result.status + ); + for edge in &result.edges { + let latency = &edge.measurement.latency_ns; + println!( + " {:<24} matched {:>6} p50 {:>10} ns p99 {:>10} ns {:?}", + edge.name, + edge.measurement.samples.matched, + latency.p50, + latency.p99, + edge.quality.verdict + ); + } + if let Some(artifact) = &result.artifact { + println!( + "Evidence artifact: {} ({} events)", + artifact.path, artifact.events + ); + } + for warning in &result.warnings { + println!("Warning: {}: {}", warning.code, warning.message); + } +} diff --git a/xprobe/cli/src/live.rs b/xprobe/cli/src/live.rs index 77ec824..8e6d587 100644 --- a/xprobe/cli/src/live.rs +++ b/xprobe/cli/src/live.rs @@ -1352,7 +1352,35 @@ pub(crate) fn measure_live_capture( options: &MeasureOptions, require_complete: bool, ) -> Result { - let capture = completed_live_capture(host_captures, collection, &options.session_id)?; + let capture = checked_live_capture( + host_captures, + collection, + &options.session_id, + require_complete, + )?; + let options = MeasureOptions { + dropped_events: capture.dropped_records, + ..options.clone() + }; + let mut result = measure(&capture.events, &options).map_err(|error| { + measurement_failure(&error, &options, &capture.events, require_complete) + })?; + apply_collection_summary(&mut result, &capture); + Ok(MeasurementExecution { + result, + events: capture.events, + }) +} + +/// Merge the live sources into one capture and reject incomplete or unknown +/// records, preserving the events as an artifact on failure. +pub(crate) fn checked_live_capture( + host_captures: &[HostCaptureResult], + collection: &LiveCollection, + session_id: &str, + require_complete: bool, +) -> Result { + let capture = completed_live_capture(host_captures, collection, session_id)?; reject_incomplete_capture(&capture, true, require_complete) .map_err(|error| error.with_artifact_events(capture.events.clone()))?; if capture.unknown_records != 0 { @@ -1368,18 +1396,7 @@ pub(crate) fn measure_live_capture( .with_hint("rebuild xprobe and the CUPTI Agent from the same release") .with_artifact_events(capture.events)); } - let options = MeasureOptions { - dropped_events: capture.dropped_records, - ..options.clone() - }; - let mut result = measure(&capture.events, &options).map_err(|error| { - measurement_failure(&error, &options, &capture.events, require_complete) - })?; - apply_collection_summary(&mut result, &capture); - Ok(MeasurementExecution { - result, - events: capture.events, - }) + Ok(capture) } pub(crate) fn completed_live_capture( diff --git a/xprobe/cli/src/main.rs b/xprobe/cli/src/main.rs index 6a93b3e..f76f102 100644 --- a/xprobe/cli/src/main.rs +++ b/xprobe/cli/src/main.rs @@ -24,6 +24,7 @@ use xprobe_protocol::{ mod args; mod dev; mod failure; +mod graph; mod inventory; mod live; mod render; @@ -415,6 +416,12 @@ fn run_measure_spec( ); } }; + if value.get("edges").is_some() { + return match graph::parse_spec(value, path) { + Ok(spec) => graph::run_measure_graph(spec, cupti_socket, agent_path, output, json), + Err(error) => emit_command_failure(error, json), + }; + } if value.get("sample_event").is_some() { return run_cpu_sampling_file_spec( value, @@ -791,6 +798,7 @@ fn emit_aggregate_inventory(result: &AggregateInventoryResult, json: bool) -> Ex fn run_validate(args: ValidateArgs) -> ExitCode { let ValidateArgs { pid, + spec, from, to, match_policy, @@ -803,6 +811,24 @@ fn run_validate(args: ValidateArgs) -> ExitCode { non_interactive: _, }, } = args; + if let Some(path) = spec { + let value = match fs::read(&path) + .map_err(|error| format!("failed to read {}: {error}", path.display())) + .and_then(|bytes| { + serde_json::from_slice(&bytes) + .map_err(|error| format!("invalid graph spec {}: {error}", path.display())) + }) { + Ok(value) => value, + Err(message) => { + return emit_error(ErrorCode::InvalidEventSelector, message, true, json); + } + }; + return match graph::parse_spec(value, &path) { + Ok(spec) => graph::run_validate_graph(&spec, json), + Err(error) => emit_command_failure(error, json), + }; + } + let pid = pid.expect("clap requires --pid without --spec"); let report = match inspect::run(pid) { Ok(report) => report, Err(error) => { diff --git a/xprobe/cli/tests/validate.rs b/xprobe/cli/tests/validate.rs index 5be41ab..b2fba26 100644 --- a/xprobe/cli/tests/validate.rs +++ b/xprobe/cli/tests/validate.rs @@ -145,3 +145,115 @@ fn validate_reports_syscall_aggregate_ebpf_requirements() { assert!(!result.requirements.target_mutation); assert_eq!(result.valid, result.issues.is_empty()); } + +fn own_start_time() -> u64 { + let stat = std::fs::read_to_string("/proc/self/stat").expect("own stat is readable"); + stat[stat.rfind(')').expect("stat has a command") + 1..] + .split_whitespace() + .nth(19) + .expect("stat has a start time") + .parse() + .expect("start time is an integer") +} + +fn validate_graph(graph: &serde_json::Value) -> std::process::Output { + let path = std::env::temp_dir().join(format!( + "xprobe-graph-{}-{}.json", + std::process::id(), + graph["name"].as_str().unwrap_or("graph") + )); + std::fs::write(&path, serde_json::to_vec(graph).unwrap()).expect("graph fixture is written"); + let output = Command::new(env!("CARGO_BIN_EXE_xprobe")) + .args(["validate", "--spec", path.to_str().unwrap()]) + .output() + .expect("xprobe validate must run"); + std::fs::remove_file(path).expect("graph fixture is removed"); + output +} + +fn graph(name: &str, edges: &serde_json::Value) -> serde_json::Value { + serde_json::json!({ + "schema_version": "3.0", + "name": name, + "target": {"pid": std::process::id(), "process_start_time": own_start_time()}, + "endpoints": { + "launch_exit": "cuda:runtime_api:cudaLaunchKernel:exit", + "kernel_start": "cuda:kernel_start", + "kernel_end": "cuda:kernel_end" + }, + "edges": edges, + "duration_ms": null, + "timeout_ms": 1000, + "max_events": null + }) +} + +#[test] +fn validates_a_cuda_graph_with_merged_requirements() { + let output = validate_graph(&graph( + "valid", + &serde_json::json!([ + {"name": "launch", "from": "launch_exit", "to": "kernel_start", "match_policy": "exact", "samples": 5}, + {"name": "kernel", "from": "kernel_start", "to": "kernel_end", "match_policy": "exact", "samples": 5} + ]), + )); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let result: xprobe_protocol::MeasurementGraphValidationResult = + serde_json::from_slice(&output.stdout).expect("stdout is graph validation JSON"); + assert_eq!(result.issues, Vec::new()); + assert!(result.valid); + assert_eq!( + result + .endpoints + .iter() + .map(|endpoint| endpoint.name.as_str()) + .collect::>(), + ["kernel_end", "kernel_start", "launch_exit"] + ); + assert!(result.requirements.needs_cupti_callback); + assert!(result.requirements.needs_cupti_activity); + assert_eq!( + result.requirements.agent_activation, + AgentActivation::InjectionRequired + ); +} + +#[test] +fn reports_scoped_graph_issues_without_failing() { + let output = validate_graph(&graph( + "invalid", + &serde_json::json!([ + {"name": "launch", "from": "launch_exit", "to": "missing", "match_policy": "exact", "samples": null} + ]), + )); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let result: xprobe_protocol::MeasurementGraphValidationResult = + serde_json::from_slice(&output.stdout).expect("stdout is graph validation JSON"); + assert!(!result.valid); + let issues = result + .issues + .iter() + .map(|issue| (issue.scope, issue.name.as_deref(), issue.message.as_str())) + .collect::>(); + assert!(issues.contains(&( + xprobe_protocol::GraphIssueScope::Edge, + Some("launch"), + "edge references unknown endpoint \"missing\"" + ))); + assert!(issues.iter().any(|(scope, name, _)| { + *scope == xprobe_protocol::GraphIssueScope::Endpoint && *name == Some("kernel_end") + })); + assert!( + issues + .iter() + .any(|(_, _, message)| message.contains("bounded")) + ); +} From d2483e3d8b41629b2e07ba68f7159bc7f7793c75 Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 14 Nov 2026 14:29:04 +0800 Subject: [PATCH 12/15] =?UTF-8?q?=F0=9F=93=9D=20docs:=20rewrite=20the=20Sk?= =?UTF-8?q?ill=20around=20triage=20and=20escalation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Skill spread one workflow across ~5,800 words in seven files, with selector syntax and quality checklists repeated, and tests pinned more than seventy exact sentences so nothing could be removed. SKILL.md now drives two commands for unknown live work (triage, then the ranked next spec or a measurement graph), keeps the offline artifact route, and adds an escalation ladder for when the first pass explains nothing: missing sources, split spans with graphs, tail exemplars, blocked threads, and regressions, ending in explicit hand-off. Four overlapping references merge into selectors, interpreting, and a new deep-dive guide; the Skill drops to ~3,400 words. Contract tests now enforce word budgets, resolve links, reject orphaned references, require every documented xprobe flag to exist in that subcommand's --help, and check routes by behavior instead of prose. --- AGENTS.md | 18 +- docs/agent-integration.md | 53 +--- skills/xprobe-measure-latency/SKILL.md | 192 ++++++------ .../xprobe-measure-latency/agents/openai.yaml | 4 +- .../examples/request-path-graph.json | 22 ++ .../references/cli-contract.md | 148 ---------- .../references/deep-dive.md | 106 +++++++ .../references/interpreting.md | 79 +++++ .../references/investigation.md | 226 --------------- .../references/multi-process.md | 110 ++----- .../references/result-quality.md | 96 ------ .../references/selectors.md | 94 ++++++ .../references/setup.md | 131 +++------ .../references/trace-analysis.md | 37 --- .../fixtures/workflow-routes.json | 103 +++++-- tests/agent-contract/test_contract.py | 273 ++++++------------ tests/agent-contract/test_workflow_routes.py | 16 +- 17 files changed, 675 insertions(+), 1033 deletions(-) create mode 100644 skills/xprobe-measure-latency/examples/request-path-graph.json delete mode 100644 skills/xprobe-measure-latency/references/cli-contract.md create mode 100644 skills/xprobe-measure-latency/references/deep-dive.md create mode 100644 skills/xprobe-measure-latency/references/interpreting.md delete mode 100644 skills/xprobe-measure-latency/references/investigation.md delete mode 100644 skills/xprobe-measure-latency/references/result-quality.md create mode 100644 skills/xprobe-measure-latency/references/selectors.md delete mode 100644 skills/xprobe-measure-latency/references/trace-analysis.md diff --git a/AGENTS.md b/AGENTS.md index f6d0244..5bde077 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -54,14 +54,16 @@ - Follow `skills/xprobe-measure-latency/SKILL.md` for measurement tasks. - Choose the shortest Skill route supported by the evidence. Existing artifacts - do not require live readiness checks; known selectors may proceed directly to - validation and bounded measurement. Use broad-to-narrow inventory only when - selectors are unknown. Classify live work as CPU-only or GPU/mixed before - choosing collectors; `discover` is only for CUDA context holders. Scope - breadth and capture duration are independent. Do not guess selectors. -- Read evidence pairs, artifact analysis, stream identity, collection quality, - and profiler overhead before interpreting latency. Summed concurrent GPU - duration is not wall time. + are analyzed offline; known boundaries go straight to bounded measurement; + unknown live work starts with one triage window and continues with its ranked + next spec or a measurement graph. Escalate only from evidence, and do not + guess selectors. `discover` is only for CUDA context holders. +- Read the quality verdict, source coverage, stream identity, and profiler + overhead before interpreting latency. Summed concurrent GPU duration is not + wall time. +- Keep the Skill entry point short and each fact in one place. Field meanings + belong to schemas and flags to `--help`; Skill tests check behavior, links, + and documented flags rather than pinning prose. - Keep reusable analysis deterministic and bundled with the Skill. Keep model orchestration in the caller rather than adding CLI commands or a service. diff --git a/docs/agent-integration.md b/docs/agent-integration.md index 598a413..65f5537 100644 --- a/docs/agent-integration.md +++ b/docs/agent-integration.md @@ -26,39 +26,14 @@ For automation, add `--agent codex|claude-code|cursor --copy --yes`. Omit install its whole directory so its references, examples, and analysis script remain available. -The Skill routes setup, completed-artifact analysis, known-boundary measurement, -unknown CPU/Python/GPU investigation, and multi-process work independently. It -checks or installs the CLI only for live work. Unknown CPU work starts with -sampled stacks, adds syscall aggregation or CPython GC only when supported by a -hypothesis, and falls back visibly to native frames when Python semantics are -unavailable. Unknown GPU work uses only relevant aggregates. Mixed CPU/GPU -inventories may run concurrently with separate contracts, bounds, outputs, and -failure handling. The bundled `scripts/analyze_trace.py` provides deterministic -kernel, copy, overlap, stream, and gap summaries. The repository tests -installation with `skills` CLI 1.5.20 in isolated -home directories. This pinned test protects released behavior while the -documented `skills@1` selector receives compatible path updates. - -For multi-process workloads, the Skill selects explicit PID/start-time -identities, inventories a representative worker when homogeneity is supported -by evidence, and asks the agent framework to launch one independent bounded -measurement per selected worker concurrently. Results, warnings, failures, and -artifacts remain per process; xprobe does not add a multi-process command or -claim cross-process causality. - -Inventory outputs are not event artifacts. CPU hotspots, syscall groups, and -GPU groups produce selector hypotheses; every exact selector still passes -read-only `validate`. The Agent must inspect the quality fields specific to each -schema and cannot equate sample proportions, aggregate duration shares, or -overlapping capture windows with exact causality. - -For containerized live targets, the Skill keeps orchestration in the caller. It -resolves an explicit application container, runs xprobe in the same PID and -mount namespaces, reacquires PID plus procfs start time there, and preserves -per-command artifacts outside the container. It never substitutes a sidecar or -host PID, and reports capabilities that cannot be added to an already-running -container. Narrow exact captures begin with bounded record headroom and an -`--events-out` artifact instead of defaulting to six-figure capacity. +The Skill's `SKILL.md` is the agent-facing workflow; this page does not +restate it. It drives a two-command path for unknown live work: `measure +--triage` over one representative window, then the ranked finding's ready +`MeasurementSpec` or a measurement graph. Existing artifacts are analyzed +offline without installing a collector, and an escalation ladder covers what +the first pass cannot explain before handing off to kernel or runtime +profilers. Selection and concurrency across processes stay in the agent; +xprobe adds no multi-process command and never claims cross-process causality. ## Contract test @@ -68,11 +43,13 @@ just test-skill-install ``` The test requires the visible command set to be exactly `doctor`, `discover`, -`validate`, and `measure`. It invokes the first three in strict JSON mode, -checks injection requirements, verifies schemas, exercises the bundled trace -analyzer, and checks adaptive task routing for unknown CPU, Python, mixed, -known-selector, existing-artifact, and unsupported-runtime scenarios, bounded -live collection, mutation guards, and result quality/evidence. +`validate`, and `measure`, invokes them in strict JSON mode, and verifies +schemas and the bundled trace analyzer. For the Skill it enforces a word budget +for the entry point and all references, resolves every link, rejects orphaned +references, checks that every documented `xprobe` command uses flags its +subcommand accepts, validates the example specs, and checks routing for +existing artifacts, unknown workloads, known boundaries, graphs, deeper +escalation, Python semantics, multiple processes, and containers. The installation test uses the real third-party CLI in isolated home directories and verifies byte-for-byte copies for Codex, Claude Code, and Cursor. diff --git a/skills/xprobe-measure-latency/SKILL.md b/skills/xprobe-measure-latency/SKILL.md index da94212..e775163 100644 --- a/skills/xprobe-measure-latency/SKILL.md +++ b/skills/xprobe-measure-latency/SKILL.md @@ -1,119 +1,97 @@ --- name: xprobe-measure-latency -description: Profile live or recorded Linux CPU, CPython, and NVIDIA CUDA workloads with bounded xprobe evidence. Use when an agent needs to install or repair xprobe, inventory unknown CPU/GPU work, find native or Python hotspots, inspect syscall or CPython GC cost, validate and measure a known function, syscall, CUDA API, kernel, transfer, synchronization point, NVTX range, or event-to-event latency, compare workers, investigate regressions, or inspect an existing xprobe artifact. +description: Find where time goes in a live or recorded Linux CPU, CPython, or NVIDIA CUDA workload with bounded xprobe evidence. Use to triage an unknown slow process, measure a known function, syscall, CUDA API, kernel, transfer, NVTX range, or event-to-event latency, measure several related latencies at once, compare workers or regressions, inspect an existing xprobe artifact, or install and repair xprobe. --- # Profile workloads with xprobe -Choose the shortest route supported by the question and existing evidence. Do -not run a fixed checklist or collect every source. +Reach an answer in two commands: triage, then one exact measurement. Choose the +shortest route the evidence allows; never collect every source by checklist. + +Commands print JSON whenever stdout is not a terminal. Measurement, graph, and +triage results use schema version `3.0`; other records and Event JSONL use +`2.0`. Treat malformed output or an unknown version as an error. ## Route the task -- **Existing artifact**: Read [references/trace-analysis.md](references/trace-analysis.md) - and [references/result-quality.md](references/result-quality.md). Analyze it - directly or use `measure --input`; skip installation, `doctor`, `discover`, - and live attachment unless a new capture is required. -- **Known live boundary**: Read [references/cli-contract.md](references/cli-contract.md) - and [references/result-quality.md](references/result-quality.md). Confirm PID - plus start time, run read-only `validate`, then one bounded measurement. Do - not run a broad inventory solely to satisfy a checklist. -- **Unknown CPU or Python workload**: Read - [references/investigation.md](references/investigation.md). Start with bounded - `--cpu-sample` evidence. Add `--syscall-aggregate` only for a kernel-facing - hypothesis; validate CPython GC boundaries only for a GC hypothesis. Narrow - from emitted selector hints rather than guessing or requiring manual symbol - inspection first. Do not run CUDA discovery. -- **Unknown GPU or mixed workload**: Read - [references/investigation.md](references/investigation.md). Use `discover` - only to select CUDA context holders. Collect the relevant bounded GPU - aggregate and, for mixed work, a bounded CPU sample inventory. Run independent - coarse captures concurrently when an aligned workload window matters, with - separate bounds, outputs, failures, and perturbation accounting. -- **Multiple processes**: Also read - [references/multi-process.md](references/multi-process.md). Select explicit - PID/start-time identities. Inventory one representative per defensible worker - class, then run independent narrow commands concurrently where useful. -- **Containerized live target**: Also read - [references/setup.md](references/setup.md). Run the matching CLI in the - target's PID and mount namespaces, then reacquire the namespace-local PID plus - procfs start time before validation. Resolve the container from explicit - workload evidence; do not guess a pod, container, or PID. -- **Setup or repair**: Read [references/setup.md](references/setup.md) only when - a required live command is absent, incompatible, or unhealthy. Existing - artifacts do not require a local collector. - -Classify live work as CPU-only or GPU/mixed before choosing collectors. Run -`doctor` when capability is unknown or a command reports an environment -failure, not as a prerequisite for valid offline or already-diagnosed work. - -## Narrow from evidence - -For unknown CPU work, validate and run one representative `--cpu-sample` -window. Inspect sample loss, stack truncation, thread coverage, symbol coverage, -`python_status`, stack groups, and inclusive/exclusive hotspots. An active -Python perf map provides Python frames; `inactive` or `unsupported` does not -invalidate native frames. Use native or Python hotspot context and emitted -entry/return selector hints to form the next hypothesis. - -Use `--syscall-aggregate` when CPU evidence or the question points to I/O, -allocation, scheduling, or VM behavior. Inspect unmatched/inflight/drop and map -capacity before using a group's selector hints. For suspected CPython garbage -collection, validate `python:gc_start` to `python:gc_end`; validation checks the -target's ELF USDT metadata and may reject a build without those probes. - -For unknown GPU work, aggregate only relevant kernel, memcpy, or memset -families over a representative cycle. Read group counts, duration totals, -bytes, completeness, and selector hints. Scope breadth and capture duration are -independent: narrowing scope is not a substitute for retaining a representative -window. - -Every hotspot or aggregate group is a hypothesis, not final attribution. Pass -the selected exact endpoints through read-only `validate`, honor its policy -recommendation explicitly, and collect one detailed bounded measurement per -stated hypothesis. A failed validation revises the hypothesis; it does not -justify unrelated collection. - -## Preserve these invariants - -- Use JSON mode. Measurement results use schema version `3.0`; other records - and Event JSONL use `2.0`. Treat malformed output and unknown versions as - errors. -- Identify a live process by PID plus procfs start time and verify it around - validation and attachment. Never substitute a reused PID. -- Bound every capture by duration or samples, timeout, and record/group/thread - capacity. Preserve each command's stdout, stderr, status, and artifact. -- For one narrow exact hypothesis, start with modest record headroom over the - requested samples and always write the bounded event artifact. Increase - `--max-events` only from observed unmatched records or capacity pressure, not - as a default substitute for narrower selectors. -- Before CUDA injection, disclose that `measure` will ptrace the process and - leave the CUPTI shared object mapped. `startup_required` NVTX work must restart - with the matching Agent before the first NVTX call. -- Inspect completeness, loss/drops, capacities, unmatched/ambiguous evidence, - symbol and stack coverage, clock alignment, method, confidence, and evidence - pairs before interpreting latency. +- **Existing artifact**: analyze it offline. Re-correlate Event JSONL with + `measure --input`; summarize GPU activity with `scripts/analyze_trace.py`. + Skip installation, `doctor`, `discover`, and live attachment unless a new + capture is required. +- **Unknown live workload**: identify the process, then triage one + representative cycle (one request, batch, or iteration, after warmup): + + ```bash + xprobe measure --pid "$PID" --triage --duration-ms 2000 + ``` + + Read `findings` from rank 1. A finding's `next.spec` is a ready measurement: + write it to a file and run `xprobe measure --spec next.json`. An + `environment` finding (throttling, runqueue wait, GPU clock limits) is a + conclusion, not a cue to profile code. +- **Known boundary**: skip triage. Run read-only `validate` when injection or + selector resolution is uncertain, then one bounded `measure`. +- **Several related boundaries** (request → launch → kernel): one measurement + graph captures every edge at once; see [selectors](references/selectors.md). +- **CUDA worker unknown**: `discover --pid ROOT` lists CUDA context holders; pick + from workload evidence. For several workers read + [multi-process](references/multi-process.md). +- **Missing CLI, container target, or environment failure**: read + [setup](references/setup.md). Run `doctor` only when capability is unknown or + a command reports an environment failure. + +Read [interpreting results](references/interpreting.md) before stating a +latency claim, and [selectors](references/selectors.md) before writing a +selector yourself. + +## Read results + +- Triage: check `sources` before trusting silence. An `unavailable` source and + every `not_covered` item are questions triage did not answer. Shares are + estimates; blocked time includes idle waiting. +- Exact results: start from `quality.verdict` and its issues. `caution` limits + the claim; say why. Rank exemplars (min, p50, p90, p99, max) are inline; the + full capture is at `evidence.artifact.path`. +- A command error is not a result. Errors keep structured `details`, including + any preserved artifact path; follow `hints` instead of retrying blindly. + +## Dig deeper when the first pass is inconclusive + +Escalate one step at a time, each driven by the previous result; details are in +[deep dive](references/deep-dive.md). + +1. **Triage found nothing actionable**: fix `unavailable` sources first, then + widen the window to cover a slow cycle, or run the focused inventory triage + omits (memcpy/memset activity, larger CPU sample, `-X perf` for Python). +2. **A span is slow but unexplained**: split it with a measurement graph into + sub-edges, and follow sampled stacks down to a callee boundary. +3. **Only the tail is slow**: read the p99 and max exemplars, then inspect the + artifact around them; re-correlate it offline with `measure --input` before + recapturing. +4. **Threads are blocked**: measure the dominant syscall exactly, check Python + GC lifecycles, and treat lock or I/O attribution as not yet covered. +5. **It is a regression**: run the same spec on the baseline and compare + distributions and quality, not single means. + +Stop escalating when a step adds no new boundary; report what is known, what +remains unexplained, and which hand-off tool fits. + +## Keep these invariants + +- Identify a live process by PID plus procfs start time; never reuse a PID-only + choice after a restart. +- Bound every capture by samples or duration, timeout, and capacity. +- Before CUDA injection, tell the user that `measure` will ptrace the process + and leave the CUPTI shared object mapped. NVTX needs the Agent at startup. - Keep process and CUDA stream identity in every claim. Summed concurrent GPU - duration is not wall time; temporal correlation is not exact causality. - -Record a baseline when investigating regression or profiler overhead. Warm up -framework and JIT work before a representative capture. Independent CPU and GPU -inventories can run concurrently, but compare their perturbation and never -merge their distinct result contracts as if they were one timeline. - -For exact GPU artifacts, run `scripts/analyze_trace.py` and inspect launch -variants, stream distribution, `busy_union_ns`, overlap factor, and gaps. -Aggregate and sampled inventories contain no event timeline and cannot be -re-correlated offline. - -## Stop or hand off deliberately + time is not wall time, sampled shares are estimates, and temporal correlation + is not exact causality. +- Record an application baseline for regressions or overhead claims. -Stop on target reuse, permission failure, invalid selectors, unavailable -required collectors, incomplete collection, loss/drops, or a clock/correlation -problem that invalidates the claim. Preserve structured failures and artifacts. +## Stop or hand off -Once evidence isolates unexplained time inside one GPU kernel, use NCU or PC -sampling for microarchitecture. Once native sampling isolates code without an -observable boundary, use a runtime-specific profiler or instrumentation. Do not -claim xprobe identifies Python semantics when `python_status` or symbolization -coverage says otherwise. +Stop on target exit or reuse, permission failure, invalid selectors, drops, or +a clock failure that invalidates the claim; report the structured error. Once +time is isolated inside one GPU kernel, hand off to NCU or PC sampling; once it +is isolated in code without an observable boundary, use a runtime-specific +profiler. Do not claim Python semantics when `python_status` says otherwise. diff --git a/skills/xprobe-measure-latency/agents/openai.yaml b/skills/xprobe-measure-latency/agents/openai.yaml index e5d9b8b..ffe2e5f 100644 --- a/skills/xprobe-measure-latency/agents/openai.yaml +++ b/skills/xprobe-measure-latency/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Xprobe Workload Profiling" - short_description: "Route bounded CPU, Python, and GPU profiling" - default_prompt: "Use $xprobe-measure-latency to choose the shortest evidence-based route for this profiling task." + short_description: "Triage and measure CPU, Python, and GPU latency" + default_prompt: "Use $xprobe-measure-latency to triage this workload and measure where its time goes." diff --git a/skills/xprobe-measure-latency/examples/request-path-graph.json b/skills/xprobe-measure-latency/examples/request-path-graph.json new file mode 100644 index 0000000..23fe09e --- /dev/null +++ b/skills/xprobe-measure-latency/examples/request-path-graph.json @@ -0,0 +1,22 @@ +{ + "schema_version": "3.0", + "name": "request_path", + "target": { + "pid": 1234, + "process_start_time": 987654 + }, + "endpoints": { + "request": "uprobe:/srv/app/server:request_begin:entry", + "launch_exit": "cuda:runtime_api:cudaLaunchKernel:exit", + "kernel_start": "cuda:kernel_start:name~^vector_add", + "kernel_end": "cuda:kernel_end:name~^vector_add" + }, + "edges": [ + {"name": "request_to_launch", "from": "request", "to": "launch_exit", "match_policy": "first_after", "samples": 100}, + {"name": "launch_to_kernel", "from": "launch_exit", "to": "kernel_start", "match_policy": "exact", "samples": 100}, + {"name": "kernel_duration", "from": "kernel_start", "to": "kernel_end", "match_policy": "exact", "samples": 100} + ], + "duration_ms": null, + "timeout_ms": 30000, + "max_events": 4000 +} diff --git a/skills/xprobe-measure-latency/references/cli-contract.md b/skills/xprobe-measure-latency/references/cli-contract.md deleted file mode 100644 index 248c8f7..0000000 --- a/skills/xprobe-measure-latency/references/cli-contract.md +++ /dev/null @@ -1,148 +0,0 @@ -# CLI contract - -xprobe exposes four public commands: `doctor`, `discover`, `validate`, and -`measure`. Use `--json --non-interactive --no-color` with every command. JSON -responses carry `schema_version: "2.0"`; diagnostics and the injection warning -go to stderr. Pre-0.3 schema versions are not maintained in parallel. - -## Discovery and selectors - -`discover --pid ROOT_PID` lists only NVML-confirmed CUDA context holders in the -root's process tree. Choose a worker from its PID, start time, command line, and -GPU UUID evidence before validation. The CLI does not choose one automatically. - -Host function selectors use: - -```text -uprobe:::entry -uprobe:::return -uprobe::symbol=:entry -uprobe::symbol=:return -uprobe::+0x:entry -uprobe::+0x:return -``` - -Use the `symbol=` form for a C++ name containing `::`; pass the complete -demangled signature. Validation returns the actual mangled ELF symbol and the -readable signature. This resolves native code mapped by a Python process, not -Python frame or `module.qualname` names. - -Linux selectors use `syscall::entry|exit` and -`tracepoint::`. CPython GC selectors use -`python:gc_start|gc_end`; `validate` resolves the mapped executable or -`libpython` ELF containing both required USDT markers. Syscall entry/exit for one name supports -`exact` per-thread lifecycle matching. Entry evidence contains scalar ABI -register values and exit evidence contains the scalar return value; xprobe -does not dereference pointers. Named tracepoints contain no payload fields. -Always let `validate` resolve the named syscall or tracepoint on the current -host before measurement. - -CUDA selectors cover Runtime and Driver API entry/exit; kernel, memcpy, and -memset activity start/end; and bounded NVTX range start/end. Kernel activity -accepts `name~REGEX`; memcpy accepts `kind=`. NVTX -uses `cuda:nvtx_range_start:name~REGEX` and -`cuda:nvtx_range_end:name~REGEX`; the regex must reduce to an exact, prefix, -suffix, or contains match shorter than 128 bytes. `validate` must accept the -complete selector and policy before measurement. - -## Correlation - -Use `exact` for CUDA events with the same CUPTI correlation ID, NVTX boundaries -with the same range kind and ID, entry/exit of one named syscall, or CPython GC -start/end on one thread. -Use `stack-nested` for entry/return of the same host function and `stream-order` -for activity events on one CUDA stream. `first-after` and `nearest` are temporal -heuristics and cannot establish causality. Read `policy_recommendation.policy`, -its machine-readable `reason`, and `compatible_policies`; xprobe never silently -changes the requested policy. - -## Bounds and failures - -Direct `measure` calls require a positive `--samples` or `--duration-ms` bound. -`--timeout-ms` defaults to 30 seconds and `--max-events` to 100,000. Use exactly -one source mode: `--pid`, one or more `--input` files, or `--spec`. - -CPU sampling is live and duration-bounded: - -```bash -xprobe validate --pid "$PID" --cpu-sample \ - --json --non-interactive --no-color -xprobe measure --pid "$PID" --cpu-sample --duration-ms 1000 \ - --frequency-hz 99 --max-samples 10000 --max-groups 256 \ - --stack-depth 64 --max-threads 1024 \ - --json --non-interactive --no-color -``` - -It samples userspace callchains with `perf_event_open` for a bounded snapshot of -the target's current threads. The result reports exact raw stack counts before -bounded grouping, inclusive/exclusive hotspots, attachable native selector -hints, native/Python/unresolved symbol coverage, Python perf-map status, lost -samples, truncated stacks, attached threads, and every capacity. It does not -emit Event JSONL or claim a continuous all-thread trace. Hotspot proportions -have sampling uncertainty and are not exact CPU-time shares. - -Syscall inventory is also live and duration-bounded: - -```bash -xprobe validate --pid "$PID" --syscall-aggregate \ - --json --non-interactive --no-color -xprobe measure --pid "$PID" --syscall-aggregate --duration-ms 1000 \ - --max-groups 256 --max-inflight 1024 \ - --json --non-interactive --no-color -``` - -PID-filtered raw tracepoints retain per-thread starts and per-syscall aggregate -count/error/duration state in bounded BPF maps. The result reports map quality, -inflight/unmatched/drop counts, known names, and entry/exit selector hints. It -does not stream one event per syscall or dereference arguments. - -Use `measure --aggregate --duration-ms ... --max-groups ...` for a live, coarse -kernel, memcpy, or memset inventory. Aggregate endpoints must be one matching -activity start/end pair with `--match exact`. The result contains bounded group -counts, total/min/max/mean duration, transfer bytes, selector hints, and table -quality. It contains no event evidence, percentiles, or correlation confidence; -`--samples`, `--input`, and `--events-out` are invalid in this mode. -Kernel regex must be an exact, prefix, suffix, or contains shape that the Agent -can apply before aggregation; other regex is rejected instead of widened. -Kernel groups expose `name_complete`. When it is false, the observed name is a -bounded prefix and the emitted selector hints deliberately remain prefix -selectors that the Agent can apply before reserving exact events. - -Kernel and other GPU activity durations require separate start and end records, -so `max-events` is record capacity rather than sample capacity. Sample completion -is checked after bounded snapshots and does not reserve space in the CUPTI -buffer. `duration-ms` limits correlation from the first selected event and also -sets a live stop from ARM completion; either samples or duration may complete a -call when both are present. Timeout bounds the complete foreground operation and -cleanup. - -Linux exact syscall filtering runs in BPF before event reservation. A duration-bound -host capture that reaches `max-events` fails with `EVENT_RATE_TOO_HIGH`; narrow -the selector or increase the explicit bound. Do not replace it with partial -evidence. - -CPU sampling, syscall aggregation, and GPU aggregation have distinct schemas. -Treat their hotspots/groups as selector hypotheses; they are not completed -event captures and cannot be passed to `measure --input`. - -`--events-out PATH` atomically writes the bounded capture with mode `0600`, not -only matched evidence. Collection completeness and CUPTI capacity, observed, -retained, dropped, and buffer utilization fields describe capture integrity -separately from correlation confidence. Correlation or clock failure after -collection still writes the artifact and reports its metadata in error -`details`; inspect it before retrying. - -Exit status `0` means the command emitted a result. Status `1` is validation, -collection, decode, export, cleanup, or internal failure; `2` is invalid CLI -syntax; `3` means the target disappeared or was reused; and `4` is a permission -failure. Treat every nonzero status as an explicit failure and read its JSON -error code, message, details, and hints. - -Validation is read-only. When it reports `injection_required`, the following -live `measure` may ptrace the target and load the matching CUDA 12 or CUDA 13 -CUPTI Agent. Measurement disables the Agent logically after collection but -does not unload the shared object. When validation reports `startup_required` -for an NVTX selector, restart the worker with `NVTX_INJECTION64_PATH` pointing -to that Agent before its first NVTX call, reacquire its PID plus start time, and -validate again. Online injection cannot install NVTX dispatch after NVTX has -initialized. diff --git a/skills/xprobe-measure-latency/references/deep-dive.md b/skills/xprobe-measure-latency/references/deep-dive.md new file mode 100644 index 0000000..dd03c4b --- /dev/null +++ b/skills/xprobe-measure-latency/references/deep-dive.md @@ -0,0 +1,106 @@ +# Dig deeper + +Use this when triage or the first exact measurement does not explain the +latency. Each step names the evidence that triggers it, the next bounded +command, and when to stop. Escalate one step at a time. + +## Triage found nothing actionable + +1. **Missing sources.** Read every `sources` entry. An `unavailable` syscall or + CPU source usually means missing BPF or perf permission; fix it through + [setup](setup.md) and triage again. Silence from a missing source is not + evidence of absence. +2. **Window missed the slow work.** Triage covers one window. If the problem is + intermittent or per-request, drive one slow request during the window or + lengthen `--duration-ms` to cover several cycles. +3. **Triage omits transfers.** GPU activity in triage covers kernels only. + Inventory copies and memsets explicitly: + + ```bash + xprobe measure --pid "$PID" --from cuda:memcpy_start --to cuda:memcpy_end \ + --match exact --aggregate --duration-ms 2000 --max-groups 256 + ``` + +4. **CPU time is spread thin.** A `CPU_BOUND` finding without a single hotspot + needs more samples or semantic frames. Raise frequency and duration, and for + CPython 3.12+ restart the target with `-X perf` so samples resolve to Python + functions: + + ```bash + xprobe measure --pid "$PID" --cpu-sample --duration-ms 5000 \ + --frequency-hz 499 --max-samples 50000 --max-groups 512 + ``` + + Read `stack_groups`, not only `hotspots`: inclusive callers show which + high-level operation owns the spread-out leaf time. + +## A span is slow but unexplained + +Split the span into sub-edges with one measurement graph instead of repeated +single captures. Choose intermediate endpoints from the sampled stacks (callee +`entry_selector_hint` and `return_selector_hint`), CUDA API boundaries, or NVTX +ranges, then compare edge p50/p99: + +```json +"edges": [ + {"name": "request_to_launch", "from": "request", "to": "launch_exit", + "match_policy": "first_after", "samples": 200}, + {"name": "launch_to_kernel", "from": "launch_exit", "to": "kernel_start", + "match_policy": "exact", "samples": 200} +] +``` + +The edge whose latency carries the regression is the next span to split. Edges +are independent pairings; they show where time is spent per edge, not that one +edge caused another. + +For a host function, descend one stack level at a time: measure the function +with `stack-nested`, then the heaviest callee from the CPU sample stack groups. +Stop when a callee no longer carries most of the parent's time. + +## Only the tail is slow + +1. Read the `p99` and `max` exemplars: their thread, stream, timestamps, and + CUDA correlation IDs identify concrete slow instances. +2. Open the evidence artifact (`evidence.artifact.path`) and list events that + overlap those instances in time: GC lifecycles, syscalls, other kernels on + the same stream. Report them as concurrent, not causal. +3. Re-correlate the same artifact with other selectors or policies through + `measure --input` before recapturing; a new capture is needed only for + endpoints the artifact does not contain. +4. If the tail is rare, raise `samples` so p99 rests on at least a few hundred + pairs, and check the `FEW_SAMPLES` quality issue. + +## Threads are blocked + +Triage reports how long threads were blocked, not where. Narrow it: + +- `SYSCALL_TIME` points at a syscall; measure it exactly + (`syscall:NAME:entry` to `syscall:NAME:exit`). `futex` time indicates lock + or condition waits; `poll`, `epoll_wait`, `read`, and `recvfrom` indicate + I/O or idle event loops, so compare against the workload's expected idle. +- In CPython, validate `python:gc_start` to `python:gc_end` and measure GC + lifecycles when blocked or tail time coincides with allocation-heavy phases. +- In CUDA workloads, measure `cudaStreamSynchronize` or `cudaDeviceSynchronize` + entry to exit; long synchronization means the CPU waits on the GPU, so return + to the GPU findings. +- Lock owner and off-CPU stack attribution are not covered by this release; + say so instead of guessing. + +## It is a regression + +Run the same spec or graph on the baseline and the regressed build or host, +with the same workload phase, window, and bounds. Compare per-edge +distributions (p50, p90, p99), matched counts, and quality verdicts. A changed +`quality` issue or capacity pressure on one side makes the comparison +unreliable; fix that before attributing the difference. + +## Hand off + +- Time isolated inside one kernel: NCU or PC sampling for occupancy, memory, + and instruction stalls. +- Time isolated in code without an attachable boundary (JIT, generated, or + inlined code): a runtime-specific profiler or added instrumentation such as + an NVTX range or an exported function boundary. +- Python semantics unavailable (`python_status` not `active`): py-spy or the + framework profiler, reported as a separate tool's evidence. diff --git a/skills/xprobe-measure-latency/references/interpreting.md b/skills/xprobe-measure-latency/references/interpreting.md new file mode 100644 index 0000000..51d25a2 --- /dev/null +++ b/skills/xprobe-measure-latency/references/interpreting.md @@ -0,0 +1,79 @@ +# Interpreting results + +Each result type has its own schema under `schemas/`; field meanings live +there. This page covers how to read them without overclaiming. + +## Triage + +- `findings` are ranked by `impact_share`, an estimated share of the window. + `null` means the evidence cannot bound the impact (clock limits, PCIe link); + such findings rank last but may still matter. +- `threads.top_threads` splits each whole-window thread into on-CPU, runqueue, + and blocked time. Blocked time includes idle waiting, so a mostly idle + thread pool is normal; read the busiest threads, not process totals. +- `gpu_devices` utilization is NVML's device-wide sampled estimate and includes + other processes on the GPU. `gpu_activity` lists the target's kernels with + summed durations, which double-count overlap. +- Full CPU, syscall, and GPU inventories are written to the artifacts listed in + `sources`; read them when the inline top five are not enough. + +## Exact measurements and graphs + +Start from `quality.verdict`: + +| Issue | Severity | Meaning | +| --- | --- | --- | +| `TIMED_OUT`, `INCOMPLETE_SESSION` | caution | Partial statistics | +| `HEURISTIC_CORRELATION`, `AMBIGUOUS_CORRELATION`, `AMBIGUOUS_PAIRS` | caution | Pairing is temporal or not unique | +| `UNMATCHED_EVENTS` | caution when unmatched exceed matched, else info | Selector scope or window edges | +| `CLOCK_ERROR_LARGE` | caution | Clock error ≥ 10% of p50 | +| `CLOCK_ERROR_UNAVAILABLE` | info | CUPTI gave no interpolation bound | +| `FEW_SAMPLES` | info | Tail ranks rest on < 100 pairs | + +Drops, no matched pairs, target changes, incomplete collection, and clock +misalignment are command errors, never results. Inspect the rank exemplars for +selector scope, thread or stream identity, and correlation IDs; use +`--evidence full` or the artifact when a claim depends on individual pairs. + +`completed` means the samples or duration bound was reached; `timed_out` is +partial. `duration-ms` measures from the first selected event and starts after +the capture is armed, so finish warmup before invoking `measure`. + +## Inventories + +CPU sampling, syscall aggregation, and GPU aggregation are separate contracts; +do not compare their capacities or completeness as one collector. + +- CPU samples: check lost samples, stack truncation, thread coverage, group + table use, and resolved native, Python, and unresolved frame counts before + using hotspot names. Proportions are sampled estimates. `python_status` + `inactive` or `unsupported` means native fallback; native frames remain + valid, Python semantics are not claimed. +- Syscall aggregation: durations are exact for matched lifecycles, but groups + have no order or percentiles; check unmatched, inflight, and drops. +- GPU aggregation: count, summed, min, max, and mean duration and bytes per + group; no ordering, overlap, or correlation. + +## GPU concurrency + +Group GPU evidence by device, context, and stream. Events on different streams +overlap, and their order is not a request relationship. For wall-clock GPU +busy time, run the bundled analyzer on an exact Event JSONL artifact: + +```bash +scripts/analyze_trace.py capture.jsonl > capture-analysis.json +``` + +Use `gpu.busy_union_ns` for wall-clock busy time and `overlap_factor` to see +concurrency; summed activity is not wall time when the factor exceeds 1. +`launch_variants` separates one kernel name by grid and block shape, and +`streams` reports per-stream counts and adjacent gaps. The analyzer describes +one capture window; it never relates distinct windows or assigns work to +requests. + +## Perturbation + +Profiling changes the workload. Record an application-level baseline before +and after, report whether the CUPTI Agent was injected (it stays mapped but +disabled), and treat a materially slower profiled run as a reason to narrow +selectors or shorten the window rather than as the answer. diff --git a/skills/xprobe-measure-latency/references/investigation.md b/skills/xprobe-measure-latency/references/investigation.md deleted file mode 100644 index 392479b..0000000 --- a/skills/xprobe-measure-latency/references/investigation.md +++ /dev/null @@ -1,226 +0,0 @@ -# Investigation playbook - -## Coordinate the workload - -Wait for the service readiness signal, CUDA context creation, framework warmup, -and JIT compilation. Run `discover` only after the target has submitted CUDA -work. Start a bounded measurement immediately before a controlled request/batch, -or while repeatable traffic is active. A quiet target commonly produces -`NO_MATCHED_SAMPLES`; it is not evidence that the selector is invalid. - -Keep launcher and worker roles separate. `discover --pid ROOT_PID` reports only -CUDA context holders, so a wrapper PID should not be selected unless it appears -as a candidate. Map GPU UUID and command/rank metadata to workload ownership. -For multiple relevant workers, follow [multi-process.md](multi-process.md): -inventory a representative worker where homogeneity is defensible, then run -independent narrow captures concurrently. After a restart, rerun discovery and -use PID plus procfs start time; never reuse an old PID-only choice. - -## Map broadly before selecting - -Choose `REPRESENTATIVE_WINDOW_MS` to cover one steady-state request, batch, or -iteration cycle, not an arbitrarily short interval. Select the broad source -from the question: sampled stacks for unknown CPU time, syscall aggregation for -kernel-facing behavior, and activity aggregation for unknown GPU operations. -Do not run all three by default. - -Unknown CPU or Python work starts with sampling: - -```bash -xprobe validate --pid "$PID" --cpu-sample \ - --json --non-interactive --no-color - -xprobe measure --pid "$PID" --cpu-sample \ - --duration-ms "$REPRESENTATIVE_WINDOW_MS" \ - --frequency-hz 99 --max-samples 10000 --max-groups 256 \ - --stack-depth 64 --max-threads 1024 --timeout-ms "$TIMEOUT_MS" \ - --json --non-interactive --no-color > coarse-cpu.json -``` - -Use syscall aggregation only when the question or sampled stacks suggest I/O, -memory mapping, allocation, scheduling, or another kernel-facing boundary: - -```bash -xprobe validate --pid "$PID" --syscall-aggregate \ - --json --non-interactive --no-color - -xprobe measure --pid "$PID" --syscall-aggregate \ - --duration-ms "$REPRESENTATIVE_WINDOW_MS" \ - --max-groups 256 --max-inflight 1024 --timeout-ms "$TIMEOUT_MS" \ - --json --non-interactive --no-color > coarse-syscalls.json -``` - -When kernel names are unknown, collect bounded aggregate kernel activity: - -```bash -xprobe validate --pid "$PID" \ - --from cuda:kernel_start --to cuda:kernel_end --match exact \ - --json --non-interactive --no-color - -xprobe measure --pid "$PID" \ - --from cuda:kernel_start --to cuda:kernel_end --match exact \ - --aggregate --duration-ms "$REPRESENTATIVE_WINDOW_MS" \ - --timeout-ms "$TIMEOUT_MS" --max-groups "$MAX_GROUPS" \ - --json --non-interactive --no-color > coarse-kernels.json -``` - -Map copies and memsets in separate bounded captures when they could matter: - -```bash -xprobe validate --pid "$PID" \ - --from cuda:memcpy_start --to cuda:memcpy_end --match exact \ - --json --non-interactive --no-color - -xprobe measure --pid "$PID" \ - --from cuda:memcpy_start --to cuda:memcpy_end --match exact \ - --aggregate --duration-ms "$REPRESENTATIVE_WINDOW_MS" \ - --timeout-ms "$TIMEOUT_MS" --max-groups "$MAX_GROUPS" \ - --json --non-interactive --no-color > coarse-memcpy.json -``` - -Do not guess a named kernel or host function in this stage. CUDA API selectors -require a concrete Runtime or Driver API name, so inventory activity first and -choose API boundaries from application, framework, or trace evidence later. -Read each inventory separately. Use its selector hints to start the next exact -measurement; device-specific groups may still need workload-level GPU routing. - -Treat group capacity as a consequence of workload diversity, not event rate. -On `EVENT_RATE_TOO_HIGH`, split event families or reduce selector scope while -retaining a representative cycle; reduce duration only when the remaining -window is still representative. CPU, syscall, and GPU aggregate modes never -turn incomplete output into successful evidence. - -For a mixed workload whose CPU and GPU phases must cover the same controlled -request, launch the CPU sample and relevant GPU aggregate as independent -concurrent commands. Use distinct output paths and bounds, preserve either -failure, and measure workload throughput against a no-profiler baseline. These -inventories support narrowing but do not establish cross-source causality. - -## Derive CUDA selectors - -Use `kernels.by_name` and `selector_hint` from the analysis report. Kernel -selectors accept a regular expression, but the CUPTI hot path can lower only a -short exact, prefix, suffix, or contains literal (under 128 characters). Complex -regex is applied after a broader capture and may fill capacity quickly. Escape -regex metacharacters and validate the final selector. - -For Triton and other JIT kernels, use captured names plus grid/block variants to -identify a launch family, then correlate it with framework cache metadata, -generated source, or application logs. xprobe does not read JIT cache contents -and cannot select by grid/block. Use each aggregate group's emitted selector -hint directly. A `name_complete: false` group contains a bounded observed prefix -and therefore emits a prefix hint; do not add an exact end anchor. When a hint -still covers several groups, derive a shorter observed-unique prefix or -contains literal and validate it before collection. - -## Derive CPU selectors - -Read `collection.completeness`, `observed_samples`, `grouped_samples`, -`lost_samples`, stack truncation, thread coverage, and table utilization before -ranking `inventory.hotspots`. -Then read resolved native/Python/unresolved frame totals and `python_status`. -An `active` Python map can expose Python function frames. An `inactive` or -`unsupported` status uses a visible native fallback; incomplete Python coverage -also leaves native CPython, extension, framework, and system-library frames -usable. Report that fallback instead of claiming Python semantic attribution. - -Prefer each hotspot's `entry_selector_hint` and `return_selector_hint`. These -are emitted only for attachable native ELF symbols observed in this target and -still require validation. When no hint exists and an exact native boundary is -necessary, the agent may inspect the mapped object itself: - -```bash -readlink -f "/proc/$PID/exe" -cat "/proc/$PID/maps" -readelf -Ws /path/to/object -nm -D --defined-only /path/to/object -nm -D --defined-only --demangle /path/to/cpp-object -``` - -Use `uprobe:::entry|return` when a symbol is available. For -an exact C++ signature containing `::`, use -`uprobe::symbol=:entry|return`; validation -returns both the mangled attach name and readable signature. For stripped or -local code, derive a file offset with `readelf`/`objdump` and use -`uprobe::+0xOFFSET:entry|return`. Always pass the exact candidate to -`validate`; do not infer a runtime virtual address from one process and reuse it -as a file offset. - -For eager PyTorch, prefer a sampled hotspot or emitted selector hint from the -mapped CPython executable, `torch._C`, or a loaded libtorch object. A validated -exported dispatcher or native operator signature such as an -`at::_ops::::call(...)` boundary can be measured with `stack-nested`. -Treat `_PyEval_EvalFrameDefault` only as a native interpreter boundary when -Python semantic frames are unavailable. - -After `torch.compile` or Triton warmup, do not assume an eager operator boundary -still encloses the fused work. Inventory CUDA kernels first and narrow using -the emitted kernel names. Generated CPU code without a stable file-backed ELF -symbol is not a valid uprobe target. Never reuse a C++ signature, mangled name, -file offset, or generated kernel name across PyTorch builds without resolving -and validating it again. - -For kernel-facing latency, use application evidence or a bounded syscall -summary to identify a candidate. Then validate -`syscall:NAME:entry` to `syscall:NAME:exit` with `exact`. Use -`tracepoint:CATEGORY:NAME` only when the kernel event itself is the intended -boundary. Do not start an unknown high-rate workload with unfiltered raw -syscall tracepoints: select first, then collect detailed evidence. - -For a garbage-collection hypothesis, check target capability rather than its -process name: - -```bash -xprobe validate --pid "$PID" \ - --from python:gc_start --to python:gc_end --match exact \ - --json --non-interactive --no-color -``` - -If valid, measure a bounded number of exact same-thread GC lifecycles. If the -mapped CPython build lacks `python:gc__start` and `python:gc__done` USDT notes, -validation rejects the selector; retain sampled native/Python evidence and do -not substitute a guessed symbol. - -## Measure one narrow hypothesis - -Choose one next boundary from evidence: - -- kernel start to end with `exact` for kernel duration; -- CUDA API exit to kernel start with `exact` for launch delay; -- kernel end to next activity start with `stream-order` for one-stream gaps; -- host function entry to return with `stack-nested` for CPU span; -- named syscall entry to exit with `exact` for kernel-facing latency; -- Python GC start to end with `exact` for one CPython collection lifecycle; -- host marker to GPU activity with `first-after` only as a disclosed heuristic. - -After capture, analyze the artifact and all result quality fields. An aggregate -inventory cannot be re-correlated because it intentionally contains no events. -Collect one exact artifact for the selected hypothesis, then use -`measure --input` when only selectors or policy change. - -For 100 requested narrow pairs, begin with about 10x record capacity rather -than a six-figure default. Two records per pair is the theoretical minimum; -the remaining headroom covers unmatched boundaries and collector stop latency. -Increase capacity only when the quality fields show it is necessary and the -selectors cannot be narrowed without changing the hypothesis. - -```bash -xprobe measure --pid "$PID" \ - --from 'cuda:kernel_start:name~^selected_kernel$' \ - --to 'cuda:kernel_end:name~^selected_kernel$' \ - --match exact --samples 100 --max-events 1000 \ - --events-out selected-kernel.jsonl --format jsonl \ - --json --non-interactive --no-color - -skills/xprobe-measure-latency/scripts/analyze_trace.py selected-kernel.jsonl \ - > selected-kernel-analysis.json -``` - -## Escalate at the right boundary - -xprobe can isolate sampled CPU hotspots, CPython GC, syscall cost, slow kernels, -launch gaps, copies, synchronization boundaries, host spans, and host-to-GPU -timing. Once the remaining time is inside a single kernel, use NCU or PC sampling -for stalls, cache behavior, occupancy, instruction mix, or Tensor Core -utilization. When sampled native code has no stable observable boundary, hand it -to a runtime-specific profiler or add application instrumentation. diff --git a/skills/xprobe-measure-latency/references/multi-process.md b/skills/xprobe-measure-latency/references/multi-process.md index fc44702..a7f45f7 100644 --- a/skills/xprobe-measure-latency/references/multi-process.md +++ b/skills/xprobe-measure-latency/references/multi-process.md @@ -1,95 +1,41 @@ # Multi-process workloads -Keep process selection and concurrency in the agent framework. xprobe has no -multi-process command: each `validate` and `measure` remains an independent, -bounded invocation for one stable target identity. +xprobe has no multi-process command. Selection and concurrency stay in the +agent; each `validate` and `measure` targets one PID plus process start time. ## Select workers -Run one `discover` after workload warmup and retain the full candidate records -used for selection. Select workers from application ownership, command/rank -metadata, GPU UUIDs, and the intended experiment. Do not select every candidate -merely because it owns a CUDA context. +Run `discover --pid ROOT` after warmup and keep the candidate records. Select +from application ownership, command line, rank, and GPU UUID, not merely +because a process holds a CUDA context. For workers that configuration and +evidence show to be homogeneous, triage one representative worker and reuse its +selectors; otherwise treat each class separately. -Treat workers as homogeneous only when application configuration and observed -workload evidence support that claim. For a homogeneous rank set, use one -representative worker for each required CPU sample, syscall, or GPU aggregate -inventory. Derive narrow -selectors from that inventory, then apply those selectors to every selected -worker. For heterogeneous workers, inventory one representative per defensible -class or investigate workers separately. +Name every spec, result, stderr log, and artifact with the worker's +`target.pid` and `target.process_start_time`. Never reuse an artifact path +across workers or attempts. -Record each selected candidate's exact `target.pid` and -`target.process_start_time`. Name every spec, result, stderr log, and event -artifact with both values. Never reuse an artifact path across workers or -attempts. +## Validate, then capture concurrently -## Revalidate identity +Write one spec per worker with its own bounds. Validate each and require that +the response's `target` exactly matches the selected identity; a mismatch or +failure removes that worker rather than substituting a new PID. Disclose any +`injection_required` worker before measuring. -Create one narrow Measurement Spec per selected worker. Copy its full discovery -identity into `target`; give every spec its own samples or duration, -`timeout_ms`, and `max_events`. Capacities are per worker, never a shared budget. +Launch one bounded `measure --spec` per worker as native concurrent tool calls +so the windows cover the same workload cycle. Wait for every command. Do not +cancel sibling commands when one fails; mark the overall experiment incomplete +and report usable per-worker results separately. -Issue one read-only `validate` call per selected worker, concurrently when the -agent framework supports concurrent tool calls. Before any mutation, require -that each validation response is valid and that its `target` exactly matches -the selected PID and `process_start_time`. Preserve each validation JSON and its -warnings. A mismatch, exit, or failed validation removes that worker from the -measurement batch; do not replace it with a newly observed PID. - -When validation reports `injection_required`, disclose and retain that worker's -mutation warning. Do not preload or repeatedly broad-profile all ranks merely -to make injection look uniform. Injection completes before that worker's ARM -window, but its application-level startup perturbation still belongs in the -report. - -## Capture concurrently - -After every selected identity has passed validation, launch one bounded -`xprobe measure --spec WORKER_SPEC` per selected worker using native concurrent -tool calls from the agent framework. Start the calls in one concurrency batch -so their capture windows cover the same controlled request, batch, or iteration -cycle. Do not serialize the commands unless the investigation explicitly needs -different workload windows. - -The same rule applies to mixed evidence on one worker: a CPU sample inventory -and GPU aggregate may run as two concurrent xprobe commands when they must cover -the same controlled window. They keep independent duration, timeout, sample or -group capacities, result schemas, stderr, and output paths. Do not merge them or -infer exact CPU/GPU causality from overlapping command times. Avoid concurrent -broad CUPTI captures for the same process because Agent activation and shared -workload perturbation make them poor independent observations. - -Keep these outputs separate for every worker: - -- spec with the discovered target identity; -- validation JSON and stderr; -- measurement JSON and stderr; -- exact Event JSONL artifact when requested; -- separate CPU, syscall, and GPU inventory JSON when collected; -- command exit status and start/end wall timestamps. - -Wait for every command. Do not cancel sibling commands or discard successful -outputs when one worker fails. Preserve injection warnings, configured capacity, -collection quality, and the structured error for each worker. Mark the overall -experiment incomplete when any selected worker fails, while describing usable -single-worker evidence separately. +On one worker, a CPU sample inventory and GPU aggregate may also run as two +concurrent commands with separate bounds and outputs. Prefer triage when one +window must cover both. ## Interpret without merging -Read every result's status, completeness, drops, utilization, ambiguity, clock, -correlation, and evidence independently. Compare per-worker distributions only -after checking that selectors, workload phase, device assignment, and quality -are comparable. - -Do not concatenate artifacts and rerun correlation across processes. CUPTI -correlation IDs, CUDA contexts, streams, host thread identity, and process -clocks are process-scoped evidence; overlapping wall timestamps do not establish -cross-process causality. Report variation or concurrent observation, not an -exact relationship between worker events. - -Measure an application baseline before the batch and again afterward. Report -per-worker command startup time and workload throughput during collection. -Concurrent captures can perturb shared CPU, GPU, storage, and driver resources, -so a result that changes the workload materially needs a smaller worker set, -narrower selectors, or a shorter but still representative window. +Read each result's quality independently and compare per-worker distributions +only when selectors, phase, and device assignment match. Do not concatenate +artifacts and correlate across processes: correlation IDs, streams, and threads +are process-scoped, and overlapping timestamps do not establish a relationship. +Concurrent captures perturb shared resources; record a workload baseline and +reduce the worker set if throughput changes materially. diff --git a/skills/xprobe-measure-latency/references/result-quality.md b/skills/xprobe-measure-latency/references/result-quality.md deleted file mode 100644 index 161374c..0000000 --- a/skills/xprobe-measure-latency/references/result-quality.md +++ /dev/null @@ -1,96 +0,0 @@ -# Result quality - -Prefer `exact` when CUDA endpoints carry the same CUPTI correlation ID or when -one named syscall has entry/exit records from the same thread and process -identity. CPython GC exact matching likewise requires ordered start/end markers -on the same thread and process identity. Prefer `stack-nested` for entry/return pairs of the same host -function. Use `stream-order` only for GPU activity endpoints on the same device, -context, and stream. Treat `first-after` and `nearest` as temporal heuristics, -never request causality. - -Start from `quality.verdict` and its issues, then inspect the rank exemplar -pairs for selector scope, process identity, correlation IDs, -device/context/stream, timestamps, and clock domains. When a claim depends on -individual pairs, use `--evidence full` or re-correlate the evidence artifact -with `measure --input`. A result is not -sufficient when records were dropped, no samples matched, target identity -changed, collection was incomplete, or clock alignment failed. Report unmatched -and ambiguous counts with matched samples. `estimated_error_ns: null` means no -quantified interpolation error bound. - -Inventory modes are separate contracts: - -- For CPU sampling, require `completeness: complete`, zero lost samples, - `observed_samples == grouped_samples`, acceptable stack truncation, expected - thread coverage, and reasonable group-table utilization. Read native, - Python, and unresolved frame counts before using hotspot names. Sample - proportions have sampling uncertainty and are not exact time shares. -- For syscall aggregation, require `completeness: complete`, zero dropped - aggregates, acceptable unmatched/inflight counts, and reasonable group-table - utilization. Durations are exact for retained matched lifecycles, but groups - contain no event order or percentiles. -- For GPU aggregate inventory, require `completeness: complete`, zero dropped - activities, equal observed and grouped activity counts, and reasonable table - utilization. Min/max/mean derive from integer totals, but there is no event - ordering, stream overlap, or correlation evidence. - -Do not compare capacities or completeness fields across these schemas as if -they described the same collector. - -## Concurrency - -Group GPU evidence by device, context, and stream before interpreting order. -Events on different streams may overlap and timestamp order does not establish a -request relationship. `stream-order` does not cross stream boundaries. Prefer -correlation-ID `exact` when a deterministic relationship exists. - -Summed kernel or activity duration double-counts overlap. Use GPU `busy_union_ns` -for wall-clock busy time and `overlap_factor` to quantify concurrency. Compare -per-stream gaps separately. A top kernel's `summed_kernel_time_share` describes -its share of summed kernel work, not its exclusive share of wall time. - -For multi-process captures, keep every result and artifact scoped to its PID -plus process start time. Compare worker summaries only after checking individual -quality and workload alignment. Do not merge Event JSONL files for correlation: -timestamp overlap, matching names, or matching correlation IDs across processes -does not establish causality. Preserve partial command failures instead of -silently reporting only successful workers. - -## Bounds and completion - -`completed` means either requested samples or duration was reached. `timed_out` -is partial and must be reported as such. For one complete kernel duration, start -and end consume at least two CUPTI records; unmatched boundaries and records -admitted by a broad hot-path filter consume additional capacity. Sample -completion is evaluated from snapshots, so a high-rate buffer can reach -`max-events` before the caller observes the requested sample count. - -For narrow start/end activity pairs, begin with: - -```text -minimum_records = samples * (start_records_per_sample + end_records_per_sample) -max_events >= minimum_records + expected_unmatched_records -``` - -Use at least 2x headroom for stable narrow selectors. For broad aggregate -inventory, size `max-groups` from expected operation or syscall diversity rather -than event rate. For CPU sampling, size `max-samples` from frequency times -duration, `max-threads` from target fanout, `stack-depth` from expected call -depth, and `max-groups` from stack diversity. Keep enough duration to cover a -representative cycle; saturation or sample loss is an explicit quality failure. - -`duration-ms` limits correlation to a window beginning at the first selected -event. In live mode it also sets a collection stop from ARM completion, so finish -readiness and warmup before invoking `measure`. When both samples and duration -are set, either bound completes the call; timeout remains the outer operation and -cleanup limit. - -## Perturbation - -Record an application-level latency distribution before profiling and repeat it -afterward under the same workload. Report the difference and whether automatic -injection occurred. The injected shared object remains mapped but is logically -disabled after collection. For mixed concurrent inventories, report each -collector's wall time, available CPU/RSS observations, and combined workload -throughput. Do not interpret a one-off profiled request as an unperturbed -baseline or claim universal superiority over another profiler. diff --git a/skills/xprobe-measure-latency/references/selectors.md b/skills/xprobe-measure-latency/references/selectors.md new file mode 100644 index 0000000..3fb58e6 --- /dev/null +++ b/skills/xprobe-measure-latency/references/selectors.md @@ -0,0 +1,94 @@ +# Selectors, policies, and specs + +Prefer selectors that xprobe emitted: triage `next.spec`, inventory +`*_selector_hint` fields, and analyzer `selector_hint`. Write one by hand only +when no emitted hint covers the boundary, and pass it through `validate`. + +## Selector syntax + +| Source | Selector | +| --- | --- | +| Native function | `uprobe:::entry\|return` | +| C++ signature with `::` | `uprobe::symbol=:entry\|return` | +| Stripped or local code | `uprobe::+0x:entry\|return` | +| Syscall | `syscall::entry\|exit` | +| Kernel tracepoint | `tracepoint::` (identity and timestamp only) | +| CPython GC | `python:gc_start`, `python:gc_end` | +| CUDA API | `cuda:runtime_api\|driver_api::entry\|exit` | +| GPU activity | `cuda:kernel_start\|kernel_end[:name~REGEX]`, `cuda:memcpy_start\|memcpy_end[:kind=HtoD\|DtoH\|DtoD\|HtoH\|PtoP]`, `cuda:memset_start\|memset_end` | +| NVTX range | `cuda:nvtx_range_start\|nvtx_range_end:name~REGEX` | + +CUDA API selectors need a concrete Runtime or Driver API name. Kernel and NVTX +regexes are filtered in the CUPTI hot path only when they reduce to an exact, +prefix, suffix, or contains literal under 128 bytes; anything else is applied +after a broader capture and fills capacity faster. Escape regex metacharacters. +A `name_complete: false` kernel group carries a truncated prefix: keep its +prefix hint and do not add an end anchor. + +## Derive a native boundary + +Use a CPU sample hotspot's `entry_selector_hint` and `return_selector_hint` +first. Without one, inspect the mapped object of this target: + +```bash +readlink -f "/proc/$PID/exe" +grep -F ' r-xp ' "/proc/$PID/maps" +readelf -Ws /path/to/object +nm -D --defined-only --demangle /path/to/object +``` + +Use a symbol when one exists; otherwise derive a file offset with `readelf` or +`objdump`. Never reuse a runtime virtual address as a file offset, and never +reuse a C++ signature, offset, or generated kernel name across builds without +validating it again. + +For eager PyTorch, prefer sampled boundaries in CPython, `torch._C`, or a +loaded libtorch object, such as `at::_ops::::call(...)` with +`stack-nested`. After `torch.compile` or Triton warmup, fused work no longer +sits inside eager operator boundaries: select by kernel name instead. Generated +CPU code without a file-backed ELF symbol cannot be a uprobe target. + +For CPython GC, check capability, not the process name: `validate` resolves the +USDT markers in the mapped interpreter or `libpython` and rejects builds without +them. Do not substitute a guessed symbol. + +```bash +xprobe validate --pid "$PID" --from python:gc_start --to python:gc_end --match exact +``` + +## Choose a policy + +| Policy | Use for | Confidence | +| --- | --- | --- | +| `exact` | Same CUPTI correlation ID (API ↔ its kernel or copy), kernel/copy start ↔ end, NVTX range ID, one syscall's entry ↔ exit on a thread, CPython GC start ↔ end | exact | +| `stack-nested` | Entry ↔ return of one host function per thread | high | +| `stream-order` | Activity end ↔ next start on one device stream | high | +| `first-after`, `nearest` | Host marker ↔ GPU activity and other unrelated sources | heuristic | + +`validate` reports `policy_recommendation` with a reason and the compatible +set; xprobe never changes the requested policy silently. Heuristic policies +never establish request-level causality. + +## Specs + +A single-edge `MeasurementSpec` (schema `2.0`) pins the target identity and +bounds; triage emits ready ones, and `examples/` holds templates. Run it with +`measure --spec FILE`. + +A measurement graph (schema `3.0`) names endpoints once and measures up to +eight edges from one capture. See `examples/request-path-graph.json`. Rules: + +- 1–8 endpoints and 1–8 edges; every edge sets `samples` or the graph sets + `duration_ms`; `max_events` bounds the whole capture. +- At most two syscall, tracepoint, or USDT endpoints; uprobe endpoints are not + limited further; CUDA endpoints share eight Agent filters. +- `validate --spec graph.json` reports issues scoped to the graph, an endpoint, + or an edge. Any failing edge fails `measure`; the artifact is still written. + +## Capacity + +Start an exact capture with modest record headroom: +`max_events ≈ 2 × samples × (records per start + records per end)` plus +expected unmatched records. Triage specs size capacity from the observed rate. +On `EVENT_RATE_TOO_HIGH`, narrow the selector before raising `max_events`, and +keep the window representative. diff --git a/skills/xprobe-measure-latency/references/setup.md b/skills/xprobe-measure-latency/references/setup.md index b0ee4ae..fe994d7 100644 --- a/skills/xprobe-measure-latency/references/setup.md +++ b/skills/xprobe-measure-latency/references/setup.md @@ -1,72 +1,48 @@ # Setup -Use this reference at the start of a task when the CLI is absent or its version -does not match this Skill. The user installs this Skill once; the agent performs -and verifies the CLI bootstrap. Do not ask the user to run a second installation -command unless their environment prevents the agent from writing a usable prefix. +The user installs this Skill once; the agent bootstraps and verifies the CLI. +Offline analysis of an existing artifact needs no CLI. ## Check and bootstrap xprobe -For live work, check the executable first. This Skill supports xprobe `0.5.x` -with schema version `2.0`; install the current release when the CLI is absent, -outside that range, or fails its required capability checks. Offline analysis -of an existing schema-v2 artifact does not require an installed CLI. - -```bash -if command -v xprobe >/dev/null 2>&1; then - xprobe --version -fi -``` - -Download the released bootstrap, which verifies the release archive before -installing under `~/.local`: +This Skill supports xprobe `0.5.x`. Install the matching release when the CLI is +absent, reports another version, or fails a required capability: ```bash +command -v xprobe >/dev/null 2>&1 && xprobe --version curl --proto '=https' --tlsv1.2 -fsSL \ https://raw.githubusercontent.com/itdevwu/xprobe/v0.5.2/install.sh \ -o /tmp/xprobe-install.sh sh /tmp/xprobe-install.sh export PATH="$HOME/.local/bin:$PATH" xprobe --version -xprobe doctor --json --non-interactive --no-color +xprobe doctor ``` -Use the downloaded script's `--prefix` option for another installation root. -Export that prefix's `bin` directory into `PATH`, rerun `xprobe --version`, then -run `xprobe doctor --json --non-interactive --no-color`. Surface bootstrap, -PATH, permission, driver, CUDA, or CUPTI failures explicitly and adjust from the -reported detail; do not continue to measurement on an unverified installation. -The CLI needs no Node.js. CUDA is optional until a GPU selector is measured. - -## Run with a containerized live target - -A host-side `measure` cannot safely inject when the target has a different -mount namespace. Do not retry with a guessed Agent path. Run the matching -xprobe release in the application container itself, or have an explicitly -privileged launcher enter both the target PID and mount namespaces. `docker -exec` or `kubectl exec` is suitable only after the caller has identified the -exact container from workload evidence. An adjacent sidecar or ephemeral -container normally has a different mount namespace and is not equivalent. - -Bootstrap xprobe under a writable prefix in that namespace, then resolve the -target again from its procfs view. Record the namespace-local PID and -`/proc/PID/stat` start time, and rerun `doctor` and read-only `validate` there -before measurement. Never carry a host PID or an earlier process identity into -the container command. Keep stdout, stderr, status, and artifacts outside the -container through an explicit writable or shared path. - -Entering a namespace does not add capabilities that the running container was -not granted. Surface missing ptrace, perf, BPF, NVIDIA device, or CUPTI access -as an environment failure; do not silently fall back to a host-side injection -or a different container. For NVTX, the matching Agent must still be configured -in the application container before its first NVTX API call. - -## Build locally when the release is unsuitable - -The `glibc 2.34` requirement applies to the precompiled release archive. When -that archive cannot run on the host, build the matching source locally with the -host glibc instead. This is a local-use fallback, not permission to weaken the -release package's `GLIBC_2.34` ceiling. +The bootstrap verifies the archive and installs under `~/.local`; pass +`--prefix DIR` for another root and export its `bin`. Surface PATH, +permission, driver, CUDA, or CUPTI failures from `doctor` details; do not +measure on an unverified installation. CUDA is needed only for GPU selectors. + +## Containerized live target + +A host-side `measure` cannot inject into a target with a different mount +namespace. Run xprobe in the application container itself, or through a +launcher that enters both its PID and mount namespaces; a sidecar is not +equivalent. Do not guess a pod, container, or PID: identify the container from +workload evidence first. + +Inside, resolve the namespace-local PID and its `/proc/PID/stat` start time +again, rerun `doctor` and `validate` there, and write outputs to a path shared +with the caller. Entering a namespace does not add capabilities the container +was not granted: report missing ptrace, perf, BPF, NVIDIA device, or CUPTI +access as an environment failure instead of falling back to host-side +injection. + +## Build locally when the release cannot run + +The release archive needs host glibc 2.34 or newer. Otherwise build the +matching source with the host glibc: ```bash git clone --depth 1 --branch v0.5.2 https://github.com/itdevwu/xprobe.git @@ -74,16 +50,10 @@ cd xprobe mamba env create --file environment.yml mamba run -n xprobe-dev just build export PATH="$PWD/target/debug:$PATH" -xprobe --version -xprobe doctor --json --non-interactive --no-color ``` -For CPU-only work, this build is sufficient; no CUDA toolkit or CUPTI Agent is -required. Do not use `scripts/package-release.sh` for this path because it -enforces the distributable archive compatibility policy. - -For GPU or mixed work, build one local Agent against the target's CUDA 12 or -CUDA 13 toolkit, then point the CLI at it: +That suffices for CPU work. For GPU work, build one Agent against the target's +CUDA 12 or 13 toolkit and export it: ```bash mamba run -n xprobe-dev env CUDA_PATH=/opt/cuda-12 cmake -S . -B build/cuda12 -G Ninja \ @@ -92,40 +62,27 @@ mamba run -n xprobe-dev cmake --build build/cuda12 --target xprobe-bpf xprobe-cu export XPROBE_CUPTI_AGENT_PATH="$PWD/build/cuda12/cupti/libxprobe-cupti.so" ``` -Replace `12` and the toolkit path with `13` for CUDA 13. CUDA/CUPTI majors other -than 12 or 13 are not supported: do not force an Agent build or bypass the -version check. For an NVTX range measurement, start or restart the target with -the same local Agent configured before the target's first NVTX call: +CUDA/CUPTI majors other than 12 or 13 are not supported. Do not use +`scripts/package-release.sh` for a local build; it enforces the release glibc +ceiling. + +## NVTX ranges + +Online injection cannot attach to an already initialized NVTX dispatch. Restart +the target with the Agent configured before its first NVTX call, record the new +PID and start time, and validate again: ```bash NVTX_INJECTION64_PATH="$XPROBE_CUPTI_AGENT_PATH" python serve.py ``` -Record the new PID plus procfs start time and run `validate` again. Do not use -online injection as a fallback for an already initialized NVTX process. +## Repair the Skill -## Repair the Skill only when needed - -The user normally installed this Skill before invoking the agent. When its files -are missing or incompatible with xprobe `0.5.x`, install the complete current -release directory through the Agent Skills CLI: +When Skill files are missing or mismatched, reinstall the whole directory; its +references, examples, and analyzer are part of the contract: ```bash npx skills@1 add \ https://github.com/itdevwu/xprobe/tree/v0.5.2/skills/xprobe-measure-latency \ --global ``` - -For non-interactive automation, select the host explicitly: - -```bash -npx --yes skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.5.2/skills/xprobe-measure-latency \ - --agent codex --global --copy --yes -``` - -Replace `codex` with `claude-code` or `cursor` as needed. Omit `--global` for a -repository-scoped installation. Without Node.js, copy the full Skill directory -from the release archive into the target agent's documented Skill location; do -not copy only `SKILL.md`, because its references, examples, and analyzer are -part of the contract. diff --git a/skills/xprobe-measure-latency/references/trace-analysis.md b/skills/xprobe-measure-latency/references/trace-analysis.md deleted file mode 100644 index a745d60..0000000 --- a/skills/xprobe-measure-latency/references/trace-analysis.md +++ /dev/null @@ -1,37 +0,0 @@ -# Trace analysis - -Run the bundled analyzer on an Event JSONL artifact: - -```bash -skills/xprobe-measure-latency/scripts/analyze_trace.py capture.jsonl \ - > capture-analysis.json -``` - -The command rejects non-2.0 events, clock-domain mismatches within an activity, -and negative durations. Unpaired activity boundaries remain visible in -`events` and produce a warning; they are excluded from duration metrics. - -Read the report in this order: - -1. `events`: confirm counts and zero unpaired starts/ends. -2. `gpu.busy_union_ns`: wall-clock union of kernel, memcpy, and memset activity. -3. `gpu.summed_activity_ns` and `overlap_factor`: quantify overlap; do not use - summed activity as wall time when the factor is above 1. -4. `kernels.by_name`: rank total time, then inspect p50/p95 and - `summed_kernel_time_share`. -5. `launch_variants`: separate the same kernel name by grid/block shape before - concluding that its latency distribution is homogeneous. -6. `streams`: compare activity counts and adjacent kernel gaps per - PID/device/context/stream. Cross-stream order is not causal evidence. -7. `memcpy.by_kind`: inspect transfer duration and bytes separately from kernel - work. - -`selector_hint` is exact for names shorter than the CUPTI filter bound. For long -names it may be a prefix or suffix unique only among names observed in that -capture; validate it and repeat a narrow measurement before treating it as -stable. Analyze every broad inventory separately; the report never establishes -wall-clock relationships between distinct capture windows. - -The analyzer reports descriptive activity timing. It does not assign work to a -request, calculate kernel hardware efficiency, or turn temporal proximity into -causality. diff --git a/tests/agent-contract/fixtures/workflow-routes.json b/tests/agent-contract/fixtures/workflow-routes.json index 76a0967..bacd34e 100644 --- a/tests/agent-contract/fixtures/workflow-routes.json +++ b/tests/agent-contract/fixtures/workflow-routes.json @@ -2,38 +2,105 @@ "scenarios": [ { "name": "existing_artifact", - "sources": ["skill"], - "required": ["measure --input", "skip installation", "unless a new capture is required"] + "sources": [ + "skill" + ], + "required": [ + "measure --input", + "skip installation", + "unless a new capture is required" + ] + }, + { + "name": "unknown_workload", + "sources": [ + "skill" + ], + "required": [ + "unknown live workload", + "--triage", + "next.spec", + "not_covered" + ] }, { "name": "known_selector", - "sources": ["skill"], - "required": ["known live boundary", "run read-only validate", "do not run a broad inventory solely to satisfy a checklist"] + "sources": [ + "skill" + ], + "required": [ + "known boundary", + "skip triage", + "validate" + ] }, { - "name": "unknown_cpu", - "sources": ["skill", "investigation"], - "required": ["unknown cpu or python workload", "--cpu-sample", "observed_samples", "entry_selector_hint"] + "name": "several_boundaries", + "sources": [ + "skill", + "selectors" + ], + "required": [ + "measurement graph", + "up to eight edges", + "at most two syscall, tracepoint, or usdt endpoints" + ] }, { - "name": "python_semantics", - "sources": ["skill", "investigation"], - "required": ["python_status", "native frames", "python:gc_start", "capability rather than its process name"] + "name": "dig_deeper", + "sources": [ + "skill", + "deep_dive" + ], + "required": [ + "dig deeper", + "split it with a measurement graph", + "p99 and max exemplars", + "not covered by this release", + "stop escalating" + ] }, { - "name": "mixed_cpu_gpu", - "sources": ["skill", "investigation"], - "required": ["bounded cpu sample inventory", "gpu aggregate", "independent concurrent commands", "separate bounds, outputs, failures"] + "name": "python_semantics", + "sources": [ + "skill", + "interpreting", + "selectors" + ], + "required": [ + "python_status", + "native fallback", + "-X perf", + "python:gc_start", + "check capability, not the process name" + ] }, { - "name": "unsupported_python_runtime", - "sources": ["skill", "investigation"], - "required": ["`inactive` or `unsupported`", "native fallback", "do not claim xprobe identifies python semantics"] + "name": "multi_process", + "sources": [ + "skill", + "multi_process" + ], + "required": [ + "discover --pid root", + "pid plus process start time", + "do not cancel sibling commands", + "do not concatenate artifacts" + ] }, { "name": "container_namespace", - "sources": ["skill", "setup"], - "required": ["containerized live target", "PID and mount namespaces", "namespace-local PID", "do not guess a pod, container, or PID", "does not add capabilities"] + "sources": [ + "skill", + "setup" + ], + "required": [ + "container target", + "pid and mount namespaces", + "namespace-local pid", + "do not guess a pod, container, or pid", + "does not add capabilities" + ] } ] } diff --git a/tests/agent-contract/test_contract.py b/tests/agent-contract/test_contract.py index a66ee99..d22da39 100755 --- a/tests/agent-contract/test_contract.py +++ b/tests/agent-contract/test_contract.py @@ -16,6 +16,8 @@ "measure", } SKILL_PATH = "skills/xprobe-measure-latency/SKILL.md" +SKILL_WORD_BUDGET = 800 +SKILL_TOTAL_WORD_BUDGET = 4000 def run_json( @@ -40,7 +42,21 @@ def run_json( return result -def check_skill(workspace: pathlib.Path) -> None: +def development_version(workspace: pathlib.Path) -> str: + cargo = (workspace / "Cargo.toml").read_text() + version_match = re.search(r'^version = "([^"]+)"$', cargo, re.MULTILINE) + assert version_match is not None + return version_match.group(1) + + +def release_version(workspace: pathlib.Path) -> str: + installer = (workspace / "install.sh").read_text() + release_match = re.search(r"^version=\$\{XPROBE_VERSION:-([^}]+)\}$", installer, re.MULTILINE) + assert release_match is not None + return release_match.group(1) + + +def check_skill(workspace: pathlib.Path, binary: pathlib.Path) -> None: skill_path = workspace / SKILL_PATH skill_root = skill_path.parent.resolve() skill = skill_path.read_text() @@ -48,95 +64,96 @@ def check_skill(workspace: pathlib.Path) -> None: assert frontmatter is not None assert re.search(r"^name: xprobe-measure-latency$", frontmatter.group(1), re.MULTILINE) assert re.search(r"^description: .+", frontmatter.group(1), re.MULTILINE) - normalized_skill = re.sub(r"\s+", " ", skill) - for route in ( - "Existing artifact", - "Known live boundary", - "Unknown CPU or Python workload", - "Unknown GPU or mixed workload", - "Multiple processes", - "Containerized live target", - "Setup or repair", - ): - assert route in normalized_skill - for adaptive_rule in ( - "Choose the shortest route", - "skip installation, `doctor`, `discover`, and live attachment", - "Do not run a broad inventory solely to satisfy a checklist", - "Do not run CUDA discovery", - "Start with bounded `--cpu-sample` evidence", - "Add `--syscall-aggregate` only for a kernel-facing hypothesis", - "inventories can run concurrently", - "modest record headroom", - "Existing artifacts do not require a local collector", - "Run `doctor` when capability is unknown", - ): - assert adaptive_rule in normalized_skill - for invariant in ( - "Measurement results use schema version `3.0`", + # The entry point stays short; details live in linked references. + references = sorted((skill_root / "references").glob("*.md")) + assert len(skill.split()) <= SKILL_WORD_BUDGET, len(skill.split()) + total = len(skill.split()) + sum(len(path.read_text().split()) for path in references) + assert total <= SKILL_TOTAL_WORD_BUDGET, total + + # Every link resolves inside the Skill, and every reference is reachable. + linked = set() + for document in [skill_path, *references]: + for relative_link in re.findall(r"\]\(([^)]+)\)", document.read_text()): + target = (document.parent / relative_link).resolve() + assert target.is_relative_to(skill_root), (document, relative_link) + assert target.is_file(), (document, relative_link) + linked.add(target) + skill_links = { + (skill_root / link).resolve() for link in re.findall(r"\]\(([^)]+)\)", skill) + } + assert {path.resolve() for path in references} <= skill_links, "orphaned reference" + + # The entry point covers the two-command flow and its safety invariants. + normalized_skill = re.sub(r"\s+", " ", skill) + for required in ( + "--triage", + "measure --spec", + "measure --input", + "validate", + "quality.verdict", + "not_covered", + "measurement graph", + "Dig deeper", "PID plus procfs start time", - "Pass the selected exact endpoints through read-only `validate`", "Bound every capture", "leave the CUPTI shared object mapped", "temporal correlation is not exact causality", - ): - assert invariant in normalized_skill - for quality_field in ( - "unmatched", - "ambiguous", - "loss/drops", - "completeness", - "symbol and stack coverage", - "clock alignment", - "method", - "confidence", - "evidence pair", - ): - assert quality_field in normalized_skill - for investigation_step in ( - "CPU-only", - "GPU or mixed", - "Scope breadth and capture duration are independent", - "python:gc_start", - "native frames", "scripts/analyze_trace.py", - "selector hints", - "busy_union_ns", - "overlap factor", "NCU or PC sampling", ): - assert investigation_step in normalized_skill - assert "leave the CUPTI shared object mapped" in normalized_skill + assert required in normalized_skill, required - for relative_link in re.findall(r"\]\(([^)]+)\)", skill): - target = (skill_root / relative_link).resolve() - assert target.is_relative_to(skill_root), relative_link - assert target.is_file(), relative_link + # Every documented xprobe command uses flags its subcommand accepts. + help_flags = {} + for document in [skill_path, *references]: + for block in re.findall(r"```bash\n(.*?)```", document.read_text(), re.DOTALL): + command = re.sub(r"\\\n\s*", " ", block) + for line in command.splitlines(): + words = line.split() + if len(words) < 2 or words[0] != "xprobe" or words[1].startswith("-"): + continue + subcommand = words[1] + if subcommand not in help_flags: + help_text = subprocess.run( + [binary, subcommand, "--help"], + check=True, + capture_output=True, + text=True, + ).stdout + help_flags[subcommand] = set(re.findall(r"--[a-z][a-z0-9-]*", help_text)) + for flag in re.findall(r"(?= 5 - policies = set() + + # Examples are valid templates for each spec shape. modes = set() - for example in examples: + policies = set() + for example in sorted((skill_root / "examples").glob("*.json")): specification = json.loads(example.read_text()) + if "edges" in specification: + assert specification["schema_version"] == "3.0", example + modes.add("graph") + names = set(specification["endpoints"]) + for edge in specification["edges"]: + assert {edge["from"], edge["to"]} <= names, example + policies.add(edge["match_policy"]) + continue assert specification["schema_version"] == "2.0", example if "sample_event" in specification: modes.add("cpu_sample") assert specification["frequency_hz"] > 0, example assert specification["max_samples"] > 0, example - assert specification["stack_depth"] > 0, example - assert specification["max_threads"] > 0, example continue if "max_inflight" in specification and "start_selector" not in specification: modes.add("syscall_aggregate") assert specification["max_groups"] > 0, example - assert specification["max_inflight"] > 0, example continue mode = specification.get("measurement_mode", "exact") modes.add(mode) @@ -146,120 +163,28 @@ def check_skill(workspace: pathlib.Path) -> None: else: assert specification["max_events"] > 0, example policies.add(specification["match_policy"]) - assert modes == {"exact", "aggregate", "cpu_sample", "syscall_aggregate"} + assert modes == {"exact", "aggregate", "cpu_sample", "syscall_aggregate", "graph"} assert {"exact", "first_after", "stack_nested", "stream_order"} <= policies openai_yaml = (skill_root / "agents/openai.yaml").read_text() assert 'display_name: "Xprobe Workload Profiling"' in openai_yaml - assert ( - 'short_description: "Route bounded CPU, Python, and GPU profiling"' - in openai_yaml - ) assert "$xprobe-measure-latency" in openai_yaml - investigation = (skill_root / "references/investigation.md").read_text() - quality = (skill_root / "references/result-quality.md").read_text() - multi_process = (skill_root / "references/multi-process.md").read_text() - trace_analysis = (skill_root / "references/trace-analysis.md").read_text() - setup = (skill_root / "references/setup.md").read_text() - normalized_investigation = re.sub(r"\s+", " ", investigation) - normalized_quality = re.sub(r"\s+", " ", quality) - normalized_multi_process = re.sub(r"\s+", " ", multi_process) - normalized_trace_analysis = re.sub(r"\s+", " ", trace_analysis) - normalized_setup = re.sub(r"\s+", " ", setup) - normalized_cli_contract = re.sub( - r"\s+", " ", (skill_root / "references/cli-contract.md").read_text() - ) - for required in ( - "Triton", - "procfs start time", - "EVENT_RATE_TOO_HIGH", - "readelf -Ws", - "NO_MATCHED_SAMPLES", - "representative cycle", - "concrete Runtime or Driver API name", - "observed_samples", - "python_status", - "python:gc_start", - "independent concurrent commands", - ): - assert required in normalized_investigation - for required in ( - "minimum_records", - "first selected event", - "ARM completion", - "Summed kernel", - "Inventory modes are separate contracts", - "max-groups", - "lost samples", - "stack truncation", - ): - assert required in normalized_quality - for required in ( - "representative worker", - "target.process_start_time", - "target` exactly matches", - "native concurrent tool calls", - "Do not cancel sibling commands", - "Never reuse an artifact path", - "Do not concatenate artifacts", - "overall experiment incomplete", - "CPU sample inventory and GPU aggregate", - ): - assert required in normalized_multi_process + setup = re.sub(r"\s+", " ", (skill_root / "references/setup.md").read_text()) + major, minor, _ = development_version(workspace).split(".") + release = release_version(workspace) for required in ( - "busy_union_ns", - "summed_activity_ns", - "launch_variants", - "Cross-stream", - "distinct capture windows", - ): - assert required in normalized_trace_analysis - for required in ( - "python:gc_start|gc_end", - "--cpu-sample", - "--syscall-aggregate", - "sampling uncertainty", - "cannot be passed to `measure --input`", - ): - assert required in normalized_cli_contract - for required in ( - "v0.5.2/install.sh", - "xprobe `0.5.x`", + f"xprobe `{major}.{minor}.x`", + f"v{release}/install.sh", "npx skills@1 add", "xprobe --version", "xprobe doctor", - "host glibc", - "CUDA/CUPTI majors other than 12 or 13 are not supported", - "scripts/package-release.sh", "PID and mount namespaces", "namespace-local PID", "does not add capabilities", + "CUDA/CUPTI majors other than 12 or 13 are not supported", ): - assert required in normalized_setup - - engineering_rules = re.sub( - r"\s+", " ", (workspace / "AGENTS.md").read_text() - ) - for required in ( - "external reviews as evidence to investigate", - "dedicated feature or fix branch", - "Rebase pull requests", - "GLIBC_2.34 ceiling", - "downloading the public archive", - "transient infrastructure failure", - "Choose the shortest Skill route", - ): - assert required in engineering_rules - - entries = { - "codex": workspace / "AGENTS.md", - "claude": workspace / "CLAUDE.md", - "cursor": workspace / ".cursor/rules/xprobe.mdc", - } - for client, path in entries.items(): - assert path.is_file(), client - assert SKILL_PATH in path.read_text(), client + assert required in setup, required def check_schemas(workspace: pathlib.Path) -> None: @@ -273,19 +198,11 @@ def check_schemas(workspace: pathlib.Path) -> None: def check_installation_docs(workspace: pathlib.Path) -> None: - cargo = (workspace / "Cargo.toml").read_text() - version_match = re.search(r'^version = "([^"]+)"$', cargo, re.MULTILINE) - assert version_match is not None - development_version = version_match.group(1) - + development = development_version(workspace) + release = release_version(workspace) installer = (workspace / "install.sh").read_text() - release_match = re.search(r"^version=\$\{XPROBE_VERSION:-([^}]+)\}$", installer, re.MULTILINE) - assert release_match is not None - release_version = release_match.group(1) - assert tuple(map(int, development_version.split("."))) >= tuple( - map(int, release_version.split(".")) - ) - assert f"default: {release_version}" in installer + assert tuple(map(int, development.split("."))) >= tuple(map(int, release.split("."))) + assert f"default: {release}" in installer for relative_path in ( "README.md", @@ -294,7 +211,7 @@ def check_installation_docs(workspace: pathlib.Path) -> None: ): document = (workspace / relative_path).read_text() assert "npx skills@1 add" in document, relative_path - assert f"/v{release_version}/" in document, relative_path + assert f"/v{release}/" in document, relative_path def main() -> None: @@ -361,7 +278,7 @@ def main() -> None: for warning in validated["warnings"] ) - check_skill(workspace) + check_skill(workspace, binary) check_schemas(workspace) check_installation_docs(workspace) print( diff --git a/tests/agent-contract/test_workflow_routes.py b/tests/agent-contract/test_workflow_routes.py index 3c9cf36..48efbc1 100644 --- a/tests/agent-contract/test_workflow_routes.py +++ b/tests/agent-contract/test_workflow_routes.py @@ -13,10 +13,13 @@ def main() -> None: skill_root = workspace / "skills/xprobe-measure-latency" sources = { "skill": normalize((skill_root / "SKILL.md").read_text()), - "investigation": normalize( - (skill_root / "references/investigation.md").read_text() - ), "setup": normalize((skill_root / "references/setup.md").read_text()), + "selectors": normalize((skill_root / "references/selectors.md").read_text()), + "interpreting": normalize((skill_root / "references/interpreting.md").read_text()), + "deep_dive": normalize((skill_root / "references/deep-dive.md").read_text()), + "multi_process": normalize( + (skill_root / "references/multi-process.md").read_text() + ), } fixture = json.loads( (workspace / "tests/agent-contract/fixtures/workflow-routes.json").read_text() @@ -29,11 +32,12 @@ def main() -> None: covered.add(scenario["name"]) assert covered == { "existing_artifact", + "unknown_workload", "known_selector", - "unknown_cpu", + "several_boundaries", + "dig_deeper", "python_semantics", - "mixed_cpu_gpu", - "unsupported_python_runtime", + "multi_process", "container_namespace", } print(json.dumps({"schema_version": "2.0", "ok": True, "routes": sorted(covered)})) From 1e0f8f7a42154869791a6e5277674ff280727532 Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 14 Nov 2026 14:56:00 +0800 Subject: [PATCH 13/15] =?UTF-8?q?=E2=9C=A8=20feat:=20split=20memcpy=20aggr?= =?UTF-8?q?egates=20by=20host=20memory=20kind?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copies from pageable host memory are staged and synchronous, so they are a common hidden transfer cost, but aggregate inventories could not tell them from pinned copies. The Agent now keys memcpy aggregate slots by CUPTI source and destination memory kinds and reports them in the previously reserved record fields, behind a new feature flag so older Agents decode as unknown. Aggregate results move to schema 3.0 with `src_memory` and `dst_memory` per group. The PyTorch live test confirms pinned host buffers are reported as pinned for both copy directions. Triage still covers kernels only and says so in `not_covered`. --- benchmarks/cuda-multiprocess/run-container.pl | 2 +- cupti/include/xprobe/cupti_agent.h | 9 ++- cupti/src/cupti_agent.c | 33 +++++++-- docs/cli-contract.md | 3 + .../aggregate-inventory-result.schema.json | 42 ++++++++++- schemas/triage-result.schema.json | 35 +++++++++ .../references/deep-dive.md | 3 +- .../references/interpreting.md | 4 +- tests/integration/test_pytorch_cuda.py | 7 ++ xprobe/cli/src/live.rs | 2 + xprobe/cli/src/triage.rs | 5 +- xprobe/collector/src/cupti.rs | 73 +++++++++++++++++-- xprobe/core/src/gpu_aggregate.rs | 44 ++++++++++- xprobe/core/src/triage.rs | 2 + xprobe/protocol/src/event.rs | 18 +++++ xprobe/protocol/src/lib.rs | 2 +- xprobe/protocol/src/measurement.rs | 6 +- xprobe/protocol/tests/contracts.rs | 4 +- 18 files changed, 263 insertions(+), 31 deletions(-) diff --git a/benchmarks/cuda-multiprocess/run-container.pl b/benchmarks/cuda-multiprocess/run-container.pl index 94d363c..5a661b2 100755 --- a/benchmarks/cuda-multiprocess/run-container.pl +++ b/benchmarks/cuda-multiprocess/run-container.pl @@ -660,7 +660,7 @@ sub run_batch_body { require_condition($inventory_exit == 0, 'representative aggregate inventory command failed'); verify_identity($representative, 'after aggregate inventory'); - my $inventory = load_json($inventory_stdout, 'aggregate inventory'); + my $inventory = load_json($inventory_stdout, 'aggregate inventory', '3.0'); require_condition($inventory->{ok} && $inventory->{status} eq 'completed', 'aggregate inventory is not complete'); my $inventory_collection = $inventory->{collection}; diff --git a/cupti/include/xprobe/cupti_agent.h b/cupti/include/xprobe/cupti_agent.h index 0a50a89..1ceac41 100644 --- a/cupti/include/xprobe/cupti_agent.h +++ b/cupti/include/xprobe/cupti_agent.h @@ -19,7 +19,8 @@ enum xprobe_cupti_feature { XPROBE_CUPTI_FEATURE_HOST_MONOTONIC_TIMESTAMPS = 1U << 0, XPROBE_CUPTI_FEATURE_TRANSFER_RECORDS = 1U << 1, XPROBE_CUPTI_FEATURE_AGGREGATE_RECORDS = 1U << 2, - XPROBE_CUPTI_FEATURE_NVTX_RECORDS = 1U << 3 + XPROBE_CUPTI_FEATURE_NVTX_RECORDS = 1U << 3, + XPROBE_CUPTI_FEATURE_MEMORY_KINDS = 1U << 4 }; enum xprobe_cupti_capture_mode { @@ -145,7 +146,11 @@ struct xprobe_cupti_aggregate_record { uint32_t kind; uint32_t device_id; uint32_t memcpy_kind; - uint32_t reserved[5]; + /* CUpti_ActivityMemoryKind of a memcpy's source and destination; 0 for + * other activity. Valid when XPROBE_CUPTI_FEATURE_MEMORY_KINDS is set. */ + uint32_t src_memory_kind; + uint32_t dst_memory_kind; + uint32_t reserved[3]; char name[XPROBE_CUPTI_NAME_LENGTH]; }; diff --git a/cupti/src/cupti_agent.c b/cupti/src/cupti_agent.c index 540bf1f..c514834 100644 --- a/cupti/src/cupti_agent.c +++ b/cupti/src/cupti_agent.c @@ -108,6 +108,8 @@ struct xprobe_cupti_aggregate_slot { uint32_t kind; uint32_t device_id; uint32_t memcpy_kind; + uint32_t src_memory_kind; + uint32_t dst_memory_kind; char name[XPROBE_CUPTI_NAME_LENGTH]; }; @@ -648,12 +650,15 @@ static uint64_t aggregate_hash(uint32_t kind, uint32_t device_id, static int aggregate_key_matches( const struct xprobe_cupti_aggregate_slot *slot, uint32_t kind, - uint32_t device_id, uint32_t memcpy_kind, const char *name) + uint32_t device_id, uint32_t memcpy_kind, uint32_t src_memory_kind, + uint32_t dst_memory_kind, const char *name) { const char *key_name = name == NULL ? "" : name; return slot->kind == kind && slot->device_id == device_id && slot->memcpy_kind == memcpy_kind && + slot->src_memory_kind == src_memory_kind && + slot->dst_memory_kind == dst_memory_kind && strncmp(slot->name, key_name, XPROBE_CUPTI_NAME_LENGTH - 1U) == 0; } @@ -704,7 +709,8 @@ static void update_aggregate_slot(struct xprobe_cupti_aggregate_slot *slot, } static void aggregate_activity(uint32_t kind, uint32_t device_id, - uint32_t memcpy_kind, const char *name, + uint32_t memcpy_kind, uint32_t src_memory_kind, + uint32_t dst_memory_kind, const char *name, uint64_t start_ns, uint64_t end_ns, uint64_t bytes) { uint64_t hash; @@ -718,7 +724,10 @@ static void aggregate_activity(uint32_t kind, uint32_t device_id, remember_output_error(); return; } - hash = aggregate_hash(kind, device_id, memcpy_kind, name); + hash = aggregate_hash(kind, device_id, + memcpy_kind | (src_memory_kind << 8U) | + (dst_memory_kind << 16U), + name); first = hash % record_capacity; for (uint64_t probe = 0U; probe < record_capacity; ++probe) { uint64_t index = (first + probe) % record_capacity; @@ -726,7 +735,8 @@ static void aggregate_activity(uint32_t kind, uint32_t device_id, uint64_t state = atomic_load_explicit(&slot->state, memory_order_acquire); if (state == hash && - aggregate_key_matches(slot, kind, device_id, memcpy_kind, name) != 0) { + aggregate_key_matches(slot, kind, device_id, memcpy_kind, + src_memory_kind, dst_memory_kind, name) != 0) { update_aggregate_slot(slot, end_ns - start_ns, bytes); return; } @@ -740,6 +750,8 @@ static void aggregate_activity(uint32_t kind, uint32_t device_id, slot->kind = kind; slot->device_id = device_id; slot->memcpy_kind = memcpy_kind; + slot->src_memory_kind = src_memory_kind; + slot->dst_memory_kind = dst_memory_kind; copy_name(slot->name, name); atomic_store_explicit(&slot->count, 0U, memory_order_relaxed); atomic_store_explicit(&slot->total_duration_ns, 0U, @@ -1191,7 +1203,7 @@ static int activity_started_during_capture(uint64_t timestamp_ns) } \ if (capture_mode == XPROBE_CUPTI_CAPTURE_AGGREGATE) { \ aggregate_activity( \ - XPROBE_CUPTI_GPU_KERNEL_START, (kernel)->deviceId, 0U, \ + XPROBE_CUPTI_GPU_KERNEL_START, (kernel)->deviceId, 0U, 0U, 0U, \ (kernel)->name, normalize_activity_timestamp((kernel)->start), \ normalize_activity_timestamp((kernel)->end), 0U); \ return; \ @@ -1314,7 +1326,9 @@ static void CUPTIAPI activity_buffer_completed(CUcontext context, uint32_t strea if (capture_mode == XPROBE_CUPTI_CAPTURE_AGGREGATE) { aggregate_activity( XPROBE_CUPTI_GPU_MEMCPY_START, memcpy_record->deviceId, - semantic_memcpy_kind((uint32_t)memcpy_record->copyKind), NULL, + semantic_memcpy_kind((uint32_t)memcpy_record->copyKind), + (uint32_t)memcpy_record->srcKind, + (uint32_t)memcpy_record->dstKind, NULL, normalize_activity_timestamp(memcpy_record->start), normalize_activity_timestamp(memcpy_record->end), memcpy_record->bytes); @@ -1338,7 +1352,7 @@ static void CUPTIAPI activity_buffer_completed(CUcontext context, uint32_t strea if (capture_mode == XPROBE_CUPTI_CAPTURE_AGGREGATE) { aggregate_activity( XPROBE_CUPTI_GPU_MEMSET_START, memset_record->deviceId, 0U, - NULL, normalize_activity_timestamp(memset_record->start), + 0U, 0U, NULL, normalize_activity_timestamp(memset_record->start), normalize_activity_timestamp(memset_record->end), memset_record->bytes); continue; @@ -1579,7 +1593,8 @@ static void initialize_output_header(struct xprobe_cupti_output_header *header, header->feature_flags |= XPROBE_CUPTI_FEATURE_NVTX_RECORDS; } if (capture_mode == XPROBE_CUPTI_CAPTURE_AGGREGATE) { - header->feature_flags |= XPROBE_CUPTI_FEATURE_AGGREGATE_RECORDS; + header->feature_flags |= XPROBE_CUPTI_FEATURE_AGGREGATE_RECORDS | + XPROBE_CUPTI_FEATURE_MEMORY_KINDS; } else if (activity_timestamps_are_host_monotonic(available) != 0) { header->feature_flags |= XPROBE_CUPTI_FEATURE_HOST_MONOTONIC_TIMESTAMPS; } @@ -1623,6 +1638,8 @@ static int write_aggregate_records(int descriptor, output.kind = slot->kind; output.device_id = slot->device_id; output.memcpy_kind = slot->memcpy_kind; + output.src_memory_kind = slot->src_memory_kind; + output.dst_memory_kind = slot->dst_memory_kind; memcpy(output.name, slot->name, sizeof(output.name)); if (write_function(descriptor, &output, sizeof(output)) != 0) { return -1; diff --git a/docs/cli-contract.md b/docs/cli-contract.md index f4acfbd..228881a 100644 --- a/docs/cli-contract.md +++ b/docs/cli-contract.md @@ -301,6 +301,9 @@ memcpy, or memset activity start/end pair. It uses `--max-groups` (default `schemas/aggregate-inventory-result.schema.json`. The result is a coarse inventory, not exact event evidence: it reports count, total/min/max/mean duration, optional transferred bytes, table occupancy, and drop completeness. +Memcpy groups also split by `src_memory` and `dst_memory` (`pageable`, +`pinned`, `device`, `managed`, ...); pageable host memory forces staged, +synchronous copies. Aggregate results use schema `3.0`. Aggregate kernel regex must be reducible to an exact, prefix, suffix, or contains filter because this mode intentionally has no Rust-side event pass. Each kernel group reports `name_complete`. CUPTI names that fill the fixed diff --git a/schemas/aggregate-inventory-result.schema.json b/schemas/aggregate-inventory-result.schema.json index ea4021b..a7023b1 100644 --- a/schemas/aggregate-inventory-result.schema.json +++ b/schemas/aggregate-inventory-result.schema.json @@ -13,7 +13,7 @@ "type": "boolean" }, "schema_version": { - "$ref": "#/$defs/SchemaVersion" + "$ref": "#/$defs/SchemaVersion3" }, "session_id": { "type": "string" @@ -150,6 +150,16 @@ "format": "uint32", "minimum": 0 }, + "dst_memory": { + "anyOf": [ + { + "$ref": "#/$defs/MemoryKind" + }, + { + "type": "null" + } + ] + }, "duration_ns": { "$ref": "#/$defs/AggregateDuration" }, @@ -178,6 +188,17 @@ "null" ] }, + "src_memory": { + "description": "Source and destination memory of memcpy groups; `null` for other\nactivity or when the Agent did not report memory kinds.", + "anyOf": [ + { + "$ref": "#/$defs/MemoryKind" + }, + { + "type": "null" + } + ] + }, "start_selector_hint": { "type": "string" }, @@ -245,10 +266,25 @@ "unknown" ] }, - "SchemaVersion": { + "MemoryKind": { + "description": "Host or device memory behind one side of a copy (`CUpti_ActivityMemoryKind`).\nPageable host memory forces staged, synchronous transfers; pinned memory\nallows direct DMA.", + "type": "string", + "enum": [ + "unknown", + "pageable", + "pinned", + "device", + "array", + "managed", + "device_static", + "managed_static" + ] + }, + "SchemaVersion3": { + "description": "Version of records whose shape changed in xprobe 0.6.\n\nRecords keep their own version so unchanged contracts, such as Event JSONL\nartifacts, remain readable at `2.0`.", "type": "string", "enum": [ - "2.0" + "3.0" ] }, "SessionStatus": { diff --git a/schemas/triage-result.schema.json b/schemas/triage-result.schema.json index ab78fc7..c0b0a1d 100644 --- a/schemas/triage-result.schema.json +++ b/schemas/triage-result.schema.json @@ -165,6 +165,16 @@ "format": "uint32", "minimum": 0 }, + "dst_memory": { + "anyOf": [ + { + "$ref": "#/$defs/MemoryKind" + }, + { + "type": "null" + } + ] + }, "duration_ns": { "$ref": "#/$defs/AggregateDuration" }, @@ -193,6 +203,17 @@ "null" ] }, + "src_memory": { + "description": "Source and destination memory of memcpy groups; `null` for other\nactivity or when the Agent did not report memory kinds.", + "anyOf": [ + { + "$ref": "#/$defs/MemoryKind" + }, + { + "type": "null" + } + ] + }, "start_selector_hint": { "type": "string" }, @@ -570,6 +591,20 @@ "unknown" ] }, + "MemoryKind": { + "description": "Host or device memory behind one side of a copy (`CUpti_ActivityMemoryKind`).\nPageable host memory forces staged, synchronous transfers; pinned memory\nallows direct DMA.", + "type": "string", + "enum": [ + "unknown", + "pageable", + "pinned", + "device", + "array", + "managed", + "device_static", + "managed_static" + ] + }, "PcieLink": { "description": "Narrowest `PCIe` link observed against the link maximum.", "type": "object", diff --git a/skills/xprobe-measure-latency/references/deep-dive.md b/skills/xprobe-measure-latency/references/deep-dive.md index dd03c4b..65ae252 100644 --- a/skills/xprobe-measure-latency/references/deep-dive.md +++ b/skills/xprobe-measure-latency/references/deep-dive.md @@ -14,7 +14,8 @@ command, and when to stop. Escalate one step at a time. intermittent or per-request, drive one slow request during the window or lengthen `--duration-ms` to cover several cycles. 3. **Triage omits transfers.** GPU activity in triage covers kernels only. - Inventory copies and memsets explicitly: + Inventory copies and memsets explicitly; groups with `src_memory` or + `dst_memory` of `pageable` point at host buffers to pin: ```bash xprobe measure --pid "$PID" --from cuda:memcpy_start --to cuda:memcpy_end \ diff --git a/skills/xprobe-measure-latency/references/interpreting.md b/skills/xprobe-measure-latency/references/interpreting.md index 51d25a2..dc1061c 100644 --- a/skills/xprobe-measure-latency/references/interpreting.md +++ b/skills/xprobe-measure-latency/references/interpreting.md @@ -52,7 +52,9 @@ do not compare their capacities or completeness as one collector. - Syscall aggregation: durations are exact for matched lifecycles, but groups have no order or percentiles; check unmatched, inflight, and drops. - GPU aggregation: count, summed, min, max, and mean duration and bytes per - group; no ordering, overlap, or correlation. + group; no ordering, overlap, or correlation. Memcpy groups split by + `src_memory` and `dst_memory`: a `pageable` host side means staged, + synchronous copies that pinned memory would avoid. ## GPU concurrency diff --git a/tests/integration/test_pytorch_cuda.py b/tests/integration/test_pytorch_cuda.py index ff12125..240a940 100644 --- a/tests/integration/test_pytorch_cuda.py +++ b/tests/integration/test_pytorch_cuda.py @@ -147,6 +147,13 @@ def run_inner(xprobe: pathlib.Path) -> None: } assert "cuda:memcpy_start:kind=HtoD" in transfer_hints assert "cuda:memcpy_start:kind=DtoH" in transfer_hints + host_sides = { + (group["memcpy_kind"], group["src_memory"], group["dst_memory"]) + for group in transfers["inventory"]["groups"] + } + # The workload copies between pinned host buffers and device memory. + assert ("HtoD", "pinned", "device") in host_sides, host_sides + assert ("DtoH", "device", "pinned") in host_sides, host_sides assert_exact(exact_kernel) assert_exact(synchronization) assert_exact(nvtx) diff --git a/xprobe/cli/src/live.rs b/xprobe/cli/src/live.rs index 8e6d587..c14726a 100644 --- a/xprobe/cli/src/live.rs +++ b/xprobe/cli/src/live.rs @@ -1167,6 +1167,8 @@ pub(crate) fn aggregate_inventory_result( name_complete: group.name_complete, device_id: group.device_id, memcpy_kind: group.memcpy_kind.clone(), + src_memory: group.src_memory, + dst_memory: group.dst_memory, count: group.count, total_duration_ns: group.total_duration_ns, min_duration_ns: group.min_duration_ns, diff --git a/xprobe/cli/src/triage.rs b/xprobe/cli/src/triage.rs index 08f916a..f840bd5 100644 --- a/xprobe/cli/src/triage.rs +++ b/xprobe/cli/src/triage.rs @@ -42,11 +42,10 @@ const SYSCALL_MAX_INFLIGHT: u64 = 1024; const GPU_MAX_GROUPS: usize = 4096; const GPU_HEALTH_INTERVAL: Duration = Duration::from_millis(250); -const NOT_COVERED: [&str; 4] = [ +const NOT_COVERED: [&str; 3] = [ "where blocked threads wait (off-CPU stacks, lock and I/O attribution)", "exact GPU busy intervals; NVML utilization is a device-wide sampled estimate", - "memcpy and memset activity; GPU activity covers kernels only", - "pageable versus pinned host memory in transfers", + "memcpy and memset activity, including pageable versus pinned host memory; run a memcpy aggregate inventory", ]; #[derive(Debug)] diff --git a/xprobe/collector/src/cupti.rs b/xprobe/collector/src/cupti.rs index 0e05ed1..47d7955 100644 --- a/xprobe/collector/src/cupti.rs +++ b/xprobe/collector/src/cupti.rs @@ -12,7 +12,7 @@ use std::{ use serde_json::Value; use xprobe_protocol::{ - ClockDomain, CudaEvent, Dim3, Event, EventSource, EventType, MemcpyKind, NvtxEvent, + ClockDomain, CudaEvent, Dim3, Event, EventSource, EventType, MemcpyKind, MemoryKind, NvtxEvent, NvtxRangeKind, SchemaVersion, }; @@ -37,10 +37,12 @@ const FEATURE_HOST_MONOTONIC_TIMESTAMPS: u32 = 1 << 0; const FEATURE_TRANSFER_RECORDS: u32 = 1 << 1; const FEATURE_AGGREGATE_RECORDS: u32 = 1 << 2; const FEATURE_NVTX_RECORDS: u32 = 1 << 3; +const FEATURE_MEMORY_KINDS: u32 = 1 << 4; const SUPPORTED_FEATURES: u32 = FEATURE_HOST_MONOTONIC_TIMESTAMPS | FEATURE_TRANSFER_RECORDS | FEATURE_AGGREGATE_RECORDS - | FEATURE_NVTX_RECORDS; + | FEATURE_NVTX_RECORDS + | FEATURE_MEMORY_KINDS; const UNKNOWN_U32: u32 = u32::MAX; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -122,6 +124,8 @@ pub struct CuptiAggregateGroup { pub name_complete: Option, pub device_id: Option, pub memcpy_kind: Option, + pub src_memory: Option, + pub dst_memory: Option, pub count: u64, pub total_duration_ns: u64, pub min_duration_ns: u64, @@ -628,7 +632,7 @@ fn decode_payload( if feature_flags & FEATURE_AGGREGATE_RECORDS != 0 { let mut groups = Vec::with_capacity(record_count); for (index, record) in bytes[HEADER_SIZE..].chunks_exact(RECORD_SIZE).enumerate() { - groups.push(decode_aggregate_record(record, index)?); + groups.push(decode_aggregate_record(record, index, feature_flags)?); } return Ok((Vec::new(), groups)); } @@ -652,6 +656,7 @@ fn decode_payload( fn decode_aggregate_record( record: &[u8], index: usize, + feature_flags: u32, ) -> Result { let count = read_u64(record, 0); let total_duration_ns = read_u64(record, 8); @@ -699,12 +704,27 @@ fn decode_aggregate_record( } let memcpy_kind = (activity == CuptiAggregateActivity::Memcpy) .then(|| decode_semantic_memcpy_kind(read_u32(record, 48))); + let memory_kinds = + activity == CuptiAggregateActivity::Memcpy && feature_flags & FEATURE_MEMORY_KINDS != 0; + let memory_kind = |offset: usize| { + memory_kinds + .then(|| decode_memory_kind(read_u32(record, offset))) + .transpose() + .map_err(|()| CuptiDecodeError::InvalidAggregateRecord { + index, + message: "memory kind is unsupported", + }) + }; + let src_memory = memory_kind(52)?; + let dst_memory = memory_kind(56)?; Ok(CuptiAggregateGroup { activity, name: has_name.then(|| name.to_owned()), name_complete: has_name.then_some(name_length + 1 < CUPTI_NAME_CAPACITY), device_id: optional_unknown(read_u32(record, 44)), memcpy_kind, + src_memory, + dst_memory, count, total_duration_ns, min_duration_ns, @@ -713,6 +733,20 @@ fn decode_aggregate_record( }) } +fn decode_memory_kind(value: u32) -> Result { + Ok(match value { + 0 => MemoryKind::Unknown, + 1 => MemoryKind::Pageable, + 2 => MemoryKind::Pinned, + 3 => MemoryKind::Device, + 4 => MemoryKind::Array, + 5 => MemoryKind::Managed, + 6 => MemoryKind::DeviceStatic, + 7 => MemoryKind::ManagedStatic, + _ => return Err(()), + }) +} + fn decode_capture_state(value: u32) -> Result { match value { 0 => Ok(CuptiCaptureState::Idle), @@ -982,7 +1016,9 @@ mod tests { FILTER_SIZE, HEADER_SIZE, OUTPUT_MAGIC, RECORD_SIZE, decode_capture, encode_control, read_snapshot, }; - use xprobe_protocol::{ClockDomain, EventSource, EventType, MemcpyKind, NvtxRangeKind}; + use xprobe_protocol::{ + ClockDomain, EventSource, EventType, MemcpyKind, MemoryKind, NvtxRangeKind, + }; fn capture(records: &[[u8; RECORD_SIZE]]) -> Vec { let mut bytes = vec![0_u8; HEADER_SIZE + records.len() * RECORD_SIZE]; @@ -1271,10 +1307,10 @@ mod tests { #[test] fn rejects_unknown_feature_flags() { let mut bytes = capture(&[]); - bytes[20..24].copy_from_slice(&16_u32.to_le_bytes()); + bytes[20..24].copy_from_slice(&32_u32.to_le_bytes()); assert_eq!( decode_capture(&bytes, "xp_test"), - Err(CuptiDecodeError::UnsupportedFeatureFlags(16)) + Err(CuptiDecodeError::UnsupportedFeatureFlags(32)) ); } @@ -1372,6 +1408,31 @@ mod tests { assert_eq!(decoded.aggregate_groups[1].total_bytes, Some(4096)); } + #[test] + fn decodes_memcpy_memory_kinds_only_when_reported() { + let mut pageable = aggregate_record(5, 2, 200, 0, 1, ""); + pageable[52..56].copy_from_slice(&1_u32.to_le_bytes()); + pageable[56..60].copy_from_slice(&3_u32.to_le_bytes()); + let mut bytes = aggregate_capture(&[pageable], 2); + let features = super::FEATURE_TRANSFER_RECORDS + | super::FEATURE_AGGREGATE_RECORDS + | super::FEATURE_MEMORY_KINDS; + bytes[20..24].copy_from_slice(&features.to_le_bytes()); + let group = &decode_capture(&bytes, "unused").unwrap().aggregate_groups[0]; + assert_eq!(group.src_memory, Some(MemoryKind::Pageable)); + assert_eq!(group.dst_memory, Some(MemoryKind::Device)); + + let older = aggregate_capture(&[pageable], 2); + let group = &decode_capture(&older, "unused").unwrap().aggregate_groups[0]; + assert_eq!((group.src_memory, group.dst_memory), (None, None)); + + let mut invalid = pageable; + invalid[52..56].copy_from_slice(&99_u32.to_le_bytes()); + let mut bytes = aggregate_capture(&[invalid], 2); + bytes[20..24].copy_from_slice(&features.to_le_bytes()); + assert!(decode_capture(&bytes, "unused").is_err()); + } + #[test] fn marks_a_full_name_buffer_as_possibly_truncated() { let name = "x".repeat(CUPTI_NAME_CAPACITY - 1); diff --git a/xprobe/core/src/gpu_aggregate.rs b/xprobe/core/src/gpu_aggregate.rs index caa40e5..a4a7613 100644 --- a/xprobe/core/src/gpu_aggregate.rs +++ b/xprobe/core/src/gpu_aggregate.rs @@ -5,7 +5,7 @@ use std::{error::Error, fmt}; use xprobe_protocol::{ AggregateActivity, AggregateCollectionSummary, AggregateDuration, AggregateGroup, AggregateInventory, AggregateInventoryResult, CaptureCompleteness, ErrorCode, MemcpyKind, - SchemaVersion, SessionStatus, + MemoryKind, SchemaVersion3, SessionStatus, }; /// One Agent aggregate table slot. Several slots may describe the same group. @@ -16,6 +16,8 @@ pub struct RawGpuAggregateGroup { pub name_complete: Option, pub device_id: Option, pub memcpy_kind: Option, + pub src_memory: Option, + pub dst_memory: Option, pub count: u64, pub total_duration_ns: u64, pub min_duration_ns: u64, @@ -115,7 +117,7 @@ pub fn analyze( .map(protocol_group) .collect::, _>>()?; Ok(AggregateInventoryResult { - schema_version: SchemaVersion::current(), + schema_version: SchemaVersion3::current(), ok: true, session_id: request.session_id, status: SessionStatus::Completed, @@ -151,6 +153,8 @@ fn merge_groups( memcpy_kind_rank(left.memcpy_kind.as_ref()) .cmp(&memcpy_kind_rank(right.memcpy_kind.as_ref())) }) + .then_with(|| left.src_memory.cmp(&right.src_memory)) + .then_with(|| left.dst_memory.cmp(&right.dst_memory)) }); let mut merged: Vec = Vec::with_capacity(groups.len()); for group in groups { @@ -162,6 +166,8 @@ fn merge_groups( || previous.name != group.name || previous.device_id != group.device_id || previous.memcpy_kind != group.memcpy_kind + || previous.src_memory != group.src_memory + || previous.dst_memory != group.dst_memory { merged.push(group); continue; @@ -220,6 +226,8 @@ fn protocol_group(group: RawGpuAggregateGroup) -> Result f64 { #[cfg(test)] mod tests { - use xprobe_protocol::{AggregateActivity, MemcpyKind}; + use xprobe_protocol::{AggregateActivity, MemcpyKind, MemoryKind}; use super::{ GpuAggregateCapture, GpuAggregateError, GpuAggregateRequest, RawGpuAggregateGroup, analyze, @@ -310,6 +318,8 @@ mod tests { name_complete: Some(complete), device_id: Some(0), memcpy_kind: None, + src_memory: None, + dst_memory: None, count, total_duration_ns: total, min_duration_ns: total / count, @@ -325,6 +335,8 @@ mod tests { name_complete: None, device_id: Some(0), memcpy_kind: Some(kind), + src_memory: None, + dst_memory: None, count, total_duration_ns: 10 * count, min_duration_ns: 10, @@ -370,6 +382,32 @@ mod tests { assert!((result.collection.table_utilization - 0.5).abs() < f64::EPSILON); } + #[test] + fn keeps_pageable_and_pinned_copies_apart() { + let mut pageable = memcpy(MemcpyKind::HostToDevice, 2, Some(64)); + pageable.src_memory = Some(MemoryKind::Pageable); + pageable.dst_memory = Some(MemoryKind::Device); + let mut pinned = pageable.clone(); + pinned.src_memory = Some(MemoryKind::Pinned); + let capture = GpuAggregateCapture { + groups: vec![pinned, pageable.clone(), pageable], + observed_activities: 6, + group_capacity: 8, + }; + let groups = analyze(request(), capture).unwrap().inventory.groups; + let kinds = groups + .iter() + .map(|group| (group.src_memory, group.count)) + .collect::>(); + assert_eq!( + kinds, + [ + (Some(MemoryKind::Pageable), 4), + (Some(MemoryKind::Pinned), 2) + ] + ); + } + #[test] fn rejects_inconsistent_counters() { let capture = GpuAggregateCapture { diff --git a/xprobe/core/src/triage.rs b/xprobe/core/src/triage.rs index 5c7862f..6d5c64b 100644 --- a/xprobe/core/src/triage.rs +++ b/xprobe/core/src/triage.rs @@ -602,6 +602,8 @@ mod tests { name_complete: Some(true), device_id: Some(0), memcpy_kind: None, + src_memory: None, + dst_memory: None, start_selector_hint: format!("cuda:kernel_start:name~^{name}$"), end_selector_hint: format!("cuda:kernel_end:name~^{name}$"), count: 10, diff --git a/xprobe/protocol/src/event.rs b/xprobe/protocol/src/event.rs index eee9de6..61bd3c7 100644 --- a/xprobe/protocol/src/event.rs +++ b/xprobe/protocol/src/event.rs @@ -171,3 +171,21 @@ pub enum MemcpyKind { #[serde(rename = "unknown")] Unknown, } + +/// Host or device memory behind one side of a copy (`CUpti_ActivityMemoryKind`). +/// Pageable host memory forces staged, synchronous transfers; pinned memory +/// allows direct DMA. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum MemoryKind { + Unknown, + Pageable, + Pinned, + Device, + Array, + Managed, + DeviceStatic, + ManagedStatic, +} diff --git a/xprobe/protocol/src/lib.rs b/xprobe/protocol/src/lib.rs index 98acfb6..4ec2fd7 100644 --- a/xprobe/protocol/src/lib.rs +++ b/xprobe/protocol/src/lib.rs @@ -31,7 +31,7 @@ pub use discover::{CudaProcessCandidate, DiscoveryResult}; pub use error::{ErrorCode, ErrorResponse, XprobeError}; pub use event::{ ArgumentValue, ClockDomain, CudaEvent, Dim3, Event, EventSource, EventType, HostEvent, - HostProbeKind, MemcpyKind, NvtxEvent, NvtxRangeKind, + HostProbeKind, MemcpyKind, MemoryKind, NvtxEvent, NvtxRangeKind, }; pub use export::{ExportFormat, TraceExportResult}; pub use graph::{ diff --git a/xprobe/protocol/src/measurement.rs b/xprobe/protocol/src/measurement.rs index 40113d5..e0c2d83 100644 --- a/xprobe/protocol/src/measurement.rs +++ b/xprobe/protocol/src/measurement.rs @@ -247,7 +247,7 @@ pub struct CuptiCollectionSummary { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct AggregateInventoryResult { - pub schema_version: SchemaVersion, + pub schema_version: SchemaVersion3, pub ok: bool, pub session_id: String, pub status: SessionStatus, @@ -274,6 +274,10 @@ pub struct AggregateGroup { pub name_complete: Option, pub device_id: Option, pub memcpy_kind: Option, + /// Source and destination memory of memcpy groups; `null` for other + /// activity or when the Agent did not report memory kinds. + pub src_memory: Option, + pub dst_memory: Option, pub start_selector_hint: String, pub end_selector_hint: String, pub count: u64, diff --git a/xprobe/protocol/tests/contracts.rs b/xprobe/protocol/tests/contracts.rs index d641b4e..2640543 100644 --- a/xprobe/protocol/tests/contracts.rs +++ b/xprobe/protocol/tests/contracts.rs @@ -188,7 +188,7 @@ fn measurement_spec_contract_round_trips() { #[test] fn aggregate_inventory_contract_round_trips() { assert_round_trip::(&json!({ - "schema_version": "2.0", + "schema_version": "3.0", "ok": true, "session_id": "xp_inventory", "status": "completed", @@ -202,6 +202,8 @@ fn aggregate_inventory_contract_round_trips() { "name_complete": true, "device_id": 0, "memcpy_kind": null, + "src_memory": null, + "dst_memory": null, "start_selector_hint": "cuda:kernel_start:name~^vector_add$", "end_selector_hint": "cuda:kernel_end:name~^vector_add$", "count": 10, From 79cd26c4ac897f9e4a3bcf881ca5251440653dff Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 14 Nov 2026 14:57:10 +0800 Subject: [PATCH 14/15] =?UTF-8?q?=F0=9F=93=9D=20docs:=20lead=20with=20tria?= =?UTF-8?q?ge=20and=20trim=20duplicated=20flags?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit README now starts from triage, a ready next spec, and graphs instead of a four-command broad-to-narrow walk. Documentation drops the `--json --non-interactive --no-color` suffix now that pipes default to JSON, and release packaging, the live gate, and public verification move from development.md into releasing.md. The contract test checks every documented xprobe command in README and docs against --help. --- README.md | 51 ++++++++-------------- docs/cli-contract.md | 34 +++++---------- docs/development.md | 50 ++------------------- docs/installation.md | 2 +- docs/releasing.md | 63 +++++++++++++++++++++++++++ tests/agent-contract/test_contract.py | 50 ++++++++++++--------- 6 files changed, 128 insertions(+), 122 deletions(-) create mode 100644 docs/releasing.md diff --git a/README.md b/README.md index 8234982..fd08a1a 100644 --- a/README.md +++ b/README.md @@ -37,47 +37,29 @@ GitHub-hosted build attestation verification. ## Measure -For an unknown CPU workload, start with one representative sampled inventory: +Triage an unknown process over one representative window. Findings are ranked +by their estimated share of the window, and each carries a ready next step: ```bash -xprobe validate --pid 4242 --cpu-sample \ - --json --non-interactive --no-color - -xprobe measure --pid 4242 \ - --cpu-sample --duration-ms 1000 \ - --json --non-interactive --no-color +xprobe measure --pid 4242 --triage --duration-ms 2000 > triage.json +jq '.findings[0].next.spec' triage.json > next.json +xprobe measure --spec next.json ``` -Use its hotspot and selector evidence, or a GPU aggregate inventory, to state a -narrow hypothesis. Then validate and measure that boundary directly: +Measure a known boundary directly, or several related boundaries from one +capture with a [measurement graph](docs/cli-contract.md#measurement-graphs): ```bash -xprobe validate --pid 4242 \ - --from 'cuda:runtime_api:cudaLaunchKernel:exit' \ - --to 'cuda:kernel_start:name~flash.*' \ - --match exact --json --non-interactive --no-color - xprobe measure --pid 4242 \ --from 'cuda:runtime_api:cudaLaunchKernel:exit' \ --to 'cuda:kernel_start:name~flash.*' \ - --match exact --samples 100 --timeout-ms 30000 \ - --events-out /tmp/xprobe-events.jsonl \ - --json --non-interactive --no-color + --match exact --samples 100 ``` -Kernel launch latency is only one event pair. xprobe can first inventory sampled -CPU/Python hotspots, syscall lifecycles, or GPU activity, then measure host -function spans, CPython GC, syscall latency, named Linux events, CUDA API calls, -GPU operation durations, transfers, NVTX ranges, and CPU/GPU paths after -selecting the correct process. - -`measure` also accepts completed `--input` captures and versioned live -`--spec` files. JSON results contain latency statistics, a quality verdict, -rank exemplar pairs, unmatched and ambiguous counts, collection completeness, -CUPTI buffer usage, clock quality, correlation confidence, and warnings. Live -captures always write the bounded event capture (`jsonl` or `chrome`, private -temporary file unless `--events-out` is set), including when correlation or -clock validation fails; `--evidence full` inlines every matched pair. +Output is JSON whenever stdout is not a terminal. Results carry latency +statistics, a `good`/`caution` quality verdict, rank exemplar pairs, and the +path of the bounded event capture, which `measure --input` re-correlates +offline. See the [CLI contract](docs/cli-contract.md) for every mode. ## Public CLI @@ -85,8 +67,8 @@ clock validation fails; `--evidence full` inlines every matched pair. | --- | --- | | `doctor` | Report local eBPF, ptrace, NVIDIA, CUDA, and CUPTI capabilities | | `discover` | List NVML-confirmed CUDA context holders under a process-tree root | -| `validate` | Check an event pair or inventory mode and report requirements without attaching | -| `measure` | Inventory bounded work or correlate bounded event evidence | +| `validate` | Check an event pair, graph, or inventory mode without attaching | +| `measure` | Triage, inventory, or correlate bounded live or recorded evidence | `measure --pid` automatically loads the matching CUDA 12 or CUDA 13 CUPTI Agent when a selected endpoint requires it. It reports the target mutation on stderr @@ -100,9 +82,11 @@ attach cannot retrofit an initialized NVTX dispatch. | Surface | Current support | | --- | --- | | OS/architecture | Linux x86_64, glibc 2.34 or newer | +| Triage | One window of thread time, cgroup throttling, CPU samples, syscalls, NVML health, and CUDA kernels | +| Measurement graphs | Up to eight edges from one capture | | CPU inventory | PID-scoped sampled user stacks with native ELF and CPython perf-map symbols | | Host events | ELF functions, named syscall/tracepoint boundaries, and CPython GC USDT | -| Coarse aggregation | Bounded syscall lifecycle and CUDA kernel/memcpy/memset groups | +| Coarse aggregation | Bounded syscall lifecycle and CUDA kernel/memcpy/memset groups, with pageable/pinned copies | | CUDA callbacks | Runtime and Driver API entry/exit | | GPU activity | Kernel, memcpy, and memset start/end | | Application ranges | Bounded ASCII NVTX thread and process ranges | @@ -122,6 +106,7 @@ offset CUPTI clock as host monotonic. - [CLI contract](docs/cli-contract.md) - [CUPTI agent and injection](docs/cupti-agent.md) - [Development and hardware tests](docs/development.md) +- [Releasing](docs/releasing.md) - [Agent integration](docs/agent-integration.md) - [Public JSON schemas](schemas/) diff --git a/docs/cli-contract.md b/docs/cli-contract.md index 228881a..cab9c7e 100644 --- a/docs/cli-contract.md +++ b/docs/cli-contract.md @@ -31,7 +31,7 @@ message, recoverability, details, and hints. ## `doctor` ```bash -xprobe doctor --json --non-interactive --no-color +xprobe doctor ``` Reports Linux/kernel identity, BTF, BPF/perf permissions, lockdown, ptrace @@ -43,8 +43,7 @@ and its resolved library path. ## `discover` ```bash -xprobe discover --pid 4242 [--limit 200] \ - --json --non-interactive --no-color +xprobe discover --pid 4242 [--limit 200] ``` Treats the PID as a process-tree root and queries NVML for active CUDA compute @@ -117,7 +116,7 @@ nonempty exact, prefix, suffix, or contains filter shorter than 128 bytes. xprobe validate --pid 4242 \ --from 'cuda:runtime_api:cudaLaunchKernel:exit' \ --to 'cuda:kernel_start:name~flash.*' \ - --match exact --json --non-interactive --no-color + --match exact ``` Validation is read-only. It verifies target identity, resolves host and Python @@ -169,8 +168,7 @@ xprobe measure --pid 4242 \ --from 'cuda:runtime_api:cudaLaunchKernel:exit' \ --to 'cuda:kernel_start:name~flash.*' \ --match exact --samples 100 --timeout-ms 30000 \ - --events-out /tmp/xprobe-events.jsonl \ - --json --non-interactive --no-color + --events-out /tmp/xprobe-events.jsonl ``` Linux syscall duration: @@ -178,8 +176,7 @@ Linux syscall duration: ```bash xprobe measure --pid 4242 \ --from 'syscall:mmap:entry' --to 'syscall:mmap:exit' \ - --match exact --samples 100 --max-events 1000 \ - --json --non-interactive --no-color + --match exact --samples 100 --max-events 1000 ``` CPU/Python hotspot inventory: @@ -187,16 +184,14 @@ CPU/Python hotspot inventory: ```bash xprobe measure --pid 4242 --cpu-sample --duration-ms 1000 \ --frequency-hz 99 --max-samples 10000 --max-groups 256 \ - --stack-depth 64 --max-threads 1024 \ - --json --non-interactive --no-color + --stack-depth 64 --max-threads 1024 ``` Syscall lifecycle inventory: ```bash xprobe measure --pid 4242 --syscall-aggregate --duration-ms 1000 \ - --max-groups 256 --max-inflight 1024 \ - --json --non-interactive --no-color + --max-groups 256 --max-inflight 1024 ``` CPython garbage-collection duration: @@ -204,8 +199,7 @@ CPython garbage-collection duration: ```bash xprobe measure --pid 4242 \ --from python:gc_start --to python:gc_end \ - --match exact --samples 100 --max-events 1000 \ - --json --non-interactive --no-color + --match exact --samples 100 --max-events 1000 ``` NVTX application range: @@ -214,8 +208,7 @@ NVTX application range: xprobe measure --pid 4242 \ --from 'cuda:nvtx_range_start:name~^forward$' \ --to 'cuda:nvtx_range_end:name~^forward$' \ - --match exact --samples 100 --max-events 1000 \ - --json --non-interactive --no-color + --match exact --samples 100 --max-events 1000 ``` Completed captures: @@ -224,15 +217,13 @@ Completed captures: xprobe measure --input host.json --input cupti.bin \ --from 'uprobe:/srv/app/server:request:entry' \ --to 'cuda:kernel_start:name~flash.*' \ - --match first-after --samples 100 \ - --json --non-interactive --no-color + --match first-after --samples 100 ``` Versioned live spec: ```bash -xprobe measure --spec measurement.json \ - --json --non-interactive --no-color +xprobe measure --spec measurement.json ``` Same-window triage of an unknown live workload: @@ -246,8 +237,7 @@ Broad GPU activity inventory: ```bash xprobe measure --pid 4242 \ --from 'cuda:kernel_start' --to 'cuda:kernel_end' \ - --match exact --aggregate --duration-ms 1000 --max-groups 4096 \ - --json --non-interactive --no-color + --match exact --aggregate --duration-ms 1000 --max-groups 4096 ``` Exactly one source mode is used: `--pid`, one or more `--input`, or `--spec`. diff --git a/docs/development.md b/docs/development.md index 8c58140..5fb927f 100644 --- a/docs/development.md +++ b/docs/development.md @@ -24,52 +24,10 @@ is tested locally with Docker and NVIDIA Container Toolkit. CPU comparison tools and a USDT-enabled CPython are installed inside the pinned benchmark container; they are not host or release archive dependencies. -## Release packaging +## Releases -A release archive requires one Agent built against each supported CUPTI major: - -```bash -CUDA_PATH=/opt/cuda-12 cmake -S . -B build/cuda12 -G Ninja \ - -DXPROBE_BUILD_BPF=OFF -DXPROBE_REQUIRE_CUPTI=ON -DXPROBE_CUDA_MAJOR=12 -cmake --build build/cuda12 --target xprobe-cupti - -CUDA_PATH=/opt/cuda-13 cmake -S . -B build/cuda13 -G Ninja \ - -DXPROBE_BUILD_BPF=OFF -DXPROBE_REQUIRE_CUPTI=ON -DXPROBE_CUDA_MAJOR=13 -cmake --build build/cuda13 --target xprobe-cupti - -scripts/package-release.sh -tests/install/test_install.sh dist/xprobe-*-linux-x86_64.tar.gz -``` - -Packaging checks both CUPTI SONAMEs, rejects build-time RPATHs, and includes the -versioned installer. It also rejects CLI or Agent ELF dependencies above -`GLIBC_2.34`, matching the supported runtime floor and preventing a release -runner change from silently raising it. The archive test also accepts a mocked -glibc 2.34 runtime, rejects 2.33, installs into a temporary prefix, runs the -packaged binary, verifies both Agents and shared resources, and uninstalls it. -After publishing the GitHub release, `scripts/verify-public-release.sh` downloads -the public archive and checksum again, repeats the installation test, and -inspects every shipped ELF. This final gate verifies the artifact users can -actually download rather than the workflow's local copy. - -GitHub Actions runs only tests and builds supported by hosted runners. Before -creating a tag, run the complete live suite on the local NVIDIA development -machine: - -```bash -mamba env create --file environment-pytorch.yml -PYTORCH_ENV="$(mamba run -n xprobe-pytorch python -c 'import sys; print(sys.prefix)')" \ - just test-release-live -``` - -`environment-pytorch.yml` pins the PyTorch fixture: Python 3.11, because -torch 2.3 Dynamo does not support 3.12, plus Triton and NumPy 1.x. - -This local gate covers BPF and perf attachment, CUDA 12 and 13 behavior, -injection, NVTX, mixed host/GPU collection, PyTorch, concurrent workers, and -the release benchmarks. The tag workflow independently rebuilds both CUPTI -Agents, packages on Ubuntu 22.04, enforces the GLIBC ceiling, and verifies the -published archive. +Release packaging, the local live gate, and public verification are described +in [Releasing](releasing.md). ## eBPF tests @@ -154,7 +112,7 @@ required: ```bash nvidia-smi -xprobe doctor --json --non-interactive --no-color +xprobe doctor ``` Record the actual GPU model, driver version, compute capability, and available diff --git a/docs/installation.md b/docs/installation.md index 8d118b5..ff96722 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -134,7 +134,7 @@ After installing the CLI, run: ```bash xprobe --version -xprobe doctor --json --non-interactive --no-color +xprobe doctor ``` Host selectors require eBPF/perf permissions. Online CUDA Agent injection diff --git a/docs/releasing.md b/docs/releasing.md new file mode 100644 index 0000000..d268f7d --- /dev/null +++ b/docs/releasing.md @@ -0,0 +1,63 @@ +# Releasing + +Work on a feature or fix branch, rebase-merge the pull request into `master`, +then tag the merged commit. The steps below gate every release. + +## 1. Prepare the version + +Bump the workspace version in `Cargo.toml` and `CMakeLists.txt`, the installer +default in `install.sh`, the Skill's supported `major.minor.x` and install +links in `skills/xprobe-measure-latency/references/setup.md`, and the versioned +links in `README.md`, `docs/installation.md`, and `docs/agent-integration.md`. +`just test-agent-contract` checks that these agree. + +## 2. Run the local live gate + +GitHub Actions runs only what hosted runners support. Before tagging, run the +complete live suite on the NVIDIA development machine: + +```bash +mamba env create --file environment-pytorch.yml +PYTORCH_ENV="$(mamba run -n xprobe-pytorch python -c 'import sys; print(sys.prefix)')" \ + just test-release-live +``` + +`environment-pytorch.yml` pins the PyTorch fixture: Python 3.11, because torch +2.3 Dynamo does not support 3.12, plus Triton and NumPy 1.x. The gate covers +BPF and perf attachment, CUDA 12 and 13, injection, NVTX, mixed host/GPU +collection, triage, PyTorch, concurrent workers, and the release benchmarks. + +## 3. Package + +A release archive requires one Agent built against each supported CUPTI major: + +```bash +CUDA_PATH=/opt/cuda-12 cmake -S . -B build/cuda12 -G Ninja \ + -DXPROBE_BUILD_BPF=OFF -DXPROBE_REQUIRE_CUPTI=ON -DXPROBE_CUDA_MAJOR=12 +cmake --build build/cuda12 --target xprobe-cupti + +CUDA_PATH=/opt/cuda-13 cmake -S . -B build/cuda13 -G Ninja \ + -DXPROBE_BUILD_BPF=OFF -DXPROBE_REQUIRE_CUPTI=ON -DXPROBE_CUDA_MAJOR=13 +cmake --build build/cuda13 --target xprobe-cupti + +scripts/package-release.sh +tests/install/test_install.sh dist/xprobe-*-linux-x86_64.tar.gz +``` + +Packaging checks both CUPTI SONAMEs, rejects build-time RPATHs, and rejects CLI +or Agent ELF dependencies above `GLIBC_2.34`. The archive test accepts a mocked +glibc 2.34 runtime, rejects 2.33, installs into a temporary prefix, runs the +binary, verifies both Agents, and uninstalls. The tag workflow independently +rebuilds both Agents, packages on Ubuntu 22.04, attests the archive and SBOM, +and publishes them. + +## 4. Verify the public release + +A release is complete only after the public artifact passes: + +```bash +scripts/verify-public-release.sh X.Y.Z +``` + +It downloads the published archive and checksum, verifies the SBOM, repeats the +installation test, and inspects every shipped ELF against the glibc ceiling. diff --git a/tests/agent-contract/test_contract.py b/tests/agent-contract/test_contract.py index d22da39..7c321dc 100755 --- a/tests/agent-contract/test_contract.py +++ b/tests/agent-contract/test_contract.py @@ -56,6 +56,30 @@ def release_version(workspace: pathlib.Path) -> str: return release_match.group(1) +def check_documented_flags(binary: pathlib.Path, documents: list[pathlib.Path]) -> set[str]: + """Require every flag in a documented `xprobe` command to exist in --help.""" + help_flags: dict[str, set[str]] = {} + for document in documents: + for block in re.findall(r"```bash\n(.*?)```", document.read_text(), re.DOTALL): + command = re.sub(r"\\\n\s*", " ", block) + for line in command.splitlines(): + words = line.split() + if len(words) < 2 or words[0] != "xprobe" or words[1].startswith("-"): + continue + subcommand = words[1] + if subcommand not in help_flags: + help_text = subprocess.run( + [binary, subcommand, "--help"], + check=True, + capture_output=True, + text=True, + ).stdout + help_flags[subcommand] = set(re.findall(r"--[a-z][a-z0-9-]*", help_text)) + for flag in re.findall(r"(? None: skill_path = workspace / SKILL_PATH skill_root = skill_path.parent.resolve() @@ -105,26 +129,8 @@ def check_skill(workspace: pathlib.Path, binary: pathlib.Path) -> None: assert required in normalized_skill, required # Every documented xprobe command uses flags its subcommand accepts. - help_flags = {} - for document in [skill_path, *references]: - for block in re.findall(r"```bash\n(.*?)```", document.read_text(), re.DOTALL): - command = re.sub(r"\\\n\s*", " ", block) - for line in command.splitlines(): - words = line.split() - if len(words) < 2 or words[0] != "xprobe" or words[1].startswith("-"): - continue - subcommand = words[1] - if subcommand not in help_flags: - help_text = subprocess.run( - [binary, subcommand, "--help"], - check=True, - capture_output=True, - text=True, - ).stdout - help_flags[subcommand] = set(re.findall(r"--[a-z][a-z0-9-]*", help_text)) - for flag in re.findall(r"(? None: check_skill(workspace, binary) check_schemas(workspace) check_installation_docs(workspace) + documented = check_documented_flags( + binary, [workspace / "README.md", *sorted((workspace / "docs").glob("*.md"))] + ) + assert {"doctor", "discover", "validate", "measure"} <= documented, sorted(documented) print( json.dumps( { From 06ec681026fb7fb28ab96a4bcb14cd5052bab968 Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 14 Nov 2026 15:02:15 +0800 Subject: [PATCH 15/15] =?UTF-8?q?=F0=9F=94=96=20chore:=20prepare=20version?= =?UTF-8?q?=200.6.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CMakeLists.txt | 2 +- Cargo.lock | 12 ++++++------ Cargo.toml | 2 +- README.md | 2 +- docs/agent-integration.md | 2 +- docs/installation.md | 12 ++++++------ install.sh | 4 ++-- skills/xprobe-measure-latency/references/setup.md | 8 ++++---- 8 files changed, 22 insertions(+), 22 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 397723b..e36d596 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.20) -project(xprobe VERSION 0.5.2 LANGUAGES C) +project(xprobe VERSION 0.6.0 LANGUAGES C) include(CTest) diff --git a/Cargo.lock b/Cargo.lock index f3a45d4..e90bab7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -732,7 +732,7 @@ dependencies = [ [[package]] name = "xprobe-cli" -version = "0.5.2" +version = "0.6.0" dependencies = [ "clap", "serde", @@ -746,7 +746,7 @@ dependencies = [ [[package]] name = "xprobe-collector" -version = "0.5.2" +version = "0.6.0" dependencies = [ "libbpf-rs", "perf-event-open", @@ -756,7 +756,7 @@ dependencies = [ [[package]] name = "xprobe-core" -version = "0.5.2" +version = "0.6.0" dependencies = [ "cpp_demangle", "nix", @@ -768,7 +768,7 @@ dependencies = [ [[package]] name = "xprobe-correlator" -version = "0.5.2" +version = "0.6.0" dependencies = [ "regex", "serde_json", @@ -777,7 +777,7 @@ dependencies = [ [[package]] name = "xprobe-exporter" -version = "0.5.2" +version = "0.6.0" dependencies = [ "serde_json", "xprobe-protocol", @@ -785,7 +785,7 @@ dependencies = [ [[package]] name = "xprobe-protocol" -version = "0.5.2" +version = "0.6.0" dependencies = [ "schemars", "serde", diff --git a/Cargo.toml b/Cargo.toml index e6fd5d7..1357027 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.5.2" +version = "0.6.0" edition = "2024" license = "Apache-2.0" repository = "https://github.com/itdevwu/xprobe" diff --git a/README.md b/README.md index fd08a1a..16f7a79 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ only installation action required from the user: ```bash npx skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.5.2/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.6.0/skills/xprobe-measure-latency \ --global ``` diff --git a/docs/agent-integration.md b/docs/agent-integration.md index 65f5537..36cfa2e 100644 --- a/docs/agent-integration.md +++ b/docs/agent-integration.md @@ -17,7 +17,7 @@ repairs the matching xprobe CLI itself: ```bash npx skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.5.2/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.6.0/skills/xprobe-measure-latency \ --global ``` diff --git a/docs/installation.md b/docs/installation.md index ff96722..a98ea9c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -13,7 +13,7 @@ user needs to run: ```bash npx skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.5.2/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.6.0/skills/xprobe-measure-latency \ --global ``` @@ -29,7 +29,7 @@ The versioned bootstrap installs to `~/.local` without root access: ```bash curl --proto '=https' --tlsv1.2 -fsSL \ - https://raw.githubusercontent.com/itdevwu/xprobe/v0.5.2/install.sh | sh + https://raw.githubusercontent.com/itdevwu/xprobe/v0.6.0/install.sh | sh ``` The bootstrap downloads the release archive and its SHA256 file, verifies the @@ -48,7 +48,7 @@ prefix, download the script and pass `--prefix`: ```bash curl --proto '=https' --tlsv1.2 -fsSLO \ - https://raw.githubusercontent.com/itdevwu/xprobe/v0.5.2/install.sh + https://raw.githubusercontent.com/itdevwu/xprobe/v0.6.0/install.sh sh install.sh --prefix /opt/xprobe ``` @@ -60,7 +60,7 @@ script with `sudo`. The installer never elevates privileges itself. For a fully explicit archive workflow: ```bash -version=0.5.2 +version=0.6.0 base=https://github.com/itdevwu/xprobe/releases/download/v$version archive=xprobe-$version-linux-x86_64.tar.gz @@ -106,7 +106,7 @@ missing or damaged, manually install the complete version-matched directory: ```bash npx skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.5.2/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.6.0/skills/xprobe-measure-latency \ --global ``` @@ -115,7 +115,7 @@ installation names the target explicitly: ```bash npx --yes skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.5.2/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.6.0/skills/xprobe-measure-latency \ --agent codex --global --copy --yes ``` diff --git a/install.sh b/install.sh index 0d6314e..d9e8ecb 100755 --- a/install.sh +++ b/install.sh @@ -2,7 +2,7 @@ set -eu repository=${XPROBE_REPOSITORY:-itdevwu/xprobe} -version=${XPROBE_VERSION:-0.5.2} +version=${XPROBE_VERSION:-0.6.0} if [ -n "${XPROBE_PREFIX:-}" ]; then prefix=$XPROBE_PREFIX elif [ -n "${HOME:-}" ]; then @@ -19,7 +19,7 @@ Install a released xprobe binary and its CUDA Agents. Usage: install.sh [--version VERSION] [--prefix DIR] [--uninstall] Options: - --version VERSION Release to install (default: 0.5.2) + --version VERSION Release to install (default: 0.6.0) --prefix DIR Installation prefix (default: $HOME/.local) --uninstall Remove xprobe from the selected prefix -h, --help Show this help diff --git a/skills/xprobe-measure-latency/references/setup.md b/skills/xprobe-measure-latency/references/setup.md index fe994d7..a2bf8cc 100644 --- a/skills/xprobe-measure-latency/references/setup.md +++ b/skills/xprobe-measure-latency/references/setup.md @@ -5,13 +5,13 @@ Offline analysis of an existing artifact needs no CLI. ## Check and bootstrap xprobe -This Skill supports xprobe `0.5.x`. Install the matching release when the CLI is +This Skill supports xprobe `0.6.x`. Install the matching release when the CLI is absent, reports another version, or fails a required capability: ```bash command -v xprobe >/dev/null 2>&1 && xprobe --version curl --proto '=https' --tlsv1.2 -fsSL \ - https://raw.githubusercontent.com/itdevwu/xprobe/v0.5.2/install.sh \ + https://raw.githubusercontent.com/itdevwu/xprobe/v0.6.0/install.sh \ -o /tmp/xprobe-install.sh sh /tmp/xprobe-install.sh export PATH="$HOME/.local/bin:$PATH" @@ -45,7 +45,7 @@ The release archive needs host glibc 2.34 or newer. Otherwise build the matching source with the host glibc: ```bash -git clone --depth 1 --branch v0.5.2 https://github.com/itdevwu/xprobe.git +git clone --depth 1 --branch v0.6.0 https://github.com/itdevwu/xprobe.git cd xprobe mamba env create --file environment.yml mamba run -n xprobe-dev just build @@ -83,6 +83,6 @@ references, examples, and analyzer are part of the contract: ```bash npx skills@1 add \ - https://github.com/itdevwu/xprobe/tree/v0.5.2/skills/xprobe-measure-latency \ + https://github.com/itdevwu/xprobe/tree/v0.6.0/skills/xprobe-measure-latency \ --global ```