diff --git a/Cargo.lock b/Cargo.lock index 76da1811..726a54f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1049,6 +1049,7 @@ dependencies = [ "aya-build", "aya-obj", "bytes", + "ghostscope-protocol", "libc", "log", "memmap2", @@ -1061,6 +1062,7 @@ dependencies = [ name = "ghostscope-protocol" version = "0.1.5" dependencies = [ + "aya", "aya-ebpf-bindings", "chrono", "ghostscope-platform", diff --git a/config-zh.toml b/config-zh.toml index 53ccb2b3..33595d93 100644 --- a/config-zh.toml +++ b/config-zh.toml @@ -34,10 +34,9 @@ log_level = "warn" [script] # 非 TUI 脚本模式的标准输出渲染方式 -# 可选值："pretty", "plain", "quiet" +# 可选值："pretty", "plain" # - pretty: 类似 TUI 的头信息，包含时间戳/PID/TID，并保留缩进后的 payload # - plain: 只输出脚本 payload 行 -# - quiet: 完全不在 stdout 打印事件输出 # 默认值："pretty" output = "pretty" @@ -241,6 +240,11 @@ mem_dump_cap = 256 # 如需在单个事件中打印较多/较大的变量，可适当增大。 max_trace_event_size = 32768 +# 每条 bt/backtrace 指令最多采集的 DWARF unwind 栈帧数。 +# 有效范围：1 到 128。 +# 默认值：128 +backtrace_depth = 128 + # 强制使用 PerfEventArray 而不是 RingBuf（仅用于测试） # 警告：仅用于测试目的。正常情况下系统会自动检测内核能力， # 并使用 RingBuf（内核 >= 5.8）或回退到 PerfEventArray。 diff --git a/config.toml b/config.toml index 0048e000..9381e506 100644 --- a/config.toml +++ b/config.toml @@ -260,6 +260,11 @@ mem_dump_cap = 256 # Increase if you plan to print many/large variables in one event. max_trace_event_size = 32768 +# Max DWARF-unwound frames captured by each bt/backtrace instruction. +# Valid range: 1 to 128. +# Default: 128 +backtrace_depth = 128 + # Force use of PerfEventArray instead of RingBuf (for testing only) # WARNING: This is for testing purposes only. Normally the system auto-detects # kernel capabilities and uses RingBuf (kernel >= 5.8) or falls back to PerfEventArray. diff --git a/docs/architecture.md b/docs/architecture.md index f4bd040d..d4419add 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -324,6 +324,14 @@ GhostScope uses an **instruction-based protocol** for flexible trace event repre | **Backtrace** | 0x10 | Stack backtrace with frame addresses | | **EndInstruction** | 0xFF | Marks end of instruction sequence | +`Backtrace` is a compact frame stream, not pre-rendered text. The compiler asks +`ghostscope-dwarf` for compact DWARF CFI rows, loads those rows into a BPF array +map, and the uprobe program records module cookies plus module-normalized PCs. +Userspace then resolves raw IPs through the process module map and asks +`ghostscope-dwarf` for function, source line, and inline-chain information. +`bt` always means DWARF unwinding; the script language intentionally does not +expose helper/fp/backend selection. + **Variable Status Tracking**: Each variable instruction includes a `status` field (u8) indicating data acquisition result: diff --git a/docs/comparison.md b/docs/comparison.md index ccea9b92..3d671301 100644 --- a/docs/comparison.md +++ b/docs/comparison.md @@ -107,7 +107,7 @@ Background: one motivation for GhostScope was that newer bpftrace versions no lo | Source line and statement probes | Supported; line-level attachment is a core path | Supported; statement probes can be resolved and attached | | Variable access (params, locals, globals) | Supported. Build PC-context read plans with gimli-backed DWARF data; render by real types; naturally ASLR and PIE friendly | Supported. DWARF location expressions are lowered through SystemTap's pipeline into eBPF-compatible logic, with verifier and stack constraints | | DWARF expression handling | Convert DWARF locations into semantic read plans and lower supported plans into eBPF runtime reads | Translate DWARF operations into internal representations and lower them into eBPF instruction sequences | -| Stack unwinding (CFI) | Not supported yet; planned via `.eh_frame` unwinding | Not supported in the eBPF backend | +| Stack unwinding (CFI) | Supported through DWARF-only `bt`/`backtrace` for compact CFI rows that can be executed safely in eBPF | Not supported in the eBPF backend | | Event transport and formatting | RingBuf (on newer kernels) or PerfEventArray; configurable pages and event size; built-in dump helpers such as `{:x.N}`, `{:s.N}`, and `{:p}` | PERF_EVENT_ARRAY plus userspace formatting/interpreter flow; formatting and string handling are more constrained | | BTF, CO-RE, linkage | Aya ecosystem, prefer RingBuf; not centered on BTF or CO-RE | No BTF or CO-RE focus; minimal libbpf-style backend | | eBPF generation pipeline | Rust and Aya loader; focused on reading userspace DWARF variables and presentation | Custom IR and assembler pipeline that emits eBPF bytecode and ELF artifacts | diff --git a/docs/configuration.md b/docs/configuration.md index b76b18c5..7d1e5b1c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -226,6 +226,7 @@ Behavior: | `--script-file ` | | Script file to execute | None | | `--script-help` | | Print the embedded script language reference and exit | Off | | `--script-output ` | | Script event stdout mode: pretty, plain | pretty | +| `--backtrace-depth ` | | Max DWARF-unwound frames captured by each `bt`/`backtrace` instruction (`1..=128`) | 128 | | `--dry-run` | | Compile the script, resolve trace targets, and exit without attaching uprobes. Requires the same eBPF privileges and kernel capabilities as a real run. | Off | | `--dry-run-details` | | Include source, inline, and variable diagnostics in dry-run output; requires `--dry-run` | Off | | `--status` | | Enable interactive DWARF/script/attach stderr status prompts | On | @@ -477,6 +478,11 @@ compare_cap = 64 # Maximum size of a single trace event (bytes). Applies to PerfEventArray accumulation buffer. max_trace_event_size = 32768 +# Max DWARF-unwound frames captured by each bt/backtrace instruction. +# Can be overridden for one run with --backtrace-depth. +# Valid range: 1 to 128. +backtrace_depth = 128 + # Recommended values: # - Simple prints: 16384 # - General use: 32768 @@ -559,6 +565,7 @@ ringbuf_size = 1048576 # 1MB buffer for high event rates mem_dump_cap = 4096 # Larger per-arg dump compare_cap = 64 # Max bytes for built-in compares (strncmp/memcmp) max_trace_event_size = 65536 # Larger event size for big formatted prints +backtrace_depth = 128 # Default full DWARF backtrace cap proc_module_offsets_max_entries = 8192 # Support many modules [general] @@ -575,6 +582,7 @@ ringbuf_size = 131072 # 128KB minimal buffer mem_dump_cap = 512 compare_cap = 32 # Smaller compare cap for minimal overhead max_trace_event_size = 16384 +backtrace_depth = 32 proc_module_offsets_max_entries = 1024 # Single process only enable_sysmon_for_target = false # Disable standalone -t lifecycle tracking @@ -674,6 +682,7 @@ GhostScope validates configuration at startup: - **ringbuf_size**: Must be power of 2, range 4096-16777216 bytes - **perf_page_count**: Must be power of 2, range 8-1024 pages - **proc_module_offsets_max_entries**: Must be in range 64-65536 + - **backtrace_depth**: Must be in range 1-128 frames - **mem_dump_cap**, **compare_cap**, and **max_trace_event_size** are runtime caps; `max_trace_event_size` may be clamped by the selected event transport. Invalid configuration will produce clear error messages with suggestions for fixes. diff --git a/docs/limitations.md b/docs/limitations.md index 1c4c4cc4..7d5ba4ff 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -45,13 +45,16 @@ Primarily tested and validated with DWARF 5 format. Theoretically supports DWARF GhostScope recognizes `DW_OP_form_tls_address`, but the runtime TLS address resolver currently handles only x86_64 executable static TLS. GhostScope resolves the current thread's TLS base at probe time, so a trace running on different pthreads reads each thread's own TLS instance for that supported executable case. The same DWARF operation is also used for dynamic/shared-library TLS; those cases require DTV/module TLS lookup and are not modeled yet, so GhostScope rejects shared-object TLS instead of guessing an address. -### 7. Highly Optimized Code Support +### 7. Stack Backtrace Coverage +`bt` uses DWARF CFI only. GhostScope does not fall back to kernel stack helpers or frame-pointer walking, and it reports an explicit stop status when CFI is unavailable, not supported by the compact eBPF fast path, or a user-stack memory read fails. Cross-module frames can be symbolized from their raw IPs when the process module map is available, but unwinding continues only while GhostScope has compact DWARF rows it can execute safely in eBPF. Deep DWARF unwinding is split through an eBPF tail-call step program so the default `backtrace_depth = 128` avoids LLVM branch-distance and verifier-size limits; `status=truncated` means the configured depth or the tail-call unwind budget was reached before a natural stop. + +### 8. Highly Optimized Code Support Compiler optimizations (-O2, -O3) can cause variables to be optimized away or generate complex DWARF expressions. GhostScope will attempt to parse them, including inline function support, but some variables may be inaccessible (shown as OptimizedOut) because the compiler optimized them away. -### 8. Dynamically Loaded Libraries (dlopen) +### 9. Dynamically Loaded Libraries (dlopen) GhostScope scans `/proc/PID/maps` at startup to obtain loaded dynamic library information. As long as GhostScope is started after `dlopen`, tracing works normally. Future plans include dynamically monitoring process `dlopen` behavior for better user experience. -### 9. Global Variables in `-t` Mode +### 10. Global Variables in `-t` Mode - **Executable targets**: When `-t` points to an executable (`-t /path/to/app`), GhostScope treats that binary as the primary module and globals are supported by default. - **Shared-library targets (existing processes)**: If GhostScope starts after the library has already been mapped (e.g., tracing a running process that loaded `libfoo.so` earlier), globals work without extra steps. @@ -60,7 +63,7 @@ GhostScope scans `/proc/PID/maps` at startup to obtain loaded dynamic library in > **Note**: The current sysmon pipeline still assumes the library is mapped when the exec event is handled; if a loader pulls it in much later, offsets are not retried yet. -### 10. `-p ` Mode inside Containers or WSL +### 11. `-p ` Mode inside Containers or WSL - See [Container Environments](container.md) for the full explanation of container / WSL scenarios, PID namespace terminology, the scenario matrix, and current implementation limits. - See [PID namespaces manual](https://www.man7.org/linux/man-pages/man7/pid_namespaces.7.html), [WSL issue #12408](https://github.com/microsoft/WSL/issues/12408), and [WSL issue #12115](https://github.com/microsoft/WSL/issues/12115) for background. diff --git a/docs/roadmap.md b/docs/roadmap.md index 7ec1e2a2..db300cca 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -25,9 +25,13 @@ GhostScope is still evolving quickly. The milestones below are ordered from “s - Keep compatibility fallbacks for kernels or libbpf/Aya paths that still require regular `uprobe` attachments. ## Stack Unwinding -- Capture full call stacks at each trace point by parsing `.eh_frame`/`.eh_frame_hdr`. -- Surface the stack in the TUI with symbol/source awareness. - Reference: +- DWARF-only `bt` / `backtrace` is now supported for compact CFI rows that can + be executed safely in eBPF, including deep stacks through a tail-call unwind + step program. +- Continue improving CFI coverage, cross-module accuracy, stop-status + diagnostics, and performance for large debug-info workloads. +- Keep the TUI and CLI renderers source-aware, with structured TUI display and + stable plain output for scripts. ## Stability & accuracy - Keep fixing defects, hardening error handling, and ensuring data consistency. diff --git a/docs/scripting.md b/docs/scripting.md index 27b6b50d..d875dfee 100644 --- a/docs/scripting.md +++ b/docs/scripting.md @@ -12,9 +12,10 @@ GhostScope uses a domain‑specific language to define trace points and actions. 7. [Expressions](#expressions) 8. [Built-in Functions](#built-in-functions) 9. [Special Variables](#special-variables) -10. [Examples](#examples) -11. [Limitations](#limitations) -12. [Runtime Expression Failures (ExprError)](#runtime-expression-failures-exprerror) +10. [Stack Backtrace](#stack-backtrace) +11. [Examples](#examples) +12. [Limitations](#limitations) +13. [Runtime Expression Failures (ExprError)](#runtime-expression-failures-exprerror) ## Basic Syntax @@ -34,7 +35,7 @@ GhostScope uses a domain‑specific language to define trace points and actions. GhostScope supports the following statements: - `trace` — define trace points and their actions - `print` — output formatted text -- `backtrace` / `bt` — reserved syntax for future stack unwinding +- `backtrace` / `bt` — print a DWARF-unwound stack backtrace - `if` / `else` — conditional execution - `let` — script variable declaration - Expression statements @@ -676,9 +677,56 @@ trace foo { } ``` -## Stack Backtrace (not implemented) +## Stack Backtrace -Backtrace printing via `backtrace;` or `bt;` is planned but not implemented yet. The syntax is reserved, and a dedicated section will be added once available. +`backtrace;` and `bt;` emit a source-aware stack backtrace at the probe point. The unwinder uses DWARF CFI directly; there is no `unwind=` option and no helper/fp fallback mode to select. + +```ghostscope +trace test_function { + print "before"; + bt; + print "after"; +} +``` + +Options: + +- `bt raw;` prints raw module cookie, module offset, and runtime IP without source symbolization. +- `bt full;` prints symbolized source-aware frames. Raw IP/cookie debug metadata is kept out of `bt full` and is only shown by `bt raw`. +- `bt inline;` enables inline call-chain rendering. This is the default. +- `bt noinline;` suppresses inline call-chain rendering. + +Backtrace depth is configured globally, not in the script. Use `--backtrace-depth ` or `[ebpf] backtrace_depth = N` in the config file. Valid range is `1..=128`; the default is `128`. +In `--script-output pretty`, backtrace payload lines are colorized when `[script] color` enables ANSI output. `--script-output plain` always emits the raw payload text without ANSI color. + +Examples: + +```ghostscope +trace test_function { + bt full; + bt raw noinline; +} +``` + +Typical output: + +```text +backtrace: complete, 4 frames (max 128) + #0 test_function(int argc, char** argv) at sample_program.c:8:5 [sample_program+0x1189] + #1 caller(int value) at sample_program.c:42:9 [sample_program+0x1234] + #2 main(int argc, char** argv) at sample_program.c:88:12 [sample_program+0x13a0] + #3 at ?? [libc.so.6+0x2a1ca] +``` + +`bt raw;` keeps the same header but prints machine-facing fields for diagnosis: + +```text +backtrace: truncated, 2 frames (max 2) + #0 0x1189 [sample_program+0x1189] raw=0x55... cookie=0x... + #1 0x1234 [sample_program+0x1234] raw=0x55... cookie=0x... +``` + +`status=complete` means DWARF unwinding reached a natural stop before the configured depth cap. `status=truncated` means GhostScope hit the configured depth cap or the eBPF tail-call unwind budget before a natural stop. Other statuses explain where unwinding stopped, for example unsupported CFI, unavailable module offsets, a failed user-memory read, or an invalid next frame. When available, `stopped:` includes a stable reason label and numeric code. ## Examples diff --git a/docs/zh/architecture.md b/docs/zh/architecture.md index de26c57d..8169e469 100644 --- a/docs/zh/architecture.md +++ b/docs/zh/architecture.md @@ -323,6 +323,12 @@ GhostScope 使用**基于指令的协议**实现灵活的追踪事件表示： | **Backtrace** | 0x10 | 带栈帧地址的栈回溯 | | **EndInstruction** | 0xFF | 标记指令序列结束 | +`Backtrace` 是紧凑的栈帧数据流，不是预先渲染好的文本。compiler 会从 +`ghostscope-dwarf` 获取 compact DWARF CFI row，并把这些 row 加载到 BPF +array map；uprobe 程序只记录 module cookie 与模块内标准化 PC。用户态再根据 +进程模块映射解析 raw IP，并交给 `ghostscope-dwarf` 查询函数、源码行号和 +inline 调用链。`bt` 始终表示 DWARF unwind，脚本语言不会暴露 helper/fp/后端选择。 + **变量状态跟踪**： 每个变量指令都包含一个 `status` 字段 (u8) 指示数据获取结果： diff --git a/docs/zh/comparison.md b/docs/zh/comparison.md index 467b6880..f3b3bf2c 100644 --- a/docs/zh/comparison.md +++ b/docs/zh/comparison.md @@ -107,7 +107,7 @@ GhostScope 的目标很明确：**针对带有 DWARF 调试信息的活跃进程 | 源码行/语句级设点 | 支持，行级附着是核心路径 | 支持，statement probe 可以解析后附着 | | 变量访问（参/局/全） | 支持。基于 gimli 读取 DWARF，并生成 PC 上下文读取计划；按真实类型渲染，天然适配 ASLR 和 PIE | 支持。DWARF 位置表达式会经过 SystemTap 的处理链路降为 eBPF 可执行逻辑，但要受验证器和栈限制 | | DWARF 表达式处理 | 将 DWARF 位置转换成语义读取计划，并把支持的计划 lower 成 eBPF 运行时读取 | 把 DWARF 操作翻译成内部表示，再继续降成 eBPF 指令序列 | -| 栈回溯（CFI） | 还不支持，计划通过 `.eh_frame` 支持 | eBPF 后端暂不支持 | +| 栈回溯（CFI） | 通过 DWARF-only `bt`/`backtrace` 支持可安全降到 eBPF 的 compact CFI row | eBPF 后端暂不支持 | | 事件传输/格式化 | 新内核优先 RingBuf，也支持 PerfEventArray；页数和事件大小可配；内置 `{:x.N}`、`{:s.N}`、`{:p}` 等 dump helper | 更偏 PERF_EVENT_ARRAY + 用户态解释/格式化流程，格式和字符串能力更受约束 | | BTF/CO-RE/链接 | Aya 生态，优先 RingBuf；不以 BTF/CO-RE 为核心 | 不以 BTF/CO-RE 为核心，更接近最小 libbpf 风格后端 | | eBPF 生成链 | Rust + Aya 装载，重点是用户态 DWARF 变量读取和展示 | 自研 IR / assembler 流水线，产出 eBPF 字节码和 ELF 产物 | diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index ce56cef9..5354899c 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -227,6 +227,7 @@ ghostscope bpffs prune --dry-run --json | `--script-file ` | | 要执行的脚本文件 | 无 | | `--script-help` | | 输出内嵌的脚本语言参考并退出 | 关 | | `--script-output ` | | 脚本事件 stdout 模式：pretty, plain | pretty | +| `--backtrace-depth ` | | 每条 `bt`/`backtrace` 指令最多采集的 DWARF unwind 栈帧数（`1..=128`） | 128 | | `--dry-run` | | 编译脚本、解析 trace 目标，然后退出，不 attach uprobe。需要与真实运行相同的 eBPF 权限和内核能力。 | 关 | | `--dry-run-details` | | 在 dry-run 输出中包含源码、inline 和变量诊断；需要同时使用 `--dry-run` | 关 | | `--status` | | 启用交互式 DWARF/脚本/attach stderr 状态提示 | 开 | @@ -472,6 +473,11 @@ compare_cap = 64 # 单条 trace 事件的最大大小（字节）。适用于 PerfEventArray 累计缓冲区。 max_trace_event_size = 32768 +# 每条 bt/backtrace 指令最多采集的 DWARF unwind 栈帧数。 +# 可通过命令行 --backtrace-depth 临时覆盖。 +# 有效范围：1 到 128。 +backtrace_depth = 128 + # 推荐值： # - 简单打印：16384 # - 通用场景：32768 @@ -552,6 +558,7 @@ ringbuf_size = 1048576 # 1MB 缓冲区用于高事件率 mem_dump_cap = 4096 # 单参数转储上限更高 compare_cap = 64 # 内置比较最大比较字节数（strncmp/memcmp） max_trace_event_size = 65536 # 大格式化输出需要更大的事件大小 +backtrace_depth = 128 # 默认完整 DWARF 回溯上限 proc_module_offsets_max_entries = 8192 # 支持更多模块 [general] @@ -568,6 +575,7 @@ ringbuf_size = 131072 # 128KB 最小缓冲区 mem_dump_cap = 512 compare_cap = 32 # 降低内置比较上限以减小开销 max_trace_event_size = 16384 +backtrace_depth = 32 proc_module_offsets_max_entries = 1024 # 仅单进程 enable_sysmon_for_target = false # 关闭独立 -t 生命周期跟踪 @@ -667,6 +675,7 @@ GhostScope 在启动时验证配置： - **ringbuf_size**：必须是 2 的幂，范围 4096-16777216 字节 - **perf_page_count**：必须是 2 的幂，范围 8-1024 页 - **proc_module_offsets_max_entries**：必须在 64-65536 范围内 + - **backtrace_depth**：必须在 1-128 栈帧范围内 - **mem_dump_cap**、**compare_cap** 和 **max_trace_event_size** 是运行时上限；`max_trace_event_size` 可能会根据实际事件传输方式被 clamp。 无效配置将产生清晰的错误消息和修复建议。 diff --git a/docs/zh/limitations.md b/docs/zh/limitations.md index 74347f85..ac186b5b 100644 --- a/docs/zh/limitations.md +++ b/docs/zh/limitations.md @@ -45,13 +45,16 @@ uprobe 和 GDB 都会修改目标进程的指令（插入断点），同时使 GhostScope 能识别 `DW_OP_form_tls_address`，但运行时 TLS 地址解析目前只支持 x86_64 可执行文件里的 static TLS。GhostScope 会在探针触发时解析当前线程的 TLS 基址，因此在这个已支持的可执行文件场景下，同一个 trace 在不同 pthread 上会读取各自线程的 TLS 实例。同一个 DWARF 操作也会用于 dynamic/shared-library TLS；这类场景需要 DTV/module TLS lookup，目前尚未建模，所以 GhostScope 会拒绝共享对象里的 TLS，而不是猜测地址。 -### 7. 高度优化代码的支持 +### 7. 栈回溯覆盖范围 +`bt` 只使用 DWARF CFI。GhostScope 不会回退到内核 stack helper 或 frame pointer walking；当 CFI 不可用、无法转换为 compact eBPF fast path，或读取用户栈内存失败时，会输出明确的停止状态。只要进程模块映射可用，跨模块栈帧可以通过 raw IP 做正确符号化；但 unwind 只有在 GhostScope 拥有能安全在 eBPF 中执行的 compact DWARF row 时才会继续。深栈 DWARF unwind 已经通过 eBPF tail-call step program 分段执行，因此默认 `backtrace_depth = 128` 不会触发 LLVM 分支距离和 verifier 程序大小限制；`status=truncated` 表示达到配置深度或 tail-call unwind 预算后仍未自然停止。 + +### 8. 高度优化代码的支持 编译器优化（-O2、-O3）会导致变量被优化掉或生成复杂的 DWARF 表达式。GhostScope 会尽力解析，包括内联函数的支持，但部分变量可能无法访问（显示为 OptimizedOut），这是因为编译器优化掉了。 -### 8. 动态加载库（dlopen） +### 9. 动态加载库（dlopen） GhostScope 启动时会扫描进程的 `/proc/PID/maps` 获取已加载的动态库信息。只要在 `dlopen` 之后启动 GhostScope，就可以正常追踪。后续计划支持动态监控进程的 `dlopen` 行为，提供更好的体验。 -### 9. -t 模式下的全局变量支持 +### 10. -t 模式下的全局变量支持 - **可执行目标**：当 `-t` 指向可执行文件（`-t /path/to/app`）时，会以该二进制作为主模块，默认支持全局变量。 - **共享库目标（已有进程）**：若 GhostScope 启动时，目标库已经被现有进程加载，例如追踪一个已运行但早先加载好 `libfoo.so` 的进程，能够直接解析全局变量。 @@ -62,7 +65,7 @@ GhostScope 启动时会扫描进程的 `/proc/PID/maps` 获取已加载的动态 > **说明**：目前 sysmon 假设共享库在 exec 事件处理时已经映射；若动态加载发生得更晚，目前不会自动重试。 -### 10. 容器 / WSL 场景下 `-p ` 模式的软限制 +### 11. 容器 / WSL 场景下 `-p ` 模式的软限制 - 容器 / WSL 场景、PID namespace 术语、场景矩阵，以及当前实现限制的完整说明，见 [容器环境](container.md)。 - 参考 [PID namespaces 手册](https://www.man7.org/linux/man-pages/man7/pid_namespaces.7.html)、[WSL issue #12408](https://github.com/microsoft/WSL/issues/12408) 和 [WSL issue #12115](https://github.com/microsoft/WSL/issues/12115)。 diff --git a/docs/zh/roadmap.md b/docs/zh/roadmap.md index 033f2066..893d283a 100644 --- a/docs/zh/roadmap.md +++ b/docs/zh/roadmap.md @@ -25,9 +25,12 @@ GhostScope 仍处在快速演进阶段，以下里程碑按照“优先修补基 - 对暂时只能走普通 `uprobe` 的内核或 libbpf/Aya 路径保留兼容性回退。 ## 栈回溯（Stack Unwinding） -- 在每个追踪点捕获完整调用栈，基于 `.eh_frame`/`.eh_frame_hdr` 信息做好解析。 -- 结合符号/源信息，TUI 中提供直观的栈帧浏览。 - 参考资料：[Unwinding the stack the hard way](https://lesenechal.fr/en/linux/unwinding-the-stack-the-hard-way#h5.1-parsing-eh_frame-and-eh_frame_hdr-with-gimli) +- 已支持 DWARF-only `bt` / `backtrace`，用于可安全降到 eBPF 执行的 + compact CFI row；深栈通过 tail-call unwind step program 分段完成。 +- 后续继续增强 CFI 覆盖、跨模块准确性、停止状态诊断，以及大体量 + debug info 场景下的性能。 +- 保持 TUI 和 CLI 的源码感知展示能力：TUI 使用结构化展示，脚本 + `plain` 输出保持稳定文本。 ## 稳定性与准确性 - 作为调试工具，持续修复缺陷、改进错误处理，确保数据一致性。 diff --git a/docs/zh/scripting.md b/docs/zh/scripting.md index 722580fa..f21fe992 100644 --- a/docs/zh/scripting.md +++ b/docs/zh/scripting.md @@ -12,9 +12,10 @@ GhostScope 使用专门的领域特定语言来定义追踪点和操作。脚本 7. [表达式](#表达式) 8. [内置函数](#内置函数) 9. [特殊变量](#特殊变量) -10. [示例](#示例) -11. [限制](#限制) -12. [运行时表达式失败（ExprError）](#运行时表达式失败exprerror) +10. [栈回溯语句](#栈回溯语句) +11. [示例](#示例) +12. [限制](#限制) +13. [运行时表达式失败（ExprError）](#运行时表达式失败exprerror) ## 基础语法 @@ -34,7 +35,7 @@ GhostScope 使用专门的领域特定语言来定义追踪点和操作。脚本 GhostScope 支持以下语句类型： - `trace` - 定义追踪点及其操作 - `print` - 输出格式化文本 -- `backtrace` / `bt` - 为后续栈回溯预留的语法 +- `backtrace` / `bt` - 输出基于 DWARF unwind 的栈回溯 - `if`/`else` - 条件执行 - `let` - 变量声明 - 表达式语句 @@ -705,18 +706,57 @@ trace foo { } ``` -## 栈回溯语句（还没有实现） +## 栈回溯语句 -`backtrace;` 和 `bt;` 是为后续栈回溯功能预留的语法，目前不会输出真实调用栈。 +`backtrace;` 和 `bt;` 会在探针触发点输出源码感知的调用栈。unwinder 直接使用 DWARF CFI；没有 `unwind=` 参数，也不暴露 helper/fp 之类的后端选择。 ```ghostscope -// 完整形式 -backtrace; +trace test_function { + print "before"; + bt; + print "after"; +} +``` + +参数： + +- `bt raw;` 输出原始 module cookie、模块内偏移和运行时 IP，不做源码符号化。 +- `bt full;` 输出符号化的源码感知栈帧。raw IP/cookie 调试元数据不会出现在 `bt full` 中，只由 `bt raw` 显示。 +- `bt inline;` 输出 inline 调用链；这是默认行为。 +- `bt noinline;` 关闭 inline 调用链输出。 + +Backtrace 深度是全局配置，不再写在脚本里。使用命令行 `--backtrace-depth `，或在配置文件 `[ebpf]` 中设置 `backtrace_depth = N`。合法范围是 `1..=128`，默认值是 `128`。 +在 `--script-output pretty` 下，如果 `[script] color` 启用了 ANSI 输出，backtrace payload 会带颜色。`--script-output plain` 始终输出不带 ANSI 的原始 payload 文本。 + +示例： + +```ghostscope +trace test_function { + bt full; + bt raw noinline; +} +``` + +典型输出： -// 简写形式 -bt; +```text +backtrace: complete, 4 frames (max 128) + #0 test_function(int argc, char** argv) at sample_program.c:8:5 [sample_program+0x1189] + #1 caller(int value) at sample_program.c:42:9 [sample_program+0x1234] + #2 main(int argc, char** argv) at sample_program.c:88:12 [sample_program+0x13a0] + #3 at ?? [libc.so.6+0x2a1ca] ``` +`bt raw;` 使用相同的 header，但会输出面向排障的机器字段： + +```text +backtrace: truncated, 2 frames (max 2) + #0 0x1189 [sample_program+0x1189] raw=0x55... cookie=0x... + #1 0x1234 [sample_program+0x1234] raw=0x55... cookie=0x... +``` + +`status=complete` 表示 DWARF unwind 在达到配置的深度上限前自然结束。`status=truncated` 表示 GhostScope 达到了配置的深度上限，或先达到了 eBPF tail-call unwind 预算。其他状态会说明 unwind 停止的原因，例如 CFI 不支持、模块偏移不可用、读取用户栈内存失败，或下一帧地址/CFA 不合法。可用时，`stopped:` 会附带稳定的原因标签和数字 code。 + ## 示例 本节集中展示常见的用法示例。 diff --git a/e2e-tests/compile-fixtures.sh b/e2e-tests/compile-fixtures.sh index 30512865..73161eef 100755 --- a/e2e-tests/compile-fixtures.sh +++ b/e2e-tests/compile-fixtures.sh @@ -67,6 +67,13 @@ run_make_fixture short_lived_long_comm_program all run_make_fixture c_multithread_program clean run_make_fixture c_multithread_program all +run_make_fixture backtrace_hot_program clean +run_make_fixture backtrace_hot_program all +run_make_fixture backtrace_hot_program backtrace_hot_program_nopie + +run_make_fixture backtrace_cross_module_program clean +run_make_fixture backtrace_cross_module_program all + run_make_fixture scalar_types_program clean run_make_fixture scalar_types_program all diff --git a/e2e-tests/tests/backtrace_execution.rs b/e2e-tests/tests/backtrace_execution.rs new file mode 100644 index 00000000..c91c8109 --- /dev/null +++ b/e2e-tests/tests/backtrace_execution.rs @@ -0,0 +1,1080 @@ +//! Backtrace execution and performance regression tests. + +mod common; + +use common::{init, FIXTURES}; +use ghostscope_dwarf::{CfaRulePlan, ModuleAddress, RegisterRecoveryPlan}; +use std::ffi::OsString; +use std::path::Path; +use std::process::Command; +use std::time::Duration; + +async fn spawn_backtrace_fixture_program( + binary_path: &Path, +) -> anyhow::Result { + let bin_dir = binary_path + .parent() + .ok_or_else(|| anyhow::anyhow!("backtrace fixture has no parent directory"))?; + let target = common::targets::TargetLauncher::binary(binary_path) + .current_dir(bin_dir) + .spawn() + .await?; + tokio::time::sleep(Duration::from_millis(250)).await; + Ok(target) +} + +async fn run_hot_backtrace_with_depth( + script: &str, + depth: u8, +) -> anyhow::Result<(usize, String, String)> { + run_hot_backtrace_with_depth_and_rate(script, depth, 250, 3).await +} + +async fn run_hot_backtrace_with_depth_and_rate( + script: &str, + depth: u8, + output_events_per_sec: u32, + timeout_secs: u64, +) -> anyhow::Result<(usize, String, String)> { + run_backtrace_fixture_with_depth_and_rate( + "backtrace_hot_program", + script, + depth, + output_events_per_sec, + timeout_secs, + ) + .await +} + +async fn run_backtrace_fixture_with_depth_and_rate( + fixture: &str, + script: &str, + depth: u8, + output_events_per_sec: u32, + timeout_secs: u64, +) -> anyhow::Result<(usize, String, String)> { + let depth_arg = depth.to_string(); + run_backtrace_fixture_with_args( + fixture, + script, + output_events_per_sec, + timeout_secs, + vec![ + OsString::from("--backtrace-depth"), + OsString::from(depth_arg), + ], + None, + ) + .await +} + +async fn run_backtrace_fixture_with_args( + fixture: &str, + script: &str, + output_events_per_sec: u32, + timeout_secs: u64, + extra_args: Vec, + config_content: Option<&str>, +) -> anyhow::Result<(usize, String, String)> { + let binary_path = FIXTURES.get_test_binary(fixture)?; + run_backtrace_binary_with_args( + &binary_path, + script, + output_events_per_sec, + timeout_secs, + extra_args, + config_content, + ) + .await +} + +async fn run_backtrace_binary_with_args( + binary_path: &Path, + script: &str, + output_events_per_sec: u32, + timeout_secs: u64, + extra_args: Vec, + config_content: Option<&str>, +) -> anyhow::Result<(usize, String, String)> { + let target = spawn_backtrace_fixture_program(binary_path).await?; + let rate_arg = output_events_per_sec.to_string(); + + let mut cli_args = vec![ + OsString::from("--script-output-events-per-sec"), + OsString::from(rate_arg), + ]; + cli_args.extend(extra_args); + + let mut runner = common::runner::GhostscopeRunner::new() + .with_script(script) + .attach_to(&target) + .timeout_secs(timeout_secs) + .enable_sysmon_for_target(false) + .with_cli_args(cli_args); + if let Some(config_content) = config_content { + runner = runner.with_config_content(config_content); + } + + let result = runner.run().await; + + target.terminate().await?; + let (exit_code, stdout, stderr) = result?; + if exit_code != 0 && stderr.contains("BPF_PROG_LOAD") { + return Ok((0, stdout, stderr)); + } + anyhow::ensure!(exit_code == 0, "stderr={stderr} stdout={stdout}"); + let count = stdout.matches("backtrace: ").count(); + Ok((count, stdout, stderr)) +} + +fn get_backtrace_hot_nopie_binary() -> anyhow::Result { + let program_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/backtrace_hot_program"); + let binary_path = program_dir.join("backtrace_hot_program_nopie"); + if binary_path.exists() { + return Ok(binary_path); + } + + let output = Command::new("make") + .arg("backtrace_hot_program_nopie") + .current_dir(&program_dir) + .output()?; + anyhow::ensure!( + output.status.success(), + "failed to build backtrace_hot_program_nopie\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + Ok(binary_path) +} + +async fn run_hot_backtrace(script: &str) -> anyhow::Result<(usize, String, String)> { + run_hot_backtrace_with_depth(script, 5).await +} + +async fn run_inline_callsite_backtrace( + script: &str, + depth: u8, +) -> anyhow::Result<(usize, String, String)> { + let binary_path = FIXTURES.get_test_binary("inline_callsite_program")?; + let depth_arg = depth.to_string(); + run_backtrace_binary_with_args( + &binary_path, + script, + 50, + 4, + vec![ + OsString::from("--backtrace-depth"), + OsString::from(depth_arg), + ], + None, + ) + .await +} + +fn first_backtrace_block_after<'a>( + stdout: &'a str, + marker: &str, + depth: u8, +) -> anyhow::Result<&'a str> { + let marker_pos = stdout + .find(marker) + .ok_or_else(|| anyhow::anyhow!("missing marker {marker:?}\nSTDOUT: {stdout}"))?; + let after_marker = &stdout[marker_pos..]; + let header = "backtrace:"; + let header_pos = after_marker + .find(header) + .ok_or_else(|| anyhow::anyhow!("missing backtrace header {header:?}\nSTDOUT: {stdout}"))?; + let block = &after_marker[header_pos..]; + let end = block.find("\n[").unwrap_or(block.len()); + let block = &block[..end]; + anyhow::ensure!( + block.contains(&format!("(max {depth})")), + "backtrace block has wrong depth, expected max {depth}\nBLOCK:\n{block}" + ); + Ok(block) +} + +fn backtrace_blocks_after(stdout: &str, marker: &str, depth: u8) -> anyhow::Result> { + let marker_pos = stdout + .find(marker) + .ok_or_else(|| anyhow::anyhow!("missing marker {marker:?}\nSTDOUT: {stdout}"))?; + let mut blocks = Vec::new(); + let mut current: Option = None; + + for line in stdout[marker_pos..].lines() { + let trimmed = line.trim_start(); + if trimmed.starts_with("backtrace:") { + if let Some(block) = current.take() { + blocks.push(block); + } + current = Some(format!("{trimmed}\n")); + continue; + } + if line.starts_with('[') { + if let Some(block) = current.take() { + blocks.push(block); + } + continue; + } + if let Some(block) = current.as_mut() { + block.push_str(trimmed); + block.push('\n'); + } + } + if let Some(block) = current { + blocks.push(block); + } + + anyhow::ensure!( + !blocks.is_empty(), + "missing backtrace block after marker {marker:?}\nSTDOUT: {stdout}" + ); + for block in &blocks { + anyhow::ensure!( + block.contains(&format!("(max {depth})")), + "backtrace block has wrong depth, expected max {depth}\nBLOCK:\n{block}" + ); + } + Ok(blocks) +} + +fn event_chunks_with_marker<'a>(stdout: &'a str, marker: &str) -> Vec<&'a str> { + stdout + .split("\n[") + .filter(|chunk| chunk.contains(marker)) + .collect() +} + +fn assert_ordered_patterns(block: &str, patterns: &[&str]) -> anyhow::Result<()> { + let mut cursor = 0usize; + for pattern in patterns { + let Some(relative) = block[cursor..].find(pattern) else { + anyhow::bail!("missing ordered pattern {pattern:?}\nBLOCK:\n{block}"); + }; + cursor += relative + pattern.len(); + } + Ok(()) +} + +fn assert_no_adjacent_duplicate_frame_locations(block: &str) -> anyhow::Result<()> { + let mut previous: Option<(String, String)> = None; + for line in block.lines() { + let trimmed = line.trim_start(); + if !trimmed.starts_with('#') { + continue; + } + let frame_id = trimmed.split_whitespace().next().unwrap_or(trimmed); + let location = match trimmed.rsplit_once('[') { + Some((_, location)) if location.ends_with(']') => location.trim_end_matches(']'), + _ => continue, + }; + let current = (trimmed.to_string(), location.to_string()); + if let Some((previous_line, previous_location)) = previous.as_ref() { + anyhow::ensure!( + previous_location != location, + "adjacent backtrace frames repeated the same module offset at {frame_id}\nPREVIOUS: {previous_line}\nCURRENT: {trimmed}\nBLOCK:\n{block}" + ); + } + previous = Some(current); + } + + Ok(()) +} + +#[tokio::test] +async fn test_hot_backtrace_compact_unwind_rows_cover_call_sites() -> anyhow::Result<()> { + init(); + + let binary_path = FIXTURES.get_test_binary("backtrace_hot_program")?; + let analyzer = ghostscope_dwarf::DwarfAnalyzer::from_exec_path(&binary_path).await?; + let leaf = analyzer + .lookup_function_address_by_name("hot_bt_leaf") + .ok_or_else(|| anyhow::anyhow!("missing hot_bt_leaf function"))?; + let lookup_pc = leaf.address + 0x19; + let ctx = analyzer.resolve_pc(&ModuleAddress::new(leaf.module_path, lookup_pc))?; + let row = analyzer + .compact_unwind_row_for_context(&ctx)? + .ok_or_else(|| anyhow::anyhow!("missing compact unwind row for hot_bt_leaf+0x19"))?; + assert!( + row.pc_start <= lookup_pc && lookup_pc < row.pc_end, + "row should cover hot_bt_leaf call-site PC 0x{lookup_pc:x}: {row:?}" + ); + assert!( + matches!( + row.cfa, + CfaRulePlan::RegPlusOffset { + register: 7, + offset: 32 + } + ), + "hot_bt_leaf call-site CFA should be rsp+32: {row:?}" + ); + assert!( + matches!( + row.return_address, + RegisterRecoveryPlan::AtCfaOffset { offset: -8 } + ), + "hot_bt_leaf return address should be at CFA-8: {row:?}" + ); + + let dummy = analyzer + .lookup_function_address_by_name("dummy_touch") + .ok_or_else(|| anyhow::anyhow!("missing dummy_touch function"))?; + let dummy_probe_pc = dummy.address + 0x10; + let ctx = analyzer.resolve_pc(&ModuleAddress::new(dummy.module_path, dummy_probe_pc))?; + let row = analyzer + .compact_unwind_row_for_context(&ctx)? + .ok_or_else(|| anyhow::anyhow!("missing compact unwind row for dummy_touch+0x10"))?; + assert!( + matches!( + row.cfa, + CfaRulePlan::RegPlusOffset { + register: 7, + offset: 240 + } + ), + "dummy_touch probe-site CFA should be rsp+240: {row:?}" + ); + assert!( + matches!( + row.return_address, + RegisterRecoveryPlan::AtCfaOffset { offset: -8 } + ), + "dummy_touch return address should be at CFA-8: {row:?}" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_special_stack_and_program_counter_registers_are_printable() -> anyhow::Result<()> { + init(); + + let binary_path = FIXTURES.get_test_binary("backtrace_hot_program")?; + let target = spawn_backtrace_fixture_program(&binary_path).await?; + let script = r#" +trace hot_bt_probe { + print "SPECIAL_SP={:p}", cast($sp, "unsigned char *"); + print "SPECIAL_PC={:p}", cast($pc, "unsigned char *"); +} +"#; + + let result = common::runner::GhostscopeRunner::new() + .with_script(script) + .attach_to(&target) + .timeout_secs(3) + .enable_sysmon_for_target(false) + .run() + .await; + + target.terminate().await?; + let (exit_code, stdout, stderr) = result?; + if exit_code != 0 && stderr.contains("BPF_PROG_LOAD") { + return Ok(()); + } + assert_eq!(exit_code, 0, "stderr={stderr} stdout={stdout}"); + assert!( + stdout.contains("SPECIAL_SP=0x"), + "expected printable stack pointer value\nSTDOUT: {stdout}\nSTDERR: {stderr}" + ); + assert!( + stdout.contains("SPECIAL_PC=0x"), + "expected printable program counter value\nSTDOUT: {stdout}\nSTDERR: {stderr}" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_hot_backtrace_full_unwinds_complete_user_stack() -> anyhow::Result<()> { + init(); + + let script = r#" +trace dummy_touch { + print "HOT_STACK"; + bt full; +} +"#; + + let (count, stdout, stderr) = run_hot_backtrace_with_depth(script, 5).await?; + if count == 0 && stderr.contains("BPF_PROG_LOAD") { + return Ok(()); + } + + let block = first_backtrace_block_after(&stdout, "HOT_STACK", 5)?; + assert!( + block.contains("backtrace: truncated, 5 frames (max 5)"), + "expected exactly the full in-binary stack prefix with intentional depth truncation\nBLOCK:\n{block}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + assert_ordered_patterns( + block, + &[ + "#0 dummy_touch", + "#1 hot_bt_leaf", + "#2 hot_bt_mid", + "#3 hot_bt_probe", + "#4 main", + ], + )?; + assert!( + !block.contains("stopped: invalid frame") + && !block.contains("stopped: read error") + && !block.contains("stopped: unsupported CFI"), + "full user stack should not stop on an unwind error\nBLOCK:\n{block}\nSTDERR:\n{stderr}" + ); + assert_no_adjacent_duplicate_frame_locations(block)?; + + Ok(()) +} + +#[tokio::test] +async fn test_hot_backtrace_defaults_to_max_depth_128() -> anyhow::Result<()> { + init(); + + let script = r#" +trace dummy_touch { + print "HOT_DEFAULT_DEPTH"; + bt full; +} +"#; + + let (count, stdout, stderr) = + run_backtrace_fixture_with_args("backtrace_hot_program", script, 250, 3, Vec::new(), None) + .await?; + if count == 0 && stderr.contains("BPF_PROG_LOAD") { + return Ok(()); + } + + let block = first_backtrace_block_after(&stdout, "HOT_DEFAULT_DEPTH", 128)?; + assert!( + block.contains("backtrace: complete") || block.contains("backtrace: truncated"), + "default-depth backtrace should render a normal status\nBLOCK:\n{block}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + assert!( + block.contains("#4 main"), + "default depth should not behave like a shallow script-local depth\nBLOCK:\n{block}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_hot_backtrace_depth_from_config_file() -> anyhow::Result<()> { + init(); + + let script = r#" +trace dummy_touch { + print "HOT_CONFIG_DEPTH"; + bt full; +} +"#; + + let (count, stdout, stderr) = run_backtrace_fixture_with_args( + "backtrace_hot_program", + script, + 250, + 3, + Vec::new(), + Some("[ebpf]\nbacktrace_depth = 4\n"), + ) + .await?; + if count == 0 && stderr.contains("BPF_PROG_LOAD") { + return Ok(()); + } + + let block = first_backtrace_block_after(&stdout, "HOT_CONFIG_DEPTH", 4)?; + assert!( + block.contains("backtrace: truncated, 4 frames (max 4)"), + "config-file depth should bound bt frames\nBLOCK:\n{block}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + assert_ordered_patterns( + block, + &[ + "#0 dummy_touch", + "#1 hot_bt_leaf", + "#2 hot_bt_mid", + "#3 hot_bt_probe", + ], + )?; + assert!( + !block.contains("#4 "), + "config-file depth=4 should not emit a fifth frame\nBLOCK:\n{block}" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_hot_backtrace_depth_one_stops_after_current_frame() -> anyhow::Result<()> { + init(); + + let script = r#" +trace dummy_touch { + print "HOT_DEPTH_ONE"; + bt full; +} +"#; + + let (count, stdout, stderr) = run_hot_backtrace_with_depth(script, 1).await?; + if count == 0 && stderr.contains("BPF_PROG_LOAD") { + return Ok(()); + } + + let block = first_backtrace_block_after(&stdout, "HOT_DEPTH_ONE", 1)?; + assert!( + block.contains("backtrace: truncated, 1 frame (max 1)"), + "depth=1 should render exactly one truncated frame\nBLOCK:\n{block}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + assert!( + block.contains("#0 dummy_touch") && !block.contains("#1 "), + "depth=1 should include only the current frame\nBLOCK:\n{block}" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_hot_backtrace_non_power_of_two_depth_keeps_frame_slots_ordered() -> anyhow::Result<()> +{ + init(); + + let script = r#" +trace dummy_touch { + print "HOT_DEPTH_SEVEN"; + bt full; +} +"#; + + let (count, stdout, stderr) = run_hot_backtrace_with_depth(script, 7).await?; + if count == 0 && stderr.contains("BPF_PROG_LOAD") { + return Ok(()); + } + + let block = first_backtrace_block_after(&stdout, "HOT_DEPTH_SEVEN", 7)?; + assert!( + block.contains("backtrace: truncated, 7 frames (max 7)"), + "depth=7 should keep the requested non-power-of-two depth\nBLOCK:\n{block}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + assert_ordered_patterns( + block, + &[ + "#0 dummy_touch", + "#1 hot_bt_leaf", + "#2 hot_bt_mid", + "#3 hot_bt_probe", + "#4 main", + "#5 ", + "#6 ", + ], + )?; + assert!( + !block.contains("#5 0x0") && !block.contains("#6 0x0"), + "dynamic frame indexes should not collapse non-power-of-two slots to zero frames\nBLOCK:\n{block}" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_multiple_deep_backtrace_statements_use_tail_calls() -> anyhow::Result<()> { + init(); + + let script = r#" +trace dummy_touch { + print "HOT_MULTI_DEEP"; + bt full; + bt raw noinline; +} +"#; + + let (count, stdout, stderr) = run_hot_backtrace_with_depth(script, 128).await?; + if count == 0 && stderr.contains("BPF_PROG_LOAD") { + return Ok(()); + } + + let blocks = backtrace_blocks_after(&stdout, "HOT_MULTI_DEEP", 128)?; + assert!( + blocks.len() >= 2, + "two bt statements should render two backtrace blocks\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + for block in blocks.iter().take(2) { + assert!( + block.contains("#5 ") && block.contains("libc.so.6+"), + "each deep bt statement should unwind past the inline frame limit\nBLOCK:\n{block}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + assert!( + !block.contains("stopped: invalid frame") + && !block.contains("stopped: read error") + && !block.contains("stopped: unsupported CFI"), + "deep multi-bt stack should not stop on an unwind error\nBLOCK:\n{block}\nSTDERR:\n{stderr}" + ); + } + assert!( + blocks[1].contains(" raw=0x") && blocks[1].contains(" cookie=0x"), + "second raw bt should retain raw debug metadata\nBLOCK:\n{}", + blocks[1] + ); + + Ok(()) +} + +#[tokio::test] +async fn test_conditional_tail_call_backtrace_ignores_skipped_slots() -> anyhow::Result<()> { + init(); + + let script = r#" +trace dummy_touch { + if value % 2 == 0 { + print "HOT_COND_EVEN"; + bt full; + } + if value % 2 != 0 { + print "HOT_COND_ODD"; + bt raw noinline; + } +} +"#; + + let (count, stdout, stderr) = + run_hot_backtrace_with_depth_and_rate(script, 128, 500, 4).await?; + if count == 0 && stderr.contains("BPF_PROG_LOAD") { + return Ok(()); + } + + let even_events = event_chunks_with_marker(&stdout, "HOT_COND_EVEN"); + let odd_events = event_chunks_with_marker(&stdout, "HOT_COND_ODD"); + assert!( + !even_events.is_empty() && !odd_events.is_empty(), + "conditional bt test should observe both mutually exclusive branches\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + + for event in even_events.iter().chain(odd_events.iter()).take(8) { + let bt_count = event.matches("backtrace:").count(); + assert_eq!( + bt_count, 1, + "each event should process only the executed bt slot\nEVENT:\n{event}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + assert!( + event.contains("#5 ") && event.contains("libc.so.6+"), + "executed conditional bt should unwind past the inline prefix\nEVENT:\n{event}" + ); + assert!( + !event.contains("stopped: invalid frame") + && !event.contains("stopped: read error") + && !event.contains("stopped: unsupported CFI"), + "conditional bt should not consume stale slot state\nEVENT:\n{event}" + ); + } + + assert!( + even_events.iter().any(|event| !event.contains(" raw=0x")), + "even branch uses `bt full` and should not require raw metadata\nSTDOUT:\n{stdout}" + ); + assert!( + odd_events + .iter() + .any(|event| event.contains(" raw=0x") && event.contains(" cookie=0x")), + "odd branch uses `bt raw noinline` and should include raw metadata\nSTDOUT:\n{stdout}" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_default_bt_and_backtrace_alias_render_distinct_modes() -> anyhow::Result<()> { + init(); + + let script = r#" +trace dummy_touch { + print "HOT_BT_ALIAS"; + bt; + backtrace raw noinline; +} +"#; + + let (count, stdout, stderr) = run_hot_backtrace_with_depth(script, 5).await?; + if count == 0 && stderr.contains("BPF_PROG_LOAD") { + return Ok(()); + } + + let blocks = backtrace_blocks_after(&stdout, "HOT_BT_ALIAS", 5)?; + assert!( + blocks.len() >= 2, + "`bt;` and `backtrace ...;` should emit separate backtrace blocks\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + + let default_bt = &blocks[0]; + assert!( + default_bt.contains("#0 dummy_touch(long unsigned int value)") + && default_bt.contains("#1 hot_bt_leaf(long unsigned int value)"), + "default `bt;` should symbolize frames and parameters\nBLOCK:\n{default_bt}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + assert!( + !default_bt.contains(" raw=0x") && !default_bt.contains(" cookie=0x"), + "default `bt;` should not expose raw debug metadata\nBLOCK:\n{default_bt}" + ); + + let raw_alias = &blocks[1]; + assert!( + raw_alias.contains(" raw=0x") && raw_alias.contains(" cookie=0x"), + "`backtrace raw noinline;` should preserve raw debug metadata\nBLOCK:\n{raw_alias}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + assert!( + !raw_alias.contains(".inline"), + "`noinline` should suppress inline pseudo-frames\nBLOCK:\n{raw_alias}" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_inline_and_noinline_bt_modes_control_inline_frames() -> anyhow::Result<()> { + init(); + + let script = r#" +trace inline_callsite_program.c:43 { + print "INLINE_BT_MODES"; + bt inline; + bt noinline; +} +"#; + + let (count, stdout, stderr) = run_inline_callsite_backtrace(script, 5).await?; + if count == 0 && stderr.contains("BPF_PROG_LOAD") { + return Ok(()); + } + + let blocks = backtrace_blocks_after(&stdout, "INLINE_BT_MODES", 5)?; + assert!( + blocks.len() >= 2, + "`bt inline;` and `bt noinline;` should emit separate backtrace blocks\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + + let inline_bt = &blocks[0]; + assert!( + inline_bt.contains("#0.inline add3"), + "`bt inline;` should include the inlined add3 pseudo-frame\nBLOCK:\n{inline_bt}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + assert!( + inline_bt.contains("#0 "), + "`bt inline;` should also include the physical frame after inline pseudo-frames\nBLOCK:\n{inline_bt}" + ); + + let noinline_bt = &blocks[1]; + assert!( + !noinline_bt.contains(".inline"), + "`bt noinline;` should suppress inline pseudo-frames\nBLOCK:\n{noinline_bt}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + assert!( + noinline_bt.contains("#0 "), + "`bt noinline;` should still include the physical current frame\nBLOCK:\n{noinline_bt}" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_non_pie_pid_backtrace_uses_load_bias_for_runtime_cfi_rows() -> anyhow::Result<()> { + init(); + + let binary_path = get_backtrace_hot_nopie_binary()?; + let script = r#" +trace dummy_touch { + print "HOT_NOPIE"; + bt full; +} +"#; + let depth_arg = OsString::from("7"); + let (count, stdout, stderr) = run_backtrace_binary_with_args( + &binary_path, + script, + 250, + 3, + vec![OsString::from("--backtrace-depth"), depth_arg], + None, + ) + .await?; + if count == 0 && stderr.contains("BPF_PROG_LOAD") { + return Ok(()); + } + + let block = first_backtrace_block_after(&stdout, "HOT_NOPIE", 7)?; + assert!( + block.contains("#1 hot_bt_leaf") && block.contains("#4 main"), + "non-PIE PID-mode CFI rows should match raw ET_EXEC PCs and unwind callers\nBLOCK:\n{block}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + assert!( + !block.contains("stopped: unsupported CFI"), + "non-PIE PID-mode rows should not be shifted by mapping base\nBLOCK:\n{block}\nSTDERR:\n{stderr}" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_cross_module_backtrace_resolves_so_and_exe_frames() -> anyhow::Result<()> { + init(); + + let script = r#" +trace cross_module_lib_leaf { + print "CROSS_MODULE_STACK"; + bt full; +} +"#; + + let (count, stdout, stderr) = run_backtrace_fixture_with_depth_and_rate( + "backtrace_cross_module_program", + script, + 5, + 250, + 3, + ) + .await?; + if count == 0 && stderr.contains("BPF_PROG_LOAD") { + return Ok(()); + } + + let block = first_backtrace_block_after(&stdout, "CROSS_MODULE_STACK", 5)?; + assert!( + block.contains("backtrace: truncated, 5 frames (max 5)"), + "expected a bounded cross-module stack prefix\nBLOCK:\n{block}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + assert_ordered_patterns( + block, + &[ + "#0 cross_module_lib_leaf", + "#1 cross_module_lib_probe", + "#2 cross_module_main_caller", + "#3 cross_module_main_loop", + "#4 main", + ], + )?; + assert!( + block.contains("[libbacktrace_cross_module.so+"), + "expected at least one shared-library frame\nBLOCK:\n{block}" + ); + assert!( + block.contains("[backtrace_cross_module_program+"), + "expected at least one executable frame\nBLOCK:\n{block}" + ); + assert!( + !block.contains("stopped: invalid frame") + && !block.contains("stopped: read error") + && !block.contains("stopped: unsupported CFI"), + "cross-module stack should not stop on an unwind error\nBLOCK:\n{block}\nSTDERR:\n{stderr}" + ); + assert_no_adjacent_duplicate_frame_locations(block)?; + + Ok(()) +} + +#[tokio::test] +async fn test_hot_backtrace_full_renders_function_parameters() -> anyhow::Result<()> { + init(); + + let script = r#" +trace dummy_touch { + print "HOT_PARAMS"; + bt full; +} +"#; + + let (count, stdout, stderr) = run_hot_backtrace_with_depth(script, 5).await?; + if count == 0 && stderr.contains("BPF_PROG_LOAD") { + return Ok(()); + } + + let block = first_backtrace_block_after(&stdout, "HOT_PARAMS", 5)?; + assert!( + block.contains("#0 dummy_touch(long unsigned int value)"), + "current frame should render its formal parameter signature\nBLOCK:\n{block}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + assert!( + block.contains("#1 hot_bt_leaf(long unsigned int value)"), + "caller frame should render its formal parameter signature\nBLOCK:\n{block}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + assert!( + block.contains("#2 hot_bt_mid(long unsigned int value)"), + "deeper caller frame should render its formal parameter signature\nBLOCK:\n{block}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + assert!( + !block.contains(" raw=0x") && !block.contains(" cookie=0x"), + "bt full should stay human-readable and must not print raw/cookie debug metadata\nBLOCK:\n{block}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_function_parameters_use_signature_lookup_path() -> anyhow::Result<()> { + init(); + + let binary_path = FIXTURES.get_test_binary("backtrace_hot_program")?; + let analyzer = ghostscope_dwarf::DwarfAnalyzer::from_exec_path(&binary_path).await?; + let dummy = analyzer + .lookup_function_address_by_name("dummy_touch") + .ok_or_else(|| anyhow::anyhow!("missing dummy_touch function"))?; + let ctx = analyzer.resolve_pc(&ModuleAddress::new(dummy.module_path, dummy.address + 0x10))?; + let params = analyzer.function_parameters(&ctx)?; + + assert_eq!(params.len(), 1, "expected one dummy_touch parameter"); + assert_eq!(params[0].name, "value"); + assert_eq!(params[0].type_name, "long unsigned int"); + assert!( + !params[0].is_artificial, + "dummy_touch parameter should be a real source parameter" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_hot_backtrace_raw_renders_debug_metadata() -> anyhow::Result<()> { + init(); + + let script = r#" +trace dummy_touch { + print "HOT_RAW_META"; + bt raw; +} +"#; + + let (count, stdout, stderr) = run_hot_backtrace_with_depth(script, 5).await?; + if count == 0 && stderr.contains("BPF_PROG_LOAD") { + return Ok(()); + } + + let block = first_backtrace_block_after(&stdout, "HOT_RAW_META", 5)?; + assert!( + block.contains(" raw=0x"), + "bt raw should print the raw instruction pointer for debugging\nBLOCK:\n{block}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + assert!( + block.contains(" cookie=0x"), + "bt raw should print the module cookie for debugging\nBLOCK:\n{block}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_hot_backtrace_full_keeps_up_with_raw() -> anyhow::Result<()> { + init(); + + let raw_script = r#" +trace hot_bt_probe { + print "HOT_RAW"; + bt raw; +} +"#; + let full_script = r#" +trace hot_bt_probe { + print "HOT_FULL"; + bt full; +} +"#; + + let (raw_count, raw_stdout, raw_stderr) = run_hot_backtrace(raw_script).await?; + let (full_count, full_stdout, full_stderr) = run_hot_backtrace(full_script).await?; + + if raw_count == 0 && raw_stderr.contains("BPF_PROG_LOAD") { + return Ok(()); + } + + assert!( + raw_count >= 100, + "raw backtrace run did not receive enough events: raw_count={raw_count}\nSTDOUT: {raw_stdout}\nSTDERR: {raw_stderr}" + ); + assert!( + full_count >= 100, + "full backtrace run fell behind badly: raw_count={raw_count} full_count={full_count}\nSTDOUT: {full_stdout}\nSTDERR: {full_stderr}" + ); + assert!( + full_count * 2 >= raw_count, + "full backtrace symbolization is much slower than raw: raw_count={raw_count} full_count={full_count}\nFULL STDOUT: {full_stdout}\nFULL STDERR: {full_stderr}" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_backtrace_depth_128_loads_with_tail_calls() -> anyhow::Result<()> { + init(); + + let script = r#" +trace hot_bt_probe { + print "HOT_DEEP"; + bt full; +} +"#; + + let (count, stdout, stderr) = run_hot_backtrace_with_depth(script, 128).await?; + assert!( + !stderr.contains("LLVM ERROR"), + "depth=128 should not hit LLVM branch range errors\nSTDOUT: {stdout}\nSTDERR: {stderr}" + ); + assert!( + count > 0, + "depth=128 backtrace did not produce events\nSTDOUT: {stdout}\nSTDERR: {stderr}" + ); + assert!( + stdout.contains("#1 "), + "depth=128 tail-call backtrace should unwind at least one caller frame\nSTDOUT: {stdout}\nSTDERR: {stderr}" + ); + assert!( + stdout.contains("#2 "), + "depth=128 tail-call backtrace should continue past the first caller frame\nSTDOUT: {stdout}\nSTDERR: {stderr}" + ); + assert!( + stdout.contains("libc.so.6+"), + "depth=128 full backtrace should cross into libc instead of stopping at the executable boundary\nSTDOUT: {stdout}\nSTDERR: {stderr}" + ); + assert!( + !stdout.contains("stopped: invalid frame") + && !stdout.contains("stopped: read error") + && !stdout.contains("stopped: unsupported CFI"), + "depth=128 full backtrace should not stop on an unwind error\nSTDOUT: {stdout}\nSTDERR: {stderr}" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_deep_full_backtrace_stays_warm_under_event_load() -> anyhow::Result<()> { + init(); + + let script = r#" +trace hot_bt_probe { + print "HOT_DEEP_WARM"; + bt full; +} +"#; + + let (count, stdout, stderr) = + run_hot_backtrace_with_depth_and_rate(script, 128, 2000, 3).await?; + if count == 0 && stderr.contains("BPF_PROG_LOAD") { + return Ok(()); + } + + assert!( + count >= 500, + "deep full backtrace should keep rendering after caches are warm: count={count}\nSTDOUT: {stdout}\nSTDERR: {stderr}" + ); + assert!( + !stdout.contains("stopped: invalid frame") + && !stdout.contains("stopped: read error") + && !stdout.contains("stopped: unsupported CFI"), + "deep full backtrace should not regress into unwind errors under event load\nSTDOUT: {stdout}\nSTDERR: {stderr}" + ); + assert!( + !stderr.contains("script output saturated"), + "deep full backtrace rendering should not saturate the script output path at the regression-test rate\nSTDOUT: {stdout}\nSTDERR: {stderr}" + ); + assert!( + stdout.contains("libc.so.6+"), + "deep full backtrace should keep symbolizing cross-module frames under event load\nSTDOUT: {stdout}\nSTDERR: {stderr}" + ); + + Ok(()) +} diff --git a/e2e-tests/tests/common/mod.rs b/e2e-tests/tests/common/mod.rs index 91f1741b..152656ac 100644 --- a/e2e-tests/tests/common/mod.rs +++ b/e2e-tests/tests/common/mod.rs @@ -33,6 +33,9 @@ lazy_static! { static ref COMPILE_SHORT_LIVED_LONG_COMM_RESULT: Mutex>> = Mutex::new(None); static ref COMPILE_C_MULTITHREAD_RESULT: Mutex>> = Mutex::new(None); + static ref COMPILE_BACKTRACE_HOT_RESULT: Mutex>> = Mutex::new(None); + static ref COMPILE_BACKTRACE_CROSS_MODULE_RESULT: Mutex>> = + Mutex::new(None); static ref COMPILE_SCALAR_TYPES_RESULT: Mutex>> = Mutex::new(None); static ref COMPILE_SCALAR_TYPES_OPTIMIZED_RESULT: Mutex>> = Mutex::new(None); @@ -76,6 +79,8 @@ enum RegisteredFixtureKind { LateGlobals, ShortLivedLongComm, CMultithread, + BacktraceHot, + BacktraceCrossModule, ScalarTypes, CastTypes, RustGlobal, @@ -146,6 +151,18 @@ const REGISTERED_FIXTURES: &[RegisteredFixture] = &[ cleanup: CleanupCommand::Make, kind: RegisteredFixtureKind::CMultithread, }, + RegisteredFixture { + name: "backtrace_hot_program", + directory: "backtrace_hot_program", + cleanup: CleanupCommand::Make, + kind: RegisteredFixtureKind::BacktraceHot, + }, + RegisteredFixture { + name: "backtrace_cross_module_program", + directory: "backtrace_cross_module_program", + cleanup: CleanupCommand::Make, + kind: RegisteredFixtureKind::BacktraceCrossModule, + }, RegisteredFixture { name: "scalar_types_program", directory: "scalar_types_program", @@ -461,6 +478,14 @@ impl RegisteredFixture { ensure_c_multithread_program_compiled()?; Ok(dir.join("c_multithread_program")) } + RegisteredFixtureKind::BacktraceHot => { + ensure_backtrace_hot_program_compiled()?; + Ok(dir.join("backtrace_hot_program")) + } + RegisteredFixtureKind::BacktraceCrossModule => { + ensure_backtrace_cross_module_program_compiled()?; + Ok(dir.join("backtrace_cross_module_program")) + } RegisteredFixtureKind::ScalarTypes => { let bin_name = match opt_level { OptimizationLevel::Debug => { @@ -1254,6 +1279,8 @@ static COMPILE_GLOBALS_OPTIMIZED: Once = Once::new(); static COMPILE_LATE_GLOBALS: Once = Once::new(); static COMPILE_SHORT_LIVED_LONG_COMM: Once = Once::new(); static COMPILE_C_MULTITHREAD: Once = Once::new(); +static COMPILE_BACKTRACE_HOT: Once = Once::new(); +static COMPILE_BACKTRACE_CROSS_MODULE: Once = Once::new(); static COMPILE_SCALAR_TYPES: Once = Once::new(); static COMPILE_SCALAR_TYPES_OPTIMIZED: Once = Once::new(); static COMPILE_CAST_TYPES: Once = Once::new(); @@ -1453,6 +1480,70 @@ fn ensure_c_multithread_program_compiled() -> anyhow::Result<()> { } } +fn ensure_backtrace_hot_program_compiled() -> anyhow::Result<()> { + COMPILE_BACKTRACE_HOT.call_once(|| { + let compile_result = compile_c_make_fixture( + "backtrace_hot_program", + FixtureCompiler::Default, + "-Wall -Wextra -g -O0", + ); + *COMPILE_BACKTRACE_HOT_RESULT.lock().unwrap() = Some(compile_result); + }); + + match COMPILE_BACKTRACE_HOT_RESULT.lock().unwrap().as_ref() { + Some(Ok(())) => Ok(()), + Some(Err(e)) => Err(anyhow::anyhow!("{e}")), + None => panic!("Compilation result should be set after call_once"), + } +} + +fn ensure_backtrace_cross_module_program_compiled() -> anyhow::Result<()> { + COMPILE_BACKTRACE_CROSS_MODULE.call_once(|| { + let compile_result = (|| -> anyhow::Result<()> { + let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/backtrace_cross_module_program"); + if let Some(result) = use_precompiled_outputs( + "backtrace_cross_module_program", + &[ + base.join("backtrace_cross_module_program"), + base.join("libbacktrace_cross_module.so"), + ], + ) { + return result; + } + + println!("Compiling backtrace_cross_module_program (Debug) in {base:?}"); + let _ = Command::new("make") + .arg("clean") + .current_dir(base.clone()) + .status() + .is_ok(); + let out = Command::new("make").arg("all").current_dir(base).output()?; + if out.status.success() { + println!("✓ Successfully compiled backtrace_cross_module_program"); + Ok(()) + } else { + let stderr = String::from_utf8_lossy(&out.stderr); + Err(anyhow::anyhow!( + "Failed to compile backtrace_cross_module_program: {}", + stderr + )) + } + })(); + *COMPILE_BACKTRACE_CROSS_MODULE_RESULT.lock().unwrap() = Some(compile_result); + }); + + match COMPILE_BACKTRACE_CROSS_MODULE_RESULT + .lock() + .unwrap() + .as_ref() + { + Some(Ok(())) => Ok(()), + Some(Err(e)) => Err(anyhow::anyhow!("{e}")), + None => panic!("Compilation result should be set after call_once"), + } +} + fn ensure_scalar_types_program_compiled() -> anyhow::Result<()> { COMPILE_SCALAR_TYPES.call_once(|| { let compile_result = compile_scalar_types_program_target( diff --git a/e2e-tests/tests/common/runner.rs b/e2e-tests/tests/common/runner.rs index 33c76ae0..6d6c12c0 100644 --- a/e2e-tests/tests/common/runner.rs +++ b/e2e-tests/tests/common/runner.rs @@ -54,6 +54,7 @@ pub struct GhostscopeRunner { enable_file_logging: bool, enable_console_logging: bool, sandbox: Option, + config_content: Option, extra_args: Vec, } @@ -72,6 +73,7 @@ impl Default for GhostscopeRunner { enable_file_logging: false, enable_console_logging: false, sandbox: None, + config_content: None, extra_args: Vec::new(), } } @@ -157,6 +159,12 @@ impl GhostscopeRunner { self } + #[allow(dead_code)] + pub fn with_config_content(mut self, content: impl Into) -> Self { + self.config_content = Some(content.into()); + self + } + pub async fn run(self) -> Result<(i32, String, String)> { let (exit_code, stdout, stderr, ()) = self .run_internal( @@ -234,13 +242,18 @@ impl GhostscopeRunner { use std::io::Write as _; script_file.write_all(self.script_content.as_bytes())?; let script_path = sandbox.path_in_sandbox(script_file.path())?; - let sysmon_config_file = if self.disable_sysmon_for_target { - let mut file = create_config_file()?; - file.write_all(b"[ebpf]\nenable_sysmon_for_target = false\n")?; - Some(file) - } else { - None - }; + let config_file = + if self.disable_sysmon_for_target || self.config_content.as_deref().is_some() { + let mut file = create_config_file()?; + let content = build_config_content( + self.config_content.as_deref(), + self.disable_sysmon_for_target, + ); + file.write_all(content.as_bytes())?; + Some(file) + } else { + None + }; let mut args: Vec = Vec::new(); if let Some(ref target) = self.target { @@ -258,7 +271,7 @@ impl GhostscopeRunner { args.push(OsString::from("--script-file")); args.push(script_path.into_os_string()); - if let Some(config_file) = sysmon_config_file.as_ref() { + if let Some(config_file) = config_file.as_ref() { args.push(OsString::from("--config")); args.push( sandbox @@ -755,6 +768,37 @@ fn create_config_file() -> Result { .map_err(Into::into) } +fn build_config_content(config_content: Option<&str>, disable_sysmon_for_target: bool) -> String { + let mut lines = config_content + .unwrap_or_default() + .lines() + .map(ToOwned::to_owned) + .collect::>(); + + if disable_sysmon_for_target && !contains_config_key(&lines, "enable_sysmon_for_target") { + let setting = "enable_sysmon_for_target = false".to_string(); + if let Some(ebpf_table_index) = lines.iter().position(|line| line.trim() == "[ebpf]") { + lines.insert(ebpf_table_index + 1, setting); + } else { + lines.push("[ebpf]".to_string()); + lines.push(setting); + } + } + + let mut content = lines.join("\n"); + if !content.is_empty() && !content.ends_with('\n') { + content.push('\n'); + } + content +} + +fn contains_config_key(lines: &[String], key: &str) -> bool { + lines.iter().any(|line| { + let trimmed = line.trim_start(); + !trimmed.starts_with('#') && trimmed.starts_with(key) + }) +} + fn env_bool(name: &str) -> Option { let raw = env::var(name).ok()?; let value = raw.trim(); diff --git a/e2e-tests/tests/entry_value_recovery_execution.rs b/e2e-tests/tests/entry_value_recovery_execution.rs index d57d622c..094402ef 100644 --- a/e2e-tests/tests/entry_value_recovery_execution.rs +++ b/e2e-tests/tests/entry_value_recovery_execution.rs @@ -1024,7 +1024,7 @@ async fn test_compact_unwind_table_exposes_pc_row() -> anyhow::Result<()> { .compact_unwind_table_for_module(ctx.module)? .ok_or_else(|| anyhow::anyhow!("no compact unwind table returned for module"))?; - assert_eq!(table_by_context, table_by_module); + assert_eq!(table_by_context.as_ref(), table_by_module.as_ref()); let stats = table_by_context.stats(); assert!(stats.row_count > 0, "compact unwind table is empty"); diff --git a/e2e-tests/tests/fixtures/backtrace_cross_module_program/Makefile b/e2e-tests/tests/fixtures/backtrace_cross_module_program/Makefile new file mode 100644 index 00000000..f4f5c408 --- /dev/null +++ b/e2e-tests/tests/fixtures/backtrace_cross_module_program/Makefile @@ -0,0 +1,26 @@ +CC ?= gcc +BASE_CFLAGS ?= -Wall -Wextra -g +CFLAGS ?= $(BASE_CFLAGS) -O0 -fomit-frame-pointer -fPIC +BINARY ?= backtrace_cross_module_program +OBJ ?= $(BINARY).o +SHARED_LIB ?= libbacktrace_cross_module.so +SHARED_OBJ ?= backtrace_cross_module_lib.o + +all: $(BINARY) $(SHARED_LIB) + +$(BINARY): $(OBJ) $(SHARED_LIB) + $(CC) $(BASE_CFLAGS) -O0 -fomit-frame-pointer -o $@ $(OBJ) -L. -lbacktrace_cross_module -Wl,-rpath,'$$ORIGIN' + +$(OBJ): backtrace_cross_module_program.c backtrace_cross_module_lib.h + $(CC) $(CFLAGS) -c -o $@ backtrace_cross_module_program.c + +$(SHARED_LIB): $(SHARED_OBJ) + $(CC) -shared -o $@ $(SHARED_OBJ) + +$(SHARED_OBJ): backtrace_cross_module_lib.c backtrace_cross_module_lib.h + $(CC) $(CFLAGS) -c -o $@ backtrace_cross_module_lib.c + +clean: + rm -f *.o backtrace_cross_module_program libbacktrace_cross_module.so + +.PHONY: all clean diff --git a/e2e-tests/tests/fixtures/backtrace_cross_module_program/backtrace_cross_module_lib.c b/e2e-tests/tests/fixtures/backtrace_cross_module_program/backtrace_cross_module_lib.c new file mode 100644 index 00000000..1eeac21f --- /dev/null +++ b/e2e-tests/tests/fixtures/backtrace_cross_module_program/backtrace_cross_module_lib.c @@ -0,0 +1,17 @@ +#include "backtrace_cross_module_lib.h" + +volatile unsigned long cross_module_shared_sink = 0; + +__attribute__((noinline)) unsigned long cross_module_lib_leaf(unsigned long value) +{ + asm volatile("" : "+r"(value) :: "memory"); + return value + 17; +} + +__attribute__((noinline)) unsigned long cross_module_lib_probe(unsigned long value) +{ + unsigned long result = cross_module_lib_leaf(value + 1); + cross_module_shared_sink += result; + asm volatile("" ::: "memory"); + return cross_module_shared_sink; +} diff --git a/e2e-tests/tests/fixtures/backtrace_cross_module_program/backtrace_cross_module_lib.h b/e2e-tests/tests/fixtures/backtrace_cross_module_program/backtrace_cross_module_lib.h new file mode 100644 index 00000000..7ec5b284 --- /dev/null +++ b/e2e-tests/tests/fixtures/backtrace_cross_module_program/backtrace_cross_module_lib.h @@ -0,0 +1,7 @@ +#ifndef GHOSTSCOPE_BACKTRACE_CROSS_MODULE_LIB_H +#define GHOSTSCOPE_BACKTRACE_CROSS_MODULE_LIB_H + +unsigned long cross_module_lib_leaf(unsigned long value); +unsigned long cross_module_lib_probe(unsigned long value); + +#endif diff --git a/e2e-tests/tests/fixtures/backtrace_cross_module_program/backtrace_cross_module_program.c b/e2e-tests/tests/fixtures/backtrace_cross_module_program/backtrace_cross_module_program.c new file mode 100644 index 00000000..9f6516ed --- /dev/null +++ b/e2e-tests/tests/fixtures/backtrace_cross_module_program/backtrace_cross_module_program.c @@ -0,0 +1,42 @@ +#include "backtrace_cross_module_lib.h" + +#include +#include +#include + +static volatile sig_atomic_t keep_running = 1; +static volatile unsigned long cross_module_main_sink = 0; + +__attribute__((noinline)) static unsigned long cross_module_main_caller(unsigned long value) +{ + unsigned long result = cross_module_lib_probe(value + 1); + asm volatile("" ::: "memory"); + return result; +} + +__attribute__((noinline)) static void cross_module_main_loop(unsigned long value) +{ + cross_module_main_sink += cross_module_main_caller(value + 1); + asm volatile("" ::: "memory"); +} + +static void handle_signal(int signo) +{ + (void)signo; + keep_running = 0; +} + +int main(void) +{ + signal(SIGINT, handle_signal); + signal(SIGTERM, handle_signal); + setvbuf(stdout, NULL, _IONBF, 0); + puts("backtrace_cross_module_program ready"); + + for (unsigned long i = 0; keep_running; i++) { + cross_module_main_loop(i); + usleep(1000); + } + + return (int)(cross_module_main_sink & 1); +} diff --git a/e2e-tests/tests/fixtures/backtrace_hot_program/Makefile b/e2e-tests/tests/fixtures/backtrace_hot_program/Makefile new file mode 100644 index 00000000..d6292237 --- /dev/null +++ b/e2e-tests/tests/fixtures/backtrace_hot_program/Makefile @@ -0,0 +1,24 @@ +CC ?= gcc +BASE_CFLAGS ?= -Wall -Wextra -g +CFLAGS ?= $(BASE_CFLAGS) -O0 -fomit-frame-pointer +BINARY ?= backtrace_hot_program +OBJ ?= $(BINARY).o + +all: $(BINARY) + +$(BINARY): $(OBJ) + $(CC) $(CFLAGS) -o $@ $^ + +$(OBJ): backtrace_hot_program.c + $(CC) $(CFLAGS) -c -o $@ $< + +backtrace_hot_program_nopie: backtrace_hot_program_nopie.o + $(CC) $(BASE_CFLAGS) -no-pie -o $@ $^ + +backtrace_hot_program_nopie.o: backtrace_hot_program.c + $(CC) $(BASE_CFLAGS) -O0 -fomit-frame-pointer -fno-pie -c -o $@ $< + +clean: + rm -f *.o backtrace_hot_program backtrace_hot_program_nopie + +.PHONY: all clean diff --git a/e2e-tests/tests/fixtures/backtrace_hot_program/backtrace_hot_program.c b/e2e-tests/tests/fixtures/backtrace_hot_program/backtrace_hot_program.c new file mode 100644 index 00000000..488c5d9c --- /dev/null +++ b/e2e-tests/tests/fixtures/backtrace_hot_program/backtrace_hot_program.c @@ -0,0 +1,112 @@ +#include +#include +#include +#include + +static volatile sig_atomic_t keep_running = 1; +static volatile uint64_t hot_sink = 0; + +typedef struct Dummy0 { uint64_t a; uint64_t b; char name[32]; } Dummy0; +typedef struct Dummy1 { uint64_t a; uint64_t b; char name[32]; } Dummy1; +typedef struct Dummy2 { uint64_t a; uint64_t b; char name[32]; } Dummy2; +typedef struct Dummy3 { uint64_t a; uint64_t b; char name[32]; } Dummy3; +typedef struct Dummy4 { uint64_t a; uint64_t b; char name[32]; } Dummy4; +typedef struct Dummy5 { uint64_t a; uint64_t b; char name[32]; } Dummy5; +typedef struct Dummy6 { uint64_t a; uint64_t b; char name[32]; } Dummy6; +typedef struct Dummy7 { uint64_t a; uint64_t b; char name[32]; } Dummy7; +typedef struct Dummy8 { uint64_t a; uint64_t b; char name[32]; } Dummy8; +typedef struct Dummy9 { uint64_t a; uint64_t b; char name[32]; } Dummy9; +typedef struct Dummy10 { uint64_t a; uint64_t b; char name[32]; } Dummy10; +typedef struct Dummy11 { uint64_t a; uint64_t b; char name[32]; } Dummy11; +typedef struct Dummy12 { uint64_t a; uint64_t b; char name[32]; } Dummy12; +typedef struct Dummy13 { uint64_t a; uint64_t b; char name[32]; } Dummy13; +typedef struct Dummy14 { uint64_t a; uint64_t b; char name[32]; } Dummy14; +typedef struct Dummy15 { uint64_t a; uint64_t b; char name[32]; } Dummy15; +typedef struct Dummy16 { uint64_t a; uint64_t b; char name[32]; } Dummy16; +typedef struct Dummy17 { uint64_t a; uint64_t b; char name[32]; } Dummy17; +typedef struct Dummy18 { uint64_t a; uint64_t b; char name[32]; } Dummy18; +typedef struct Dummy19 { uint64_t a; uint64_t b; char name[32]; } Dummy19; +typedef struct Dummy20 { uint64_t a; uint64_t b; char name[32]; } Dummy20; +typedef struct Dummy21 { uint64_t a; uint64_t b; char name[32]; } Dummy21; +typedef struct Dummy22 { uint64_t a; uint64_t b; char name[32]; } Dummy22; +typedef struct Dummy23 { uint64_t a; uint64_t b; char name[32]; } Dummy23; +typedef struct Dummy24 { uint64_t a; uint64_t b; char name[32]; } Dummy24; +typedef struct Dummy25 { uint64_t a; uint64_t b; char name[32]; } Dummy25; +typedef struct Dummy26 { uint64_t a; uint64_t b; char name[32]; } Dummy26; +typedef struct Dummy27 { uint64_t a; uint64_t b; char name[32]; } Dummy27; +typedef struct Dummy28 { uint64_t a; uint64_t b; char name[32]; } Dummy28; +typedef struct Dummy29 { uint64_t a; uint64_t b; char name[32]; } Dummy29; +typedef struct Dummy30 { uint64_t a; uint64_t b; char name[32]; } Dummy30; +typedef struct Dummy31 { uint64_t a; uint64_t b; char name[32]; } Dummy31; + +/* + * Keep a sizeable set of compact DWARF CFI rows before the real hot stack. + * This makes the runtime row lookup exercise non-trivial map indexes instead + * of only the first few rows. + */ +#define CAT2(a, b) a##b +#define CAT(a, b) CAT2(a, b) +#define DECL_BT_FILLER() \ + __attribute__((noinline, used, unused)) static uint64_t CAT(bt_filler_, __COUNTER__)(uint64_t value) \ + { \ + asm volatile("" : "+r"(value)); \ + return value + (uint64_t)__COUNTER__; \ + } +#define REPEAT_1(M) M() +#define REPEAT_2(M) REPEAT_1(M) REPEAT_1(M) +#define REPEAT_4(M) REPEAT_2(M) REPEAT_2(M) +#define REPEAT_8(M) REPEAT_4(M) REPEAT_4(M) +#define REPEAT_16(M) REPEAT_8(M) REPEAT_8(M) +#define REPEAT_32(M) REPEAT_16(M) REPEAT_16(M) +#define REPEAT_64(M) REPEAT_32(M) REPEAT_32(M) +#define REPEAT_128(M) REPEAT_64(M) REPEAT_64(M) +#define REPEAT_256(M) REPEAT_128(M) REPEAT_128(M) +#define REPEAT_512(M) REPEAT_256(M) REPEAT_256(M) + +REPEAT_512(DECL_BT_FILLER) + +__attribute__((noinline)) static uint64_t dummy_touch(uint64_t value) +{ + Dummy0 d0 = {value, value + 1, "d0"}; + Dummy7 d7 = {value + 7, value + 8, "d7"}; + Dummy15 d15 = {value + 15, value + 16, "d15"}; + Dummy31 d31 = {value + 31, value + 32, "d31"}; + return d0.a + d7.b + d15.a + d31.b; +} + +__attribute__((noinline)) static void hot_bt_leaf(uint64_t value) +{ + hot_sink += dummy_touch(value); + asm volatile("" ::: "memory"); +} + +__attribute__((noinline)) static void hot_bt_mid(uint64_t value) +{ + hot_bt_leaf(value + 1); +} + +__attribute__((noinline)) void hot_bt_probe(uint64_t value) +{ + hot_bt_mid(value + 1); +} + +static void handle_signal(int signo) +{ + (void)signo; + keep_running = 0; +} + +int main(void) +{ + signal(SIGINT, handle_signal); + signal(SIGTERM, handle_signal); + setvbuf(stdout, NULL, _IONBF, 0); + puts("backtrace_hot_program ready"); + + for (uint64_t i = 0; keep_running; i++) { + hot_bt_probe(i); + usleep(1000); + } + + return (int)(hot_sink & 1); +} diff --git a/e2e-tests/tests/script_execution.rs b/e2e-tests/tests/script_execution.rs index 16895928..92b7d121 100644 --- a/e2e-tests/tests/script_execution.rs +++ b/e2e-tests/tests/script_execution.rs @@ -302,6 +302,79 @@ trace print_record { Ok(()) } +#[tokio::test] +async fn test_backtrace_outputs_dwarf_frames_between_prints() -> anyhow::Result<()> { + init(); + ensure_global_cleanup_registered(); + + let opt_level = OptimizationLevel::Debug; + let target = get_global_test_target_with_opt(opt_level).await?; + + let script_content = r#" +trace test_function { + print "before-bt"; + bt full; + print "after-bt"; +} +"#; + + let (exit_code, stdout, stderr) = common::runner::GhostscopeRunner::new() + .with_script(script_content) + .attach_to(&target) + .timeout_secs(5) + .enable_sysmon_for_target(false) + .with_cli_args(["--backtrace-depth", "3"]) + .run() + .await?; + + if exit_code != 0 && stderr.contains("BPF_PROG_LOAD") { + return Ok(()); + } + assert_eq!( + exit_code, 0, + "unexpected error: stderr={stderr}\nstdout={stdout}" + ); + + let before = stdout + .find("before-bt") + .ok_or_else(|| anyhow::anyhow!("missing before print:\n{stdout}"))?; + let backtrace = stdout + .find("backtrace:") + .ok_or_else(|| anyhow::anyhow!("missing backtrace header:\n{stdout}"))?; + assert!( + stdout[backtrace..].contains("(max 3)"), + "backtrace header should show configured max depth:\n{stdout}" + ); + let frame = stdout + .find("#0") + .ok_or_else(|| anyhow::anyhow!("missing backtrace frame:\n{stdout}"))?; + let function = stdout + .find("test_function") + .ok_or_else(|| anyhow::anyhow!("missing DWARF function name:\n{stdout}"))?; + let after = stdout + .find("after-bt") + .ok_or_else(|| anyhow::anyhow!("missing after print:\n{stdout}"))?; + + assert!( + before < backtrace, + "backtrace should follow first print:\n{stdout}" + ); + assert!( + backtrace < frame, + "frame should follow backtrace header:\n{stdout}" + ); + assert!( + frame <= function, + "function should be rendered in frame:\n{stdout}" + ); + assert!( + function < after, + "second print should follow frames:\n{stdout}" + ); + + Ok(()) +} + #[tokio::test] async fn test_special_pid_in_if_condition() -> anyhow::Result<()> { init(); diff --git a/ghostscope-compiler/src/ebpf/codegen/backtrace.rs b/ghostscope-compiler/src/ebpf/codegen/backtrace.rs index 7690c92d..d3dc54b0 100644 --- a/ghostscope-compiler/src/ebpf/codegen/backtrace.rs +++ b/ghostscope-compiler/src/ebpf/codegen/backtrace.rs @@ -1,80 +1,3742 @@ use super::*; +use crate::script::{BacktraceStatement, Statement}; +use aya_ebpf_bindings::bindings::bpf_func_id::{BPF_FUNC_map_lookup_elem, BPF_FUNC_tail_call}; +use ghostscope_dwarf::{ + CfaRulePlan, CompactUnwindRow, MemoryAccessSize, ModuleAddress, RegisterRecoveryPlan, +}; +use inkwell::basic_block::BasicBlock; +use std::{path::PathBuf, time::Instant}; + +const X86_64_DWARF_RIP: u16 = 16; +const X86_64_DWARF_RBP: u16 = 6; +const X86_64_DWARF_RSP: u16 = 7; +// DWARF row lookup expands into BPF branches, so large depths move to the +// tail-call step program after a short prefix to avoid LLVM branch-range limits. +const BPF_INLINE_BACKTRACE_FRAME_LIMIT: u8 = 5; +const BPF_BACKTRACE_FRAMES_PER_TAIL_CALL: u8 = 4; +const BPF_BACKTRACE_MAX_STEP_INVOCATIONS: u8 = 32; +const BPF_BACKTRACE_STEP_PROG_INDEX: u32 = 0; + +struct RuntimeBtUnwindRow<'ctx> { + found: IntValue<'ctx>, + cfa_register: IntValue<'ctx>, + cfa_offset: IntValue<'ctx>, + ra_kind: IntValue<'ctx>, + ra_register: IntValue<'ctx>, + ra_offset: IntValue<'ctx>, + rbp_kind: IntValue<'ctx>, + rbp_register: IntValue<'ctx>, + rbp_offset: IntValue<'ctx>, +} + +struct RuntimeBtRowScratch<'ctx> { + found_ptr: PointerValue<'ctx>, + cfa_register_ptr: PointerValue<'ctx>, + cfa_offset_ptr: PointerValue<'ctx>, + ra_kind_ptr: PointerValue<'ctx>, + ra_register_ptr: PointerValue<'ctx>, + ra_offset_ptr: PointerValue<'ctx>, + rbp_kind_ptr: PointerValue<'ctx>, + rbp_register_ptr: PointerValue<'ctx>, + rbp_offset_ptr: PointerValue<'ctx>, +} + +struct BtScratch<'ctx> { + row: RuntimeBtRowScratch<'ctx>, + next_rbp_ptr: PointerValue<'ctx>, + next_error_code_ptr: PointerValue<'ctx>, +} + +#[derive(Clone, Copy)] +struct BtRegisterState<'ctx> { + ip: IntValue<'ctx>, + rsp: IntValue<'ctx>, + rbp: IntValue<'ctx>, +} + +#[derive(Clone, Copy)] +struct BtNextFrame<'ctx> { + ip: IntValue<'ctx>, + rsp: IntValue<'ctx>, + rbp: IntValue<'ctx>, + error_code: IntValue<'ctx>, +} + +struct BtFrameValidation<'ctx> { + valid: IntValue<'ctx>, + complete: IntValue<'ctx>, + error_code: IntValue<'ctx>, +} impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { - // PrintVariableError instruction has been removed; compile-time errors are returned as Err, - // runtime errors are carried via per-variable status in Print* instructions. + pub(crate) fn prepare_backtrace_unwind_rows(&mut self, statements: &[Statement]) { + self.backtrace_unwind_rows.clear(); + self.backtrace_unwind_rows_use_runtime_pcs = false; + self.backtrace_tail_call_slots = 1; + self.next_backtrace_tail_call_slot = 0; + if !statements_have_backtrace(statements) { + return; + } + + let Some(analyzer) = self.process_analyzer else { + return; + }; + let Some(compile_ctx) = self.current_compile_time_context.clone() else { + return; + }; - /// Generate Backtrace instruction - pub fn generate_backtrace_instruction(&mut self, depth: u8) -> Result<()> { + let runtime_rows = self.runtime_backtrace_unwind_rows(analyzer); + if !runtime_rows.is_empty() { + self.backtrace_unwind_rows = runtime_rows; + self.backtrace_unwind_rows_use_runtime_pcs = true; + self.backtrace_tail_call_slots = self.required_backtrace_tail_call_slots(statements); + return; + } + + let module_address = ModuleAddress::new( + PathBuf::from(&compile_ctx.module_path), + compile_ctx.pc_address, + ); + let Ok(ctx) = analyzer.resolve_pc(&module_address) else { + return; + }; + let Ok(Some(table)) = analyzer.compact_unwind_table_for_context(&ctx) else { + return; + }; + + self.backtrace_unwind_rows = table + .rows + .iter() + .filter_map(backtrace_unwind_row_from_compact) + .collect(); + self.backtrace_unwind_rows + .sort_by_key(|row| (row.pc_start, row.pc_end)); + self.backtrace_tail_call_slots = self.required_backtrace_tail_call_slots(statements); + } + + fn required_backtrace_tail_call_slots(&self, statements: &[Statement]) -> u8 { + let depth = self + .compile_options + .backtrace_depth + .clamp(1, crate::MAX_BACKTRACE_DEPTH); + if depth <= BPF_INLINE_BACKTRACE_FRAME_LIMIT { + return 1; + } + count_backtrace_statements(statements).clamp(1, u8::MAX as usize) as u8 + } + + fn runtime_backtrace_unwind_rows( + &self, + analyzer: &ghostscope_dwarf::DwarfAnalyzer, + ) -> Vec { + let started_at = Instant::now(); + let mut rows = Vec::new(); + let mut modules = 0usize; + for module in analyzer.loaded_module_runtime_info() { + let Some(module_bias) = module.load_bias else { + continue; + }; + let Some(module_id) = analyzer.module_id_for_path(&module.module_path) else { + continue; + }; + let module_started_at = Instant::now(); + let Ok(Some(table)) = analyzer.compact_unwind_table_for_module(module_id) else { + continue; + }; + let row_start = rows.len(); + rows.extend(table.rows.iter().filter_map(|row| { + let mut wire = backtrace_unwind_row_from_compact(row)?; + wire.pc_start = wire.pc_start.checked_add(module_bias)?; + wire.pc_end = wire.pc_end.checked_add(module_bias)?; + Some(wire) + })); + modules += 1; + tracing::info!( + module = %module.module_path.display(), + compact_rows = table.rows.len(), + bpf_rows = rows.len().saturating_sub(row_start), + elapsed_ms = module_started_at.elapsed().as_millis(), + "Prepared bt unwind rows for module" + ); + } + rows.sort_by_key(|row| (row.pc_start, row.pc_end)); + tracing::info!( + modules, + rows = rows.len(), + elapsed_ms = started_at.elapsed().as_millis(), + "Prepared runtime DWARF bt unwind rows" + ); + rows + } + + pub fn generate_backtrace_instruction(&mut self, stmt: &BacktraceStatement) -> Result<()> { + if self.should_use_tail_call_backtrace() { + return self.generate_tail_call_backtrace_instruction(stmt); + } + + self.generate_inline_backtrace_instruction(stmt) + } + + fn should_use_tail_call_backtrace(&self) -> bool { + let depth = self + .compile_options + .backtrace_depth + .clamp(1, crate::MAX_BACKTRACE_DEPTH); + depth > BPF_INLINE_BACKTRACE_FRAME_LIMIT + && !self.backtrace_unwind_rows.is_empty() + && self.current_compile_time_context.is_some() + } + + /// Generate a DWARF-backed Backtrace instruction. + /// + /// eBPF records `(module_cookie, normalized_pc, raw_ip)` frames and advances + /// the unwind state from compact DWARF CFI rows. Userspace owns final source + /// line and inline-chain symbolization. + fn generate_inline_backtrace_instruction(&mut self, stmt: &BacktraceStatement) -> Result<()> { + let depth = self + .compile_options + .backtrace_depth + .clamp(1, crate::MAX_BACKTRACE_DEPTH); + let flags = backtrace_flags(stmt); info!("Generating Backtrace instruction: depth={}", depth); - // Reserve space directly for Backtrace instruction + let payload_size = + BACKTRACE_DATA_SIZE + depth as usize * std::mem::size_of::(); + let instruction_size = INSTRUCTION_HEADER_SIZE + payload_size; let inst_buffer = self - .reserve_instruction_region_or_return_zero( - (std::mem::size_of::() + std::mem::size_of::()) - as u64, - )? + .reserve_instruction_region_or_return_zero(instruction_size as u64)? .into_value_after_runtime_returns(); - // Write InstructionHeader.inst_type - // SAFETY: inst_buffer points at a reserved instruction region and the - // offset is within InstructionHeader. - let inst_type_ptr = unsafe { + self.store_u8_const( + inst_buffer, + std::mem::offset_of!(InstructionHeader, inst_type), + InstructionType::Backtrace as u8, + "bt_inst_type", + )?; + self.store_u16_const( + inst_buffer, + std::mem::offset_of!(InstructionHeader, data_length), + payload_size as u16, + "bt_data_length", + )?; + + let data_base = INSTRUCTION_HEADER_SIZE; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_REQUESTED_DEPTH_OFFSET, + depth, + "bt_requested_depth", + )?; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, + 1, + "bt_frame_count", + )?; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_FLAGS_OFFSET, + flags, + "bt_flags", + )?; + self.store_u16_const( + inst_buffer, + data_base + BACKTRACE_DATA_ERROR_CODE_OFFSET, + 0, + "bt_error_code", + )?; + + let Some(compile_ctx) = self.current_compile_time_context.clone() else { + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, + 0, + "bt_frame_count_no_context", + )?; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + BacktraceStatus::DwarfUnavailable as u8, + "bt_status_no_context", + )?; + return Ok(()); + }; + + let module_cookie = self.cookie_for_module_or_fallback(&compile_ctx.module_path); + let pt_regs = self.get_pt_regs_parameter()?; + let raw_ip = self.load_dwarf_register_i64(X86_64_DWARF_RIP, pt_regs)?; + let (module_bias, offsets_found) = self.generate_runtime_address_from_offsets( + self.context.i64_type().const_zero(), + 0, + module_cookie, + )?; + let normalized_pc = self.normalized_pc_from_raw(raw_ip, module_bias, offsets_found)?; + + self.store_backtrace_frame(inst_buffer, 0, module_cookie, normalized_pc, raw_ip, 0)?; + + if depth == 1 { + let status = + self.status_or_offsets_unavailable(BacktraceStatus::Truncated, offsets_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_status_depth_one", + )?; + return Ok(()); + } + + let row = self + .compact_unwind_row_for_backtrace(&compile_ctx.module_path, compile_ctx.pc_address) + .and_then(|row| backtrace_unwind_row_from_compact(&row)); + let initial_status = if row.is_some() { + BacktraceStatus::ReadError + } else if self.process_analyzer.is_some() { + BacktraceStatus::UnsupportedCfi + } else { + BacktraceStatus::DwarfUnavailable + }; + let status = self.status_or_offsets_unavailable(initial_status, offsets_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_initial_status", + )?; + + let Some(row) = row else { + return Ok(()); + }; + + let i64_type = self.context.i64_type(); + let ip_ptr = self.build_entry_alloca(i64_type, "bt_state_ip")?; + let rsp_ptr = self.build_entry_alloca(i64_type, "bt_state_rsp")?; + let rbp_ptr = self.build_entry_alloca(i64_type, "bt_state_rbp")?; + let scratch = self.allocate_backtrace_scratch()?; + self.builder + .build_store(ip_ptr, raw_ip) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let initial_rsp = self.load_dwarf_register_i64(X86_64_DWARF_RSP, pt_regs)?; + let initial_rbp = self.load_dwarf_register_i64(X86_64_DWARF_RBP, pt_regs)?; + self.builder + .build_store(rsp_ptr, initial_rsp) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rbp_ptr, initial_rbp) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + let current_fn = self.current_function("generate DWARF backtrace")?; + let done = self.context.append_basic_block(current_fn, "bt_done"); + + let runtime_row = self.runtime_row_from_static(row); + let state = BtRegisterState { + ip: self.load_i64(ip_ptr, "bt_initial_current_ip")?, + rsp: self.load_i64(rsp_ptr, "bt_initial_current_rsp")?, + rbp: self.load_i64(rbp_ptr, "bt_initial_current_rbp")?, + }; + let next = self.recover_next_frame_from_runtime_row(&runtime_row, state, &scratch)?; + let validation = self.validate_backtrace_next_frame(state, next)?; + let initial_store_block = self + .context + .append_basic_block(current_fn, "bt_initial_store_frame"); + let initial_stop_block = self + .context + .append_basic_block(current_fn, "bt_initial_stop"); + self.builder + .build_conditional_branch(validation.valid, initial_store_block, initial_stop_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(initial_stop_block); + let stop_status = self.status_for_backtrace_stop( + validation.complete, + validation.error_code, + offsets_found, + )?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + stop_status, + "bt_initial_stop_status", + )?; + self.store_u16_value( + inst_buffer, + data_base + BACKTRACE_DATA_ERROR_CODE_OFFSET, + validation.error_code, + "bt_initial_stop_error_code", + )?; + self.builder + .build_unconditional_branch(done) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(initial_store_block); + let next_pc = self.normalized_pc_from_raw(next.ip, module_bias, offsets_found)?; + self.store_backtrace_frame(inst_buffer, 1, module_cookie, next_pc, next.ip, 0)?; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, + 2, + "bt_initial_frame_count", + )?; + let status = if depth == 2 { + BacktraceStatus::Truncated + } else { + BacktraceStatus::ReadError + }; + let status = self.status_or_offsets_unavailable(status, offsets_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_initial_status_after_frame", + )?; + self.builder + .build_store(ip_ptr, next.ip) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rsp_ptr, next.rsp) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rbp_ptr, next.rbp) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + if depth == 2 { self.builder - .build_gep( - self.context.i8_type(), + .build_unconditional_branch(done) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + } else if self.backtrace_unwind_rows.is_empty() { + let status = + self.status_or_offsets_unavailable(BacktraceStatus::UnsupportedCfi, offsets_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_status_no_rows", + )?; + self.builder + .build_unconditional_branch(done) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + } else { + let inline_depth = depth.min(BPF_INLINE_BACKTRACE_FRAME_LIMIT); + for frame_index in 2..inline_depth { + let current_ip = self.load_i64(ip_ptr, "bt_lookup_ip")?; + let lookup_raw = self.add_signed_offset(current_ip, -1, "bt_lookup_raw")?; + let lookup_pc = + self.backtrace_lookup_pc_from_raw(lookup_raw, module_bias, offsets_found)?; + let runtime_row = self.lookup_backtrace_unwind_row(lookup_pc, &scratch.row)?; + let found_block = self.context.append_basic_block(current_fn, "bt_row_found"); + let missing_block = self + .context + .append_basic_block(current_fn, "bt_row_missing"); + self.builder + .build_conditional_branch(runtime_row.found, found_block, missing_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(missing_block); + let status = self.status_or_offsets_unavailable( + BacktraceStatus::UnsupportedCfi, + offsets_found, + )?; + self.store_u8_value( inst_buffer, - &[self.context.i32_type().const_int( - std::mem::offset_of!(InstructionHeader, inst_type) as u64, - false, - )], - "bt_inst_type_ptr", - ) - .map_err(|e| CodeGenError::LLVMError(format!("Failed to get inst_type GEP: {e}")))? + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_status_missing_row", + )?; + self.builder + .build_unconditional_branch(done) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(found_block); + let state = BtRegisterState { + ip: self.load_i64(ip_ptr, "bt_current_ip")?, + rsp: self.load_i64(rsp_ptr, "bt_current_rsp")?, + rbp: self.load_i64(rbp_ptr, "bt_current_rbp")?, + }; + let next = + self.recover_next_frame_from_runtime_row(&runtime_row, state, &scratch)?; + let validation = self.validate_backtrace_next_frame(state, next)?; + let store_block = self + .context + .append_basic_block(current_fn, "bt_store_frame"); + let stop_block = self.context.append_basic_block(current_fn, "bt_stop"); + self.builder + .build_conditional_branch(validation.valid, store_block, stop_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(stop_block); + let stop_status = self.status_for_backtrace_stop( + validation.complete, + validation.error_code, + offsets_found, + )?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + stop_status, + "bt_status_stop", + )?; + self.store_u16_value( + inst_buffer, + data_base + BACKTRACE_DATA_ERROR_CODE_OFFSET, + validation.error_code, + "bt_error_code_stop", + )?; + self.builder + .build_unconditional_branch(done) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(store_block); + let next_pc = self.normalized_pc_from_raw(next.ip, module_bias, offsets_found)?; + self.store_backtrace_frame( + inst_buffer, + frame_index as usize, + module_cookie, + next_pc, + next.ip, + 0, + )?; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, + frame_index + 1, + "bt_frame_count", + )?; + let next_status = if frame_index + 1 == inline_depth { + BacktraceStatus::Truncated + } else { + BacktraceStatus::ReadError + }; + let next_status = self.status_or_offsets_unavailable(next_status, offsets_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + next_status, + "bt_status_after_frame", + )?; + self.builder + .build_store(ip_ptr, next.ip) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rsp_ptr, next.rsp) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rbp_ptr, next.rbp) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + if frame_index + 1 == inline_depth { + self.builder + .build_unconditional_branch(done) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + } + } + } + + self.builder.position_at_end(done); + Ok(()) + } + + fn generate_tail_call_backtrace_instruction( + &mut self, + stmt: &BacktraceStatement, + ) -> Result<()> { + let depth = self + .compile_options + .backtrace_depth + .clamp(1, crate::MAX_BACKTRACE_DEPTH); + let flags = backtrace_flags(stmt); + info!( + "Generating tail-call Backtrace instruction: depth={}", + depth + ); + + let payload_size = + BACKTRACE_DATA_SIZE + depth as usize * std::mem::size_of::(); + let instruction_size = INSTRUCTION_HEADER_SIZE + payload_size; + let offset_ptr = self.event_offset_alloca.ok_or_else(|| { + CodeGenError::LLVMError("event_offset not allocated in entry block".to_string()) + })?; + let inst_offset = self + .builder + .build_load(self.context.i32_type(), offset_ptr, "bt_tail_inst_offset") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let inst_buffer = self + .reserve_instruction_region_or_return_zero(instruction_size as u64)? + .into_value_after_runtime_returns(); + + self.store_u8_const( + inst_buffer, + std::mem::offset_of!(InstructionHeader, inst_type), + InstructionType::Backtrace as u8, + "bt_inst_type", + )?; + self.store_u16_const( + inst_buffer, + std::mem::offset_of!(InstructionHeader, data_length), + payload_size as u16, + "bt_data_length", + )?; + + let data_base = INSTRUCTION_HEADER_SIZE; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_REQUESTED_DEPTH_OFFSET, + depth, + "bt_requested_depth", + )?; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, + 1, + "bt_frame_count", + )?; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_FLAGS_OFFSET, + flags, + "bt_flags", + )?; + self.store_u16_const( + inst_buffer, + data_base + BACKTRACE_DATA_ERROR_CODE_OFFSET, + 0, + "bt_error_code", + )?; + + let Some(compile_ctx) = self.current_compile_time_context.clone() else { + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, + 0, + "bt_frame_count_no_context", + )?; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + BacktraceStatus::DwarfUnavailable as u8, + "bt_status_no_context", + )?; + return Ok(()); + }; + + let module_cookie = self.cookie_for_module_or_fallback(&compile_ctx.module_path); + let pt_regs = self.get_pt_regs_parameter()?; + let raw_ip = self.load_dwarf_register_i64(X86_64_DWARF_RIP, pt_regs)?; + let initial_rsp = self.load_dwarf_register_i64(X86_64_DWARF_RSP, pt_regs)?; + let initial_rbp = self.load_dwarf_register_i64(X86_64_DWARF_RBP, pt_regs)?; + let (module_bias, offsets_found) = self.generate_runtime_address_from_offsets( + self.context.i64_type().const_zero(), + 0, + module_cookie, + )?; + let normalized_pc = self.normalized_pc_from_raw(raw_ip, module_bias, offsets_found)?; + + self.store_backtrace_frame(inst_buffer, 0, module_cookie, normalized_pc, raw_ip, 0)?; + + if depth == 1 { + let status = + self.status_or_offsets_unavailable(BacktraceStatus::Truncated, offsets_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_status_depth_one", + )?; + return Ok(()); + } + + if self.backtrace_unwind_rows.is_empty() { + let status = + self.status_or_offsets_unavailable(BacktraceStatus::UnsupportedCfi, offsets_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_status_no_rows", + )?; + return Ok(()); + } + + let row = self + .compact_unwind_row_for_backtrace(&compile_ctx.module_path, compile_ctx.pc_address) + .and_then(|row| backtrace_unwind_row_from_compact(&row)); + let Some(row) = row else { + let status = + self.status_or_offsets_unavailable(BacktraceStatus::UnsupportedCfi, offsets_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_status_no_initial_row", + )?; + return Ok(()); + }; + + let status = + self.status_or_offsets_unavailable(BacktraceStatus::ReadError, offsets_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_tail_initial_status", + )?; + + let scratch = self.allocate_backtrace_scratch()?; + let current_fn = self.current_function("initialize bt tail-call state")?; + let done_block = self + .context + .append_basic_block(current_fn, "bt_tail_state_done"); + let i64_type = self.context.i64_type(); + let ip_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_ip")?; + let rsp_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_rsp")?; + let rbp_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_rbp")?; + + let runtime_row = self.runtime_row_from_static(row); + let state = BtRegisterState { + ip: raw_ip, + rsp: initial_rsp, + rbp: initial_rbp, }; - let inst_type_val = self + let next = self.recover_next_frame_from_runtime_row(&runtime_row, state, &scratch)?; + let validation = self.validate_backtrace_next_frame(state, next)?; + let initial_store_block = self + .context + .append_basic_block(current_fn, "bt_tail_initial_store_frame"); + let initial_stop_block = self .context - .i8_type() - .const_int(InstructionType::Backtrace as u64, false); + .append_basic_block(current_fn, "bt_tail_initial_stop"); self.builder - .build_store(inst_type_ptr, inst_type_val) - .map_err(|e| CodeGenError::LLVMError(format!("Failed to store inst_type: {e}")))?; + .build_conditional_branch(validation.valid, initial_store_block, initial_stop_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - // Write InstructionHeader.data_length (u16) - // SAFETY: inst_buffer points at a reserved instruction region and the - // data_length offset is within InstructionHeader. - let data_length_ptr = unsafe { + self.builder.position_at_end(initial_stop_block); + let stop_status = self.status_for_backtrace_stop( + validation.complete, + validation.error_code, + offsets_found, + )?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + stop_status, + "bt_tail_initial_stop_status", + )?; + self.store_u16_value( + inst_buffer, + data_base + BACKTRACE_DATA_ERROR_CODE_OFFSET, + validation.error_code, + "bt_tail_initial_stop_error_code", + )?; + self.builder + .build_unconditional_branch(done_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(initial_store_block); + let next_pc = self.normalized_pc_from_raw(next.ip, module_bias, offsets_found)?; + self.store_backtrace_frame(inst_buffer, 1, module_cookie, next_pc, next.ip, 0)?; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, + 2, + "bt_tail_initial_frame_count", + )?; + let status = if depth == 2 { + BacktraceStatus::Truncated + } else { + BacktraceStatus::ReadError + }; + let status = self.status_or_offsets_unavailable(status, offsets_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_tail_initial_status_after_frame", + )?; + self.builder + .build_store(ip_ptr, next.ip) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rsp_ptr, next.rsp) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rbp_ptr, next.rbp) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + if depth == 2 { self.builder - .build_gep( - self.context.i8_type(), - inst_buffer, - &[self.context.i32_type().const_int( - std::mem::offset_of!(InstructionHeader, data_length) as u64, - false, - )], - "bt_data_length_ptr", - ) - .map_err(|e| { - CodeGenError::LLVMError(format!("Failed to get data_length GEP: {e}")) - })? + .build_unconditional_branch(done_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder.position_at_end(done_block); + return Ok(()); + } + + let prefix_depth = depth.min(BPF_INLINE_BACKTRACE_FRAME_LIMIT); + for frame_index in 2..prefix_depth { + let current_ip = self.load_i64(ip_ptr, "bt_tail_prefix_lookup_ip")?; + let lookup_raw = self.add_signed_offset(current_ip, -1, "bt_tail_prefix_lookup_raw")?; + let lookup_pc = + self.backtrace_lookup_pc_from_raw(lookup_raw, module_bias, offsets_found)?; + let runtime_row = self.lookup_backtrace_unwind_row(lookup_pc, &scratch.row)?; + let found_block = self + .context + .append_basic_block(current_fn, "bt_tail_prefix_row_found"); + let missing_block = self + .context + .append_basic_block(current_fn, "bt_tail_prefix_row_missing"); + self.builder + .build_conditional_branch(runtime_row.found, found_block, missing_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(missing_block); + let status = + self.status_or_offsets_unavailable(BacktraceStatus::UnsupportedCfi, offsets_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_tail_prefix_status_missing_row", + )?; + self.builder + .build_unconditional_branch(done_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(found_block); + let state = BtRegisterState { + ip: self.load_i64(ip_ptr, "bt_tail_prefix_current_ip")?, + rsp: self.load_i64(rsp_ptr, "bt_tail_prefix_current_rsp")?, + rbp: self.load_i64(rbp_ptr, "bt_tail_prefix_current_rbp")?, + }; + let next = self.recover_next_frame_from_runtime_row(&runtime_row, state, &scratch)?; + let validation = self.validate_backtrace_next_frame(state, next)?; + let store_block = self + .context + .append_basic_block(current_fn, "bt_tail_prefix_store_frame"); + let stop_block = self + .context + .append_basic_block(current_fn, "bt_tail_prefix_stop"); + self.builder + .build_conditional_branch(validation.valid, store_block, stop_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(stop_block); + let stop_status = self.status_for_backtrace_stop( + validation.complete, + validation.error_code, + offsets_found, + )?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + stop_status, + "bt_tail_prefix_stop_status", + )?; + self.store_u16_value( + inst_buffer, + data_base + BACKTRACE_DATA_ERROR_CODE_OFFSET, + validation.error_code, + "bt_tail_prefix_stop_error_code", + )?; + self.builder + .build_unconditional_branch(done_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(store_block); + let next_pc = self.normalized_pc_from_raw(next.ip, module_bias, offsets_found)?; + self.store_backtrace_frame( + inst_buffer, + frame_index as usize, + module_cookie, + next_pc, + next.ip, + 0, + )?; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, + frame_index + 1, + "bt_tail_prefix_frame_count", + )?; + let status = if frame_index + 1 == depth { + BacktraceStatus::Truncated + } else { + BacktraceStatus::ReadError + }; + let status = self.status_or_offsets_unavailable(status, offsets_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_tail_prefix_status_after_frame", + )?; + self.builder + .build_store(ip_ptr, next.ip) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rsp_ptr, next.rsp) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rbp_ptr, next.rbp) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + if frame_index + 1 == depth { + self.builder + .build_unconditional_branch(done_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + } + } + + if prefix_depth == depth { + self.builder.position_at_end(done_block); + return Ok(()); + } + + let tail_slot = self.next_backtrace_tail_call_slot; + self.next_backtrace_tail_call_slot = self.next_backtrace_tail_call_slot.saturating_add(1); + if self.pending_backtrace_tail_call.is_none() { + let step_program_name = format!( + "{}_bt_step", + self.current_function("name bt tail-call step")? + .get_name() + .to_string_lossy() + ); + self.pending_backtrace_tail_call = + Some(crate::ebpf::context::PendingBacktraceTailCall { + step_program_name, + module_cookie, + depth, + instruction_size, + }); + } + + let tail_enabled_ptr = self.get_or_create_backtrace_tail_enabled_flag()?; + + let state_ptr = self.lookup_bt_state_ptr(tail_slot as u32)?; + let state_is_null = self + .builder + .build_is_null(state_ptr, "bt_state_is_null") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let init_block = self + .context + .append_basic_block(current_fn, "bt_tail_state_init"); + let null_block = self + .context + .append_basic_block(current_fn, "bt_tail_state_null"); + self.builder + .build_conditional_branch(state_is_null, null_block, init_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(null_block); + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + BacktraceStatus::InternalError as u8, + "bt_status_state_null", + )?; + self.builder + .build_store(tail_enabled_ptr, self.context.i8_type().const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_unconditional_branch(done_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(init_block); + let tail_ip = self.load_i64(ip_ptr, "bt_tail_state_prefix_ip")?; + let tail_rsp = self.load_i64(rsp_ptr, "bt_tail_state_prefix_rsp")?; + let tail_rbp = self.load_i64(rbp_ptr, "bt_tail_state_prefix_rbp")?; + self.store_state_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_CURRENT_IP_OFFSET, + tail_ip, + "bt_state_ip", + )?; + self.store_state_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_CURRENT_RSP_OFFSET, + tail_rsp, + "bt_state_rsp", + )?; + self.store_state_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_CURRENT_RBP_OFFSET, + tail_rbp, + "bt_state_rbp", + )?; + self.store_state_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, + module_bias, + "bt_state_module_bias", + )?; + self.store_u64_const( + state_ptr, + crate::BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET, + module_cookie, + "bt_state_module_cookie", + )?; + self.store_state_i32( + state_ptr, + crate::BACKTRACE_TAIL_STATE_INST_OFFSET_OFFSET, + inst_offset, + "bt_state_inst_offset", + )?; + self.store_state_i32( + state_ptr, + crate::BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, + self.context.i32_type().const_zero(), + "bt_state_event_size", + )?; + self.store_u8_const( + state_ptr, + crate::BACKTRACE_TAIL_STATE_FRAME_COUNT_OFFSET, + prefix_depth, + "bt_state_frame_count", + )?; + self.store_u8_const( + state_ptr, + crate::BACKTRACE_TAIL_STATE_REQUESTED_DEPTH_OFFSET, + depth, + "bt_state_requested_depth", + )?; + let offsets_found_u8 = self + .builder + .build_select( + offsets_found, + self.context.i8_type().const_int(1, false), + self.context.i8_type().const_zero(), + "bt_offsets_found_u8", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + self.store_u8_value( + state_ptr, + crate::BACKTRACE_TAIL_STATE_OFFSETS_FOUND_OFFSET, + offsets_found_u8, + "bt_state_offsets_found", + )?; + self.store_u8_const( + state_ptr, + crate::BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, + 1, + "bt_state_tail_calls", + )?; + self.store_u8_const( + state_ptr, + crate::BACKTRACE_TAIL_STATE_FLAGS_OFFSET, + flags, + "bt_state_flags", + )?; + self.store_u8_const( + state_ptr, + crate::BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, + tail_slot, + "bt_state_active_slot", + )?; + self.store_u16_const( + state_ptr, + crate::BACKTRACE_TAIL_STATE_ERROR_CODE_OFFSET, + BACKTRACE_ERROR_NONE, + "bt_state_error_code", + )?; + self.store_u8_const( + state_ptr, + crate::BACKTRACE_TAIL_STATE_NEXT_SLOT_OFFSET, + crate::BACKTRACE_TAIL_NO_NEXT_SLOT, + "bt_state_next_slot", + )?; + self.link_backtrace_tail_slot(tail_slot, offsets_found_u8, done_block)?; + + self.builder.position_at_end(done_block); + Ok(()) + } + + pub(crate) fn finish_event_after_instructions(&mut self) -> Result<()> { + let Some(plan) = self.pending_backtrace_tail_call.clone() else { + return self.emit_accumulated_event_output_from_stack_offset(); }; - let data_length_i16_ptr = self + + let main_block = self.current_insert_block("finish bt tail-call event")?; + let main_pm_key_alloca = self.pm_key_alloca; + self.generate_backtrace_tail_call_step_program(&plan)?; + self.pm_key_alloca = main_pm_key_alloca; + self.builder.position_at_end(main_block); + + let tail_enabled_ptr = self.get_or_create_backtrace_tail_enabled_flag()?; + let enabled_value = self .builder - .build_pointer_cast( - data_length_ptr, - self.context.ptr_type(AddressSpace::default()), - "bt_data_length_i16_ptr", + .build_load( + self.context.i8_type(), + tail_enabled_ptr, + "bt_tail_enabled_value", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let enabled = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + enabled_value, + self.context.i8_type().const_zero(), + "bt_tail_enabled", ) - .map_err(|e| CodeGenError::LLVMError(format!("Failed to cast data_length ptr: {e}")))?; - let dl_val = self + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let current_fn = self.current_function("finish bt tail-call event")?; + let tail_block = self .context - .i16_type() - .const_int(std::mem::size_of::() as u64, false); + .append_basic_block(current_fn, "bt_tail_dispatch"); + let output_block = self + .context + .append_basic_block(current_fn, "bt_tail_fallback_output"); self.builder - .build_store(data_length_i16_ptr, dl_val) - .map_err(|e| CodeGenError::LLVMError(format!("Failed to store data_length: {e}")))?; + .build_conditional_branch(enabled, tail_block, output_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - // Already accumulated; EndInstruction will send the whole event. Depth currently unused at BPF level. - Ok(()) + self.builder.position_at_end(tail_block); + let state0_ptr = self.lookup_bt_state_ptr(0)?; + let state0_is_null = self + .builder + .build_is_null(state0_ptr, "bt_tail_dispatch_state_null") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let state_ok_block = self + .context + .append_basic_block(current_fn, "bt_tail_dispatch_state_ok"); + self.builder + .build_conditional_branch(state0_is_null, output_block, state_ok_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(state_ok_block); + let active_slot = self.load_row_i8( + state0_ptr, + crate::BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, + "bt_tail_dispatch_active_slot", + )?; + let state_ptr = self.lookup_bt_state_ptr_dynamic(active_slot)?; + let state_is_null = self + .builder + .build_is_null(state_ptr, "bt_tail_dispatch_active_state_null") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let active_state_ok_block = self + .context + .append_basic_block(current_fn, "bt_tail_dispatch_active_state_ok"); + self.builder + .build_conditional_branch(state_is_null, output_block, active_state_ok_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(active_state_ok_block); + let event_size = self + .builder + .build_load( + self.context.i32_type(), + self.event_offset_alloca.ok_or_else(|| { + CodeGenError::LLVMError("event_offset not allocated in entry block".to_string()) + })?, + "bt_tail_event_size", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + self.store_state_i32( + state_ptr, + crate::BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, + event_size, + "bt_tail_state_event_size", + )?; + self.emit_bpf_tail_call(BPF_BACKTRACE_STEP_PROG_INDEX)?; + self.builder + .build_unconditional_branch(output_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(output_block); + self.emit_accumulated_event_output_from_stack_offset() + } + + fn generate_backtrace_tail_call_step_program( + &mut self, + plan: &crate::ebpf::context::PendingBacktraceTailCall, + ) -> Result<()> { + self.create_tail_call_function(&plan.step_program_name)?; + let current_fn = self.current_function("generate bt tail-call step")?; + let return_block = self + .context + .append_basic_block(current_fn, "bt_step_return"); + let state_ok_block = self + .context + .append_basic_block(current_fn, "bt_step_state_ok"); + let accum_ok_block = self + .context + .append_basic_block(current_fn, "bt_step_accum_ok"); + let bounds_ok_block = self + .context + .append_basic_block(current_fn, "bt_step_bounds_ok"); + let inst_bounds_ok_block = self + .context + .append_basic_block(current_fn, "bt_step_inst_bounds_ok"); + let finalize_block = self + .context + .append_basic_block(current_fn, "bt_step_finalize"); + + let state0_ptr = self.lookup_bt_state_ptr(0)?; + let state_is_null = self + .builder + .build_is_null(state0_ptr, "bt_step_state_null") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_conditional_branch(state_is_null, return_block, state_ok_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(state_ok_block); + let active_slot = self.load_row_i8( + state0_ptr, + crate::BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, + "bt_step_active_slot", + )?; + let state_ptr = self.lookup_bt_state_ptr_dynamic(active_slot)?; + let active_state_is_null = self + .builder + .build_is_null(state_ptr, "bt_step_active_state_null") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let active_state_ok_block = self + .context + .append_basic_block(current_fn, "bt_step_active_state_ok"); + self.builder + .build_conditional_branch(active_state_is_null, return_block, active_state_ok_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(active_state_ok_block); + let accum_buffer = self.lookup_percpu_value_ptr("event_accum_buffer", 0)?; + let accum_is_null = self + .builder + .build_is_null(accum_buffer, "bt_step_accum_null") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_conditional_branch(accum_is_null, return_block, accum_ok_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(accum_ok_block); + let inst_offset = self.load_state_i32( + state_ptr, + crate::BACKTRACE_TAIL_STATE_INST_OFFSET_OFFSET, + "bt_step_inst_offset", + )?; + let event_size = self.load_state_i32( + state_ptr, + crate::BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, + "bt_step_event_size", + )?; + let max_event_size = self + .context + .i32_type() + .const_int(self.compile_options.max_trace_event_size as u64, false); + let max_inst_offset = self.context.i32_type().const_int( + self.compile_options + .max_trace_event_size + .saturating_sub(plan.instruction_size as u32) as u64, + false, + ); + let inst_in_bounds = self + .builder + .build_int_compare( + inkwell::IntPredicate::ULE, + inst_offset, + max_inst_offset, + "bt_step_inst_offset_in_bounds", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_conditional_branch(inst_in_bounds, inst_bounds_ok_block, return_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(inst_bounds_ok_block); + let event_in_bounds = self + .builder + .build_int_compare( + inkwell::IntPredicate::ULE, + event_size, + max_event_size, + "bt_step_event_size_in_bounds", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_conditional_branch(event_in_bounds, bounds_ok_block, return_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(bounds_ok_block); + let inst_offset_i64 = self + .builder + .build_int_z_extend( + inst_offset, + self.context.i64_type(), + "bt_step_inst_offset_i64", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let inst_buffer = + self.dynamic_byte_gep(accum_buffer, inst_offset_i64, "bt_step_inst_buffer")?; + let scratch = self.allocate_backtrace_scratch()?; + for _ in 0..BPF_BACKTRACE_FRAMES_PER_TAIL_CALL { + self.generate_backtrace_tail_call_step_iteration( + plan.depth, + plan.module_cookie, + state_ptr, + inst_buffer, + &scratch, + finalize_block, + )?; + } + + let tail_calls = self.load_row_i8( + state_ptr, + crate::BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, + "bt_step_tail_calls", + )?; + let can_tail_call = self + .builder + .build_int_compare( + inkwell::IntPredicate::ULT, + tail_calls, + self.context + .i8_type() + .const_int(BPF_BACKTRACE_MAX_STEP_INVOCATIONS as u64, false), + "bt_step_can_tail_call", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let self_tail_block = self + .context + .append_basic_block(current_fn, "bt_step_self_tail"); + let tail_budget_done = self + .context + .append_basic_block(current_fn, "bt_step_tail_budget_done"); + self.builder + .build_conditional_branch(can_tail_call, self_tail_block, tail_budget_done) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(tail_budget_done); + self.store_tail_backtrace_status( + inst_buffer, + BacktraceStatus::Truncated, + BACKTRACE_ERROR_NONE, + )?; + self.builder + .build_unconditional_branch(finalize_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(self_tail_block); + let next_tail_calls = self + .builder + .build_int_add( + tail_calls, + self.context.i8_type().const_int(1, false), + "bt_step_next_tail_calls", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.store_u8_value( + state_ptr, + crate::BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, + next_tail_calls, + "bt_step_store_tail_calls", + )?; + self.emit_bpf_tail_call(BPF_BACKTRACE_STEP_PROG_INDEX)?; + self.store_tail_backtrace_status( + inst_buffer, + BacktraceStatus::InternalError, + BACKTRACE_ERROR_NONE, + )?; + self.builder + .build_unconditional_branch(finalize_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(finalize_block); + let next_slot = self.load_row_i8( + state_ptr, + crate::BACKTRACE_TAIL_STATE_NEXT_SLOT_OFFSET, + "bt_final_next_slot", + )?; + let has_next_slot = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + next_slot, + self.context + .i8_type() + .const_int(crate::BACKTRACE_TAIL_NO_NEXT_SLOT as u64, false), + "bt_final_has_next_slot", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let tail_calls = self.load_row_i8( + state_ptr, + crate::BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, + "bt_final_tail_calls", + )?; + let can_tail_call_next = self + .builder + .build_int_compare( + inkwell::IntPredicate::ULT, + tail_calls, + self.context + .i8_type() + .const_int(BPF_BACKTRACE_MAX_STEP_INVOCATIONS as u64, false), + "bt_final_can_tail_call_next", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let should_continue_next = self + .builder + .build_and( + has_next_slot, + can_tail_call_next, + "bt_final_should_continue_next_slot", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let next_slot_block = self + .context + .append_basic_block(current_fn, "bt_final_next_slot"); + let emit_block = self.context.append_basic_block(current_fn, "bt_final_emit"); + self.builder + .build_conditional_branch(should_continue_next, next_slot_block, emit_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(next_slot_block); + self.store_u8_value( + state0_ptr, + crate::BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, + next_slot, + "bt_store_active_slot", + )?; + let next_state_ptr = self.lookup_bt_state_ptr_dynamic(next_slot)?; + let next_state_is_null = self + .builder + .build_is_null(next_state_ptr, "bt_next_state_null") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let next_state_ok_block = self + .context + .append_basic_block(current_fn, "bt_next_state_ok"); + self.builder + .build_conditional_branch(next_state_is_null, emit_block, next_state_ok_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(next_state_ok_block); + self.store_state_i32( + next_state_ptr, + crate::BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, + event_size, + "bt_next_slot_event_size", + )?; + let next_tail_calls = self + .builder + .build_int_add( + tail_calls, + self.context.i8_type().const_int(1, false), + "bt_next_slot_tail_calls", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.store_u8_value( + next_state_ptr, + crate::BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, + next_tail_calls, + "bt_next_slot_store_tail_calls", + )?; + self.emit_bpf_tail_call(BPF_BACKTRACE_STEP_PROG_INDEX)?; + self.builder + .build_unconditional_branch(emit_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(emit_block); + self.emit_tail_final_event(state_ptr, accum_buffer)?; + self.builder + .build_unconditional_branch(return_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(return_block); + self.build_return_zero() + } + + fn generate_backtrace_tail_call_step_iteration( + &mut self, + depth: u8, + module_cookie: u64, + state_ptr: PointerValue<'ctx>, + inst_buffer: PointerValue<'ctx>, + scratch: &BtScratch<'ctx>, + finalize_block: BasicBlock<'ctx>, + ) -> Result<()> { + let current_fn = self.current_function("generate bt tail-call step iteration")?; + let depth_block = self + .context + .append_basic_block(current_fn, "bt_step_depth_done"); + let unwind_block = self + .context + .append_basic_block(current_fn, "bt_step_unwind"); + let frame_count = self.load_row_i8( + state_ptr, + crate::BACKTRACE_TAIL_STATE_FRAME_COUNT_OFFSET, + "bt_step_frame_count", + )?; + let depth_value = self.context.i8_type().const_int(depth as u64, false); + let at_depth = self + .builder + .build_int_compare( + inkwell::IntPredicate::UGE, + frame_count, + depth_value, + "bt_step_at_depth", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_conditional_branch(at_depth, depth_block, unwind_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(depth_block); + self.store_tail_backtrace_status( + inst_buffer, + BacktraceStatus::Truncated, + BACKTRACE_ERROR_NONE, + )?; + self.builder + .build_unconditional_branch(finalize_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(unwind_block); + let current_ip = self.load_row_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_CURRENT_IP_OFFSET, + "bt_step_current_ip", + )?; + let current_rsp = self.load_row_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_CURRENT_RSP_OFFSET, + "bt_step_current_rsp", + )?; + let current_rbp = self.load_row_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_CURRENT_RBP_OFFSET, + "bt_step_current_rbp", + )?; + let module_bias = self.load_row_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, + "bt_step_module_bias", + )?; + let offsets_found_u8 = self.load_row_i8( + state_ptr, + crate::BACKTRACE_TAIL_STATE_OFFSETS_FOUND_OFFSET, + "bt_step_offsets_found_u8", + )?; + let offsets_found = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + offsets_found_u8, + self.context.i8_type().const_zero(), + "bt_step_offsets_found", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let is_first_unwind = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + frame_count, + self.context.i8_type().const_int(1, false), + "bt_step_first_unwind", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let caller_lookup_ip = + self.add_signed_offset(current_ip, -1, "bt_step_caller_lookup_ip")?; + let lookup_raw = self + .builder + .build_select::, _>( + is_first_unwind, + current_ip.into(), + caller_lookup_ip.into(), + "bt_step_lookup_raw", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let lookup_pc = + self.backtrace_lookup_pc_from_raw(lookup_raw, module_bias, offsets_found)?; + let runtime_row = self.lookup_backtrace_unwind_row(lookup_pc, &scratch.row)?; + let row_found_block = self + .context + .append_basic_block(current_fn, "bt_step_row_found"); + let row_missing_block = self + .context + .append_basic_block(current_fn, "bt_step_row_missing"); + self.builder + .build_conditional_branch(runtime_row.found, row_found_block, row_missing_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(row_missing_block); + self.store_tail_backtrace_status( + inst_buffer, + BacktraceStatus::UnsupportedCfi, + BACKTRACE_ERROR_NONE, + )?; + self.builder + .build_unconditional_branch(finalize_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(row_found_block); + let state = BtRegisterState { + ip: current_ip, + rsp: current_rsp, + rbp: current_rbp, + }; + let next = self.recover_next_frame_from_runtime_row(&runtime_row, state, scratch)?; + let validation = self.validate_backtrace_next_frame(state, next)?; + let store_block = self + .context + .append_basic_block(current_fn, "bt_step_store_frame"); + let stop_block = self.context.append_basic_block(current_fn, "bt_step_stop"); + self.builder + .build_conditional_branch(validation.valid, store_block, stop_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(stop_block); + let stop_status = self.status_for_backtrace_stop( + validation.complete, + validation.error_code, + offsets_found, + )?; + self.store_u8_value( + inst_buffer, + INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_STATUS_OFFSET, + stop_status, + "bt_step_stop_status", + )?; + self.store_u16_value( + inst_buffer, + INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_ERROR_CODE_OFFSET, + validation.error_code, + "bt_step_stop_error", + )?; + self.builder + .build_unconditional_branch(finalize_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(store_block); + let next_pc = self.normalized_pc_from_raw(next.ip, module_bias, offsets_found)?; + self.store_backtrace_frame_dynamic( + inst_buffer, + frame_count, + depth.saturating_sub(1), + module_cookie, + next_pc, + next.ip, + )?; + let next_count = self + .builder + .build_int_add( + frame_count, + self.context.i8_type().const_int(1, false), + "bt_step_next_frame_count", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.store_u8_value( + state_ptr, + crate::BACKTRACE_TAIL_STATE_FRAME_COUNT_OFFSET, + next_count, + "bt_step_state_frame_count", + )?; + self.store_u8_value( + inst_buffer, + INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_FRAME_COUNT_OFFSET, + next_count, + "bt_step_inst_frame_count", + )?; + self.store_state_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_CURRENT_IP_OFFSET, + next.ip, + "bt_step_state_next_ip", + )?; + self.store_state_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_CURRENT_RSP_OFFSET, + next.rsp, + "bt_step_state_next_rsp", + )?; + self.store_state_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_CURRENT_RBP_OFFSET, + next.rbp, + "bt_step_state_next_rbp", + )?; + + let reached_depth = self + .builder + .build_int_compare( + inkwell::IntPredicate::UGE, + next_count, + depth_value, + "bt_step_reached_depth", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let reached_depth_block = self + .context + .append_basic_block(current_fn, "bt_step_reached_depth"); + let continue_block = self + .context + .append_basic_block(current_fn, "bt_step_continue"); + self.builder + .build_conditional_branch(reached_depth, reached_depth_block, continue_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(reached_depth_block); + self.store_tail_backtrace_status( + inst_buffer, + BacktraceStatus::Truncated, + BACKTRACE_ERROR_NONE, + )?; + self.builder + .build_unconditional_branch(finalize_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(continue_block); + Ok(()) + } + + fn emit_bpf_tail_call(&mut self, index: u32) -> Result<()> { + let ctx = self.get_pt_regs_parameter()?; + let prog_array = self.lookup_bt_prog_array_ptr()?; + let args = [ + ctx.into(), + prog_array.into(), + self.context + .i32_type() + .const_int(index as u64, false) + .into(), + ]; + let _ = self.create_bpf_helper_call( + BPF_FUNC_tail_call as u64, + &args, + self.context.i64_type().into(), + "bt_bpf_tail_call", + )?; + Ok(()) + } + + fn store_tail_backtrace_status( + &self, + inst_buffer: PointerValue<'ctx>, + status: BacktraceStatus, + error_code: u16, + ) -> Result<()> { + self.store_u8_const( + inst_buffer, + INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_STATUS_OFFSET, + status as u8, + "bt_tail_status", + )?; + self.store_u16_const( + inst_buffer, + INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_ERROR_CODE_OFFSET, + error_code, + "bt_tail_error_code", + ) + } + + fn emit_tail_final_event( + &mut self, + state_ptr: PointerValue<'ctx>, + accum_buffer: PointerValue<'ctx>, + ) -> Result<()> { + let event_size = self.load_state_i32( + state_ptr, + crate::BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, + "bt_final_event_size", + )?; + self.emit_accumulated_event_output(accum_buffer, event_size) + } + + fn build_return_zero(&mut self) -> Result<()> { + self.builder + .build_return(Some(&self.context.i32_type().const_zero())) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(()) + } + + fn compact_unwind_row_for_backtrace( + &self, + module_path: &str, + pc: u64, + ) -> Option { + let analyzer = self.process_analyzer?; + let module_address = ModuleAddress::new(PathBuf::from(module_path), pc); + let ctx = analyzer.resolve_pc(&module_address).ok()?; + analyzer.compact_unwind_row_for_context(&ctx).ok().flatten() + } + + fn runtime_row_from_static( + &self, + row: ghostscope_protocol::BacktraceUnwindRow, + ) -> RuntimeBtUnwindRow<'ctx> { + let i8_type = self.context.i8_type(); + let i16_type = self.context.i16_type(); + let i64_type = self.context.i64_type(); + RuntimeBtUnwindRow { + found: self.context.bool_type().const_int(1, false), + cfa_register: i16_type.const_int(row.cfa_register as u64, false), + cfa_offset: i64_type.const_int(row.cfa_offset as u64, true), + ra_kind: i8_type.const_int(row.ra_kind as u64, false), + ra_register: i16_type.const_int(row.ra_register as u64, false), + ra_offset: i64_type.const_int(row.ra_offset as u64, true), + rbp_kind: i8_type.const_int(row.rbp_kind as u64, false), + rbp_register: i16_type.const_int(row.rbp_register as u64, false), + rbp_offset: i64_type.const_int(row.rbp_offset as u64, true), + } + } + + fn allocate_backtrace_scratch(&self) -> Result> { + let i16_type = self.context.i16_type(); + let i32_type = self.context.i32_type(); + let i64_type = self.context.i64_type(); + + Ok(BtScratch { + row: RuntimeBtRowScratch { + found_ptr: self.build_entry_alloca(i32_type, "bt_row_found")?, + cfa_register_ptr: self.build_entry_alloca(i16_type, "bt_row_cfa_register")?, + cfa_offset_ptr: self.build_entry_alloca(i64_type, "bt_row_cfa_offset")?, + ra_kind_ptr: self.build_entry_alloca(self.context.i8_type(), "bt_row_ra_kind")?, + ra_register_ptr: self.build_entry_alloca(i16_type, "bt_row_ra_register")?, + ra_offset_ptr: self.build_entry_alloca(i64_type, "bt_row_ra_offset")?, + rbp_kind_ptr: self.build_entry_alloca(self.context.i8_type(), "bt_row_rbp_kind")?, + rbp_register_ptr: self.build_entry_alloca(i16_type, "bt_row_rbp_register")?, + rbp_offset_ptr: self.build_entry_alloca(i64_type, "bt_row_rbp_offset")?, + }, + next_rbp_ptr: self.build_entry_alloca(i64_type, "bt_next_rbp")?, + next_error_code_ptr: self.build_entry_alloca(i16_type, "bt_next_error_code")?, + }) + } + + fn lookup_backtrace_unwind_row( + &mut self, + normalized_pc: IntValue<'ctx>, + scratch: &RuntimeBtRowScratch<'ctx>, + ) -> Result> { + let row_count = self.backtrace_unwind_rows.len(); + let i16_type = self.context.i16_type(); + let i32_type = self.context.i32_type(); + let i64_type = self.context.i64_type(); + let i8_type = self.context.i8_type(); + let sentinel = i32_type.const_int(row_count as u64, false); + + self.builder + .build_store(scratch.found_ptr, sentinel) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.cfa_register_ptr, i16_type.const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.cfa_offset_ptr, i64_type.const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.ra_kind_ptr, i8_type.const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.ra_register_ptr, i16_type.const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.ra_offset_ptr, i64_type.const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.rbp_kind_ptr, i8_type.const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.rbp_register_ptr, i16_type.const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.rbp_offset_ptr, i64_type.const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + let current_fn = self.current_function("lookup bt unwind row")?; + let return_block = self + .context + .append_basic_block(current_fn, "bt_row_lookup_return"); + if row_count == 0 { + self.builder + .build_unconditional_branch(return_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + } else { + let lo_ptr = self.build_entry_alloca(i32_type, "bt_row_lo")?; + let hi_ptr = self.build_entry_alloca(i32_type, "bt_row_hi")?; + self.builder + .build_store(lo_ptr, i32_type.const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(hi_ptr, i32_type.const_int(row_count as u64, false)) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.emit_backtrace_row_runtime_binary_search( + normalized_pc, + scratch, + lo_ptr, + hi_ptr, + row_count, + return_block, + )?; + } + self.builder.position_at_end(return_block); + let final_found_idx = self.load_i32(scratch.found_ptr, "bt_final_found_idx")?; + let found = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + final_found_idx, + sentinel, + "bt_final_row_found", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(RuntimeBtUnwindRow { + found, + cfa_register: self.load_i16(scratch.cfa_register_ptr, "bt_final_cfa_reg")?, + cfa_offset: self.load_i64(scratch.cfa_offset_ptr, "bt_final_cfa_off")?, + ra_kind: self.load_i8(scratch.ra_kind_ptr, "bt_final_ra_kind")?, + ra_register: self.load_i16(scratch.ra_register_ptr, "bt_final_ra_reg")?, + ra_offset: self.load_i64(scratch.ra_offset_ptr, "bt_final_ra_off")?, + rbp_kind: self.load_i8(scratch.rbp_kind_ptr, "bt_final_rbp_kind")?, + rbp_register: self.load_i16(scratch.rbp_register_ptr, "bt_final_rbp_reg")?, + rbp_offset: self.load_i64(scratch.rbp_offset_ptr, "bt_final_rbp_off")?, + }) + } + + fn emit_backtrace_row_runtime_binary_search( + &mut self, + normalized_pc: IntValue<'ctx>, + scratch: &RuntimeBtRowScratch<'ctx>, + lo_ptr: PointerValue<'ctx>, + hi_ptr: PointerValue<'ctx>, + row_count: usize, + return_block: BasicBlock<'ctx>, + ) -> Result<()> { + let current_fn = self.current_function("emit bt row lookup tree")?; + let i32_type = self.context.i32_type(); + let sentinel = i32_type.const_int(row_count as u64, false); + let max_steps = backtrace_row_binary_search_steps(row_count); + + for _ in 0..max_steps { + let found_idx = self.load_i32(scratch.found_ptr, "bt_lookup_found_idx")?; + let not_found = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + found_idx, + sentinel, + "bt_lookup_not_found", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let lo = self.load_i32(lo_ptr, "bt_lookup_lo")?; + let hi = self.load_i32(hi_ptr, "bt_lookup_hi")?; + let range_active = self + .builder + .build_int_compare(inkwell::IntPredicate::ULT, lo, hi, "bt_lookup_range_active") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let should_search = self + .builder + .build_and(not_found, range_active, "bt_lookup_should_search") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + let search_block = self + .context + .append_basic_block(current_fn, "bt_lookup_search"); + let skip_block = self + .context + .append_basic_block(current_fn, "bt_lookup_skip"); + let after_block = self + .context + .append_basic_block(current_fn, "bt_lookup_after"); + self.builder + .build_conditional_branch(should_search, search_block, skip_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(skip_block); + self.builder + .build_unconditional_branch(after_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(search_block); + let lo_plus_hi = self + .builder + .build_int_add(lo, hi, "bt_lookup_lo_plus_hi") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let mid = self + .builder + .build_right_shift( + lo_plus_hi, + i32_type.const_int(1, false), + false, + "bt_lookup_mid", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let row_ptr = self.lookup_bt_unwind_row_ptr(mid)?; + let row_is_null = self + .builder + .build_is_null(row_ptr, "bt_lookup_row_is_null") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let row_null_block = self + .context + .append_basic_block(current_fn, "bt_lookup_row_null"); + let row_load_block = self + .context + .append_basic_block(current_fn, "bt_lookup_row_load"); + self.builder + .build_conditional_branch(row_is_null, row_null_block, row_load_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(row_null_block); + self.builder + .build_store(lo_ptr, hi) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_unconditional_branch(after_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(row_load_block); + let pc_start = self.load_row_i64( + row_ptr, + crate::BACKTRACE_UNWIND_ROW_PC_START_OFFSET, + "bt_lookup_row_pc_start", + )?; + let pc_end = self.load_row_i64( + row_ptr, + crate::BACKTRACE_UNWIND_ROW_PC_END_OFFSET, + "bt_lookup_row_pc_end", + )?; + let before = self + .builder + .build_int_compare( + inkwell::IntPredicate::ULT, + normalized_pc, + pc_start, + "bt_lookup_pc_before", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let before_block = self + .context + .append_basic_block(current_fn, "bt_lookup_before"); + let not_before_block = self + .context + .append_basic_block(current_fn, "bt_lookup_not_before"); + self.builder + .build_conditional_branch(before, before_block, not_before_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(before_block); + self.builder + .build_store(hi_ptr, mid) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_unconditional_branch(after_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(not_before_block); + let after = self + .builder + .build_int_compare( + inkwell::IntPredicate::UGE, + normalized_pc, + pc_end, + "bt_lookup_pc_after", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let after_range_block = self + .context + .append_basic_block(current_fn, "bt_lookup_after_range"); + let match_block = self + .context + .append_basic_block(current_fn, "bt_lookup_match"); + self.builder + .build_conditional_branch(after, after_range_block, match_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(after_range_block); + let mid_plus_one = self + .builder + .build_int_add(mid, i32_type.const_int(1, false), "bt_lookup_mid_plus_one") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(lo_ptr, mid_plus_one) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_unconditional_branch(after_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(match_block); + self.store_backtrace_unwind_row_from_ptr(row_ptr, mid, scratch)?; + self.builder + .build_unconditional_branch(after_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(after_block); + } + + self.builder + .build_unconditional_branch(return_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(()) + } + + fn store_backtrace_unwind_row_from_ptr( + &self, + row_ptr: PointerValue<'ctx>, + row_index: IntValue<'ctx>, + scratch: &RuntimeBtRowScratch<'ctx>, + ) -> Result<()> { + self.builder + .build_store(scratch.found_ptr, row_index) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let cfa_register = self.load_row_i16( + row_ptr, + crate::BACKTRACE_UNWIND_ROW_CFA_REGISTER_OFFSET, + "bt_tree_row_cfa_reg", + )?; + let cfa_offset = self.load_row_i64( + row_ptr, + crate::BACKTRACE_UNWIND_ROW_CFA_OFFSET_OFFSET, + "bt_tree_row_cfa_off", + )?; + let ra_kind = self.load_row_i8( + row_ptr, + crate::BACKTRACE_UNWIND_ROW_RA_KIND_OFFSET, + "bt_tree_row_ra_kind", + )?; + let ra_register = self.load_row_i16( + row_ptr, + crate::BACKTRACE_UNWIND_ROW_RA_REGISTER_OFFSET, + "bt_tree_row_ra_reg", + )?; + let ra_offset = self.load_row_i64( + row_ptr, + crate::BACKTRACE_UNWIND_ROW_RA_OFFSET_OFFSET, + "bt_tree_row_ra_off", + )?; + let rbp_kind = self.load_row_i8( + row_ptr, + crate::BACKTRACE_UNWIND_ROW_RBP_KIND_OFFSET, + "bt_tree_row_rbp_kind", + )?; + let rbp_register = self.load_row_i16( + row_ptr, + crate::BACKTRACE_UNWIND_ROW_RBP_REGISTER_OFFSET, + "bt_tree_row_rbp_reg", + )?; + let rbp_offset = self.load_row_i64( + row_ptr, + crate::BACKTRACE_UNWIND_ROW_RBP_OFFSET_OFFSET, + "bt_tree_row_rbp_off", + )?; + self.builder + .build_store(scratch.cfa_register_ptr, cfa_register) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.cfa_offset_ptr, cfa_offset) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.ra_kind_ptr, ra_kind) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.ra_register_ptr, ra_register) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.ra_offset_ptr, ra_offset) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.rbp_kind_ptr, rbp_kind) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.rbp_register_ptr, rbp_register) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.rbp_offset_ptr, rbp_offset) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(()) + } + + fn recover_next_frame_from_runtime_row( + &mut self, + row: &RuntimeBtUnwindRow<'ctx>, + state: BtRegisterState<'ctx>, + scratch: &BtScratch<'ctx>, + ) -> Result> { + let cfa_base = self.select_register_state(row.cfa_register, state, "bt_cfa_base")?; + let cfa = self + .builder + .build_int_add(cfa_base, row.cfa_offset, "bt_runtime_cfa") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + let ra_addr = self + .builder + .build_int_add(cfa, row.ra_offset, "bt_runtime_ra_addr") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store( + scratch.next_error_code_ptr, + self.context + .i16_type() + .const_int(BACKTRACE_ERROR_NONE as u64, false), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let (ra_from_memory, ra_read_failed) = self.generate_memory_read_with_fail_flag( + RuntimeAddress::available(ra_addr, self.context), + MemoryAccessSize::U64, + "bt_ra_read", + )?; + let ra_from_memory = ra_from_memory.into_int_value(); + let ra_uses_memory = self.is_recovery_kind( + row.ra_kind, + crate::BACKTRACE_RECOVERY_AT_CFA_OFFSET, + "bt_ra_at_kind", + )?; + let ra_read_failed = self + .builder + .build_and(ra_read_failed, ra_uses_memory, "bt_ra_read_failed") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.store_backtrace_error_code_if( + scratch.next_error_code_ptr, + ra_read_failed, + BACKTRACE_ERROR_RETURN_ADDRESS_READ, + "bt_ra_error_code", + )?; + let ra_from_val = self + .builder + .build_int_add(cfa, row.ra_offset, "bt_runtime_ra_val") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let ra_from_register = self.select_register_state(row.ra_register, state, "bt_ra_reg")?; + let ra_is_val = self.is_recovery_kind( + row.ra_kind, + crate::BACKTRACE_RECOVERY_VAL_CFA_OFFSET, + "bt_ra_val_kind", + )?; + let ra_is_register = self.is_recovery_kind( + row.ra_kind, + crate::BACKTRACE_RECOVERY_REGISTER, + "bt_ra_reg_kind", + )?; + let ra_is_same = self.is_recovery_kind( + row.ra_kind, + crate::BACKTRACE_RECOVERY_SAME_VALUE, + "bt_ra_same_kind", + )?; + let ra_is_register_like = self + .builder + .build_or(ra_is_register, ra_is_same, "bt_ra_register_like") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let ra_value_or_memory = self + .builder + .build_select::, _>( + ra_is_val, + ra_from_val.into(), + ra_from_memory.into(), + "bt_ra_val_or_memory", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let next_ip = self + .builder + .build_select::, _>( + ra_is_register_like, + ra_from_register.into(), + ra_value_or_memory.into(), + "bt_next_ip", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let next_rbp = self.recover_rbp_from_runtime_row( + row, + cfa, + state, + scratch.next_rbp_ptr, + scratch.next_error_code_ptr, + )?; + let error_code = self.load_i16(scratch.next_error_code_ptr, "bt_next_error_code_value")?; + + Ok(BtNextFrame { + ip: next_ip, + rsp: cfa, + rbp: next_rbp, + error_code, + }) + } + + fn validate_backtrace_next_frame( + &self, + state: BtRegisterState<'ctx>, + next: BtNextFrame<'ctx>, + ) -> Result> { + let i64_type = self.context.i64_type(); + let i16_type = self.context.i16_type(); + let zero = i64_type.const_zero(); + let zero_i16 = i16_type.const_zero(); + let min_user_ip = i64_type.const_int(0x1000, false); + let high_byte_mask = i64_type.const_int(0xff00_0000_0000_0000, false); + + let no_read_error = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + next.error_code, + zero_i16, + "bt_no_read_error", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let ip_high_enough = self + .builder + .build_int_compare( + inkwell::IntPredicate::UGE, + next.ip, + min_user_ip, + "bt_next_ip_user_min", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let ip_high_byte = self + .builder + .build_and(next.ip, high_byte_mask, "bt_next_ip_high_byte") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let ip_is_zero = self + .builder + .build_int_compare(inkwell::IntPredicate::EQ, next.ip, zero, "bt_next_ip_zero") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let ip_not_kernel_like = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + ip_high_byte, + zero, + "bt_next_ip_not_kernel_like", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let cfa_nonzero = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + next.rsp, + zero, + "bt_next_cfa_nonzero", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let cfa_changed = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + next.rsp, + state.rsp, + "bt_next_cfa_changed", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let ip_changed = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + next.ip, + state.ip, + "bt_next_ip_changed", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let cfa_progress = self + .builder + .build_or(cfa_changed, ip_changed, "bt_next_frame_progress") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + let ip_valid = self + .builder + .build_and(ip_high_enough, ip_not_kernel_like, "bt_next_ip_valid") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let cfa_valid = self + .builder + .build_and(cfa_nonzero, cfa_progress, "bt_next_cfa_valid") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let frame_shape_valid = self + .builder + .build_and(ip_valid, cfa_valid, "bt_next_frame_valid") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let valid = self + .builder + .build_and( + no_read_error, + frame_shape_valid, + "bt_next_frame_valid_no_read_error", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let complete = self + .builder + .build_and(no_read_error, ip_is_zero, "bt_next_frame_complete") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + let mut error_code = next.error_code; + error_code = self.select_backtrace_error_code_if( + error_code, + ip_high_enough, + BACKTRACE_ERROR_NEXT_IP_BELOW_USER, + "bt_next_ip_below_user_code", + )?; + error_code = self.select_backtrace_error_code_if( + error_code, + ip_not_kernel_like, + BACKTRACE_ERROR_NEXT_IP_KERNEL_LIKE, + "bt_next_ip_kernel_like_code", + )?; + error_code = self.select_backtrace_error_code_if( + error_code, + cfa_nonzero, + BACKTRACE_ERROR_NEXT_CFA_ZERO, + "bt_next_cfa_zero_code", + )?; + error_code = self.select_backtrace_error_code_if( + error_code, + cfa_progress, + BACKTRACE_ERROR_NEXT_CFA_NOT_ADVANCING, + "bt_next_cfa_not_advancing_code", + )?; + error_code = self + .builder + .build_select::, _>( + complete, + self.context + .i16_type() + .const_int(BACKTRACE_ERROR_NONE as u64, false) + .into(), + error_code.into(), + "bt_complete_error_code", + ) + .map(|value| value.into_int_value()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + Ok(BtFrameValidation { + valid, + complete, + error_code, + }) + } + + fn select_backtrace_error_code_if( + &self, + current: IntValue<'ctx>, + condition_ok: IntValue<'ctx>, + error_code: u16, + name: &str, + ) -> Result> { + let i16_type = self.context.i16_type(); + let current_is_none = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + current, + i16_type.const_int(BACKTRACE_ERROR_NONE as u64, false), + &format!("{name}_current_none"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let condition_failed = self + .builder + .build_not(condition_ok, &format!("{name}_condition_failed")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let should_set = self + .builder + .build_and( + current_is_none, + condition_failed, + &format!("{name}_should_set"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_select::, _>( + should_set, + i16_type.const_int(error_code as u64, false).into(), + current.into(), + name, + ) + .map(|value| value.into_int_value()) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + } + + fn store_backtrace_error_code_if( + &self, + error_code_ptr: PointerValue<'ctx>, + condition: IntValue<'ctx>, + error_code: u16, + name: &str, + ) -> Result<()> { + let current = self.load_i16(error_code_ptr, &format!("{name}_current"))?; + let i16_type = self.context.i16_type(); + let current_is_none = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + current, + i16_type.const_int(BACKTRACE_ERROR_NONE as u64, false), + &format!("{name}_current_none"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let should_set = self + .builder + .build_and(condition, current_is_none, &format!("{name}_should_set")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let next = self + .builder + .build_select::, _>( + should_set, + i16_type.const_int(error_code as u64, false).into(), + current.into(), + name, + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(error_code_ptr, next) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(()) + } + + fn recover_rbp_from_runtime_row( + &mut self, + row: &RuntimeBtUnwindRow<'ctx>, + cfa: IntValue<'ctx>, + state: BtRegisterState<'ctx>, + next_rbp_ptr: PointerValue<'ctx>, + next_error_code_ptr: PointerValue<'ctx>, + ) -> Result> { + let is_at = self.is_recovery_kind( + row.rbp_kind, + crate::BACKTRACE_RECOVERY_AT_CFA_OFFSET, + "bt_rbp_at_kind", + )?; + let cfa_uses_rbp = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + row.cfa_register, + self.context + .i16_type() + .const_int(X86_64_DWARF_RBP as u64, false), + "bt_rbp_cfa_uses_rbp", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let cfa_offset_is_frame_pointer_call_frame = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + row.cfa_offset, + self.context.i64_type().const_int(16, false), + "bt_rbp_cfa_offset_is_16", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let frame_pointer_call_frame = self + .builder + .build_and( + cfa_uses_rbp, + cfa_offset_is_frame_pointer_call_frame, + "bt_rbp_frame_pointer_call_frame", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let is_at = self + .builder + .build_or( + is_at, + frame_pointer_call_frame, + "bt_rbp_at_or_frame_pointer", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let rbp_offset = self + .builder + .build_select::, _>( + frame_pointer_call_frame, + self.context + .i64_type() + .const_int((-16i64) as u64, true) + .into(), + row.rbp_offset.into(), + "bt_rbp_effective_offset", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let current_fn = self.current_function("recover bt rbp")?; + let at_block = self.context.append_basic_block(current_fn, "bt_rbp_at"); + let non_at_block = self.context.append_basic_block(current_fn, "bt_rbp_non_at"); + let join_block = self.context.append_basic_block(current_fn, "bt_rbp_join"); + self.builder + .build_conditional_branch(is_at, at_block, non_at_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(at_block); + let rbp_addr = self + .builder + .build_int_add(cfa, rbp_offset, "bt_runtime_rbp_addr") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let (rbp_from_memory, rbp_read_failed) = self.generate_memory_read_with_fail_flag( + RuntimeAddress::available(rbp_addr, self.context), + MemoryAccessSize::U64, + "bt_rbp_read", + )?; + self.store_backtrace_error_code_if( + next_error_code_ptr, + rbp_read_failed, + BACKTRACE_ERROR_FRAME_POINTER_READ, + "bt_rbp_error_code", + )?; + self.builder + .build_store(next_rbp_ptr, rbp_from_memory.into_int_value()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_unconditional_branch(join_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(non_at_block); + let rbp_from_val = self + .builder + .build_int_add(cfa, row.rbp_offset, "bt_runtime_rbp_val") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let rbp_from_register = + self.select_register_state(row.rbp_register, state, "bt_rbp_reg")?; + let rbp_is_val = self.is_recovery_kind( + row.rbp_kind, + crate::BACKTRACE_RECOVERY_VAL_CFA_OFFSET, + "bt_rbp_val_kind", + )?; + let rbp_is_register = self.is_recovery_kind( + row.rbp_kind, + crate::BACKTRACE_RECOVERY_REGISTER, + "bt_rbp_reg_kind", + )?; + let rbp_is_same = self.is_recovery_kind( + row.rbp_kind, + crate::BACKTRACE_RECOVERY_SAME_VALUE, + "bt_rbp_same_kind", + )?; + let rbp_is_register_like = self + .builder + .build_or(rbp_is_register, rbp_is_same, "bt_rbp_register_like") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let rbp_value_or_current = self + .builder + .build_select::, _>( + rbp_is_val, + rbp_from_val.into(), + state.rbp.into(), + "bt_rbp_val_or_current", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let rbp_non_at = self + .builder + .build_select::, _>( + rbp_is_register_like, + rbp_from_register.into(), + rbp_value_or_current.into(), + "bt_rbp_non_at_value", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + self.builder + .build_store(next_rbp_ptr, rbp_non_at) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_unconditional_branch(join_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(join_block); + self.load_i64(next_rbp_ptr, "bt_next_rbp_value") + } + + fn select_register_state( + &self, + register: IntValue<'ctx>, + state: BtRegisterState<'ctx>, + name: &str, + ) -> Result> { + let is_rbp = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + register, + self.context + .i16_type() + .const_int(X86_64_DWARF_RBP as u64, false), + &format!("{name}_is_rbp"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let is_rip = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + register, + self.context + .i16_type() + .const_int(X86_64_DWARF_RIP as u64, false), + &format!("{name}_is_rip"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let rbp_or_rsp = self + .builder + .build_select::, _>( + is_rbp, + state.rbp.into(), + state.rsp.into(), + &format!("{name}_rbp_or_rsp"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + self.builder + .build_select::, _>( + is_rip, + state.ip.into(), + rbp_or_rsp.into(), + name, + ) + .map(|value| value.into_int_value()) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + } + + fn is_recovery_kind( + &self, + kind: IntValue<'ctx>, + expected: u8, + name: &str, + ) -> Result> { + self.builder + .build_int_compare( + inkwell::IntPredicate::EQ, + kind, + self.context.i8_type().const_int(expected as u64, false), + name, + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + } + + fn lookup_bt_unwind_row_ptr( + &mut self, + row_index: IntValue<'ctx>, + ) -> Result> { + let ptr_type = self.context.ptr_type(AddressSpace::default()); + let i32_type = self.context.i32_type(); + let map_global = self + .module + .get_global("bt_unwind_rows") + .ok_or_else(|| CodeGenError::LLVMError("bt_unwind_rows map not found".to_string()))?; + let map_ptr = self + .builder + .build_bit_cast( + map_global.as_pointer_value(), + ptr_type, + "bt_unwind_rows_map_ptr", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let key_alloca = self.pm_key_alloca.ok_or_else(|| { + CodeGenError::LLVMError("pm_key not allocated in entry block".to_string()) + })?; + let key_arr_ty = i32_type.array_type(4); + let zero = i32_type.const_zero(); + // SAFETY: pm_key_alloca is a [4 x i32] entry-block alloca and [0, 0] + // addresses its first element, which is sufficient for an Array u32 key. + let key_ptr = unsafe { + self.builder + .build_gep( + key_arr_ty, + key_alloca, + &[zero, zero], + "bt_unwind_row_key_ptr", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + }; + self.builder + .build_store(key_ptr, row_index) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let key_ptr = self + .builder + .build_bit_cast(key_ptr, ptr_type, "bt_unwind_row_key_void") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let result = self.create_bpf_helper_call( + BPF_FUNC_map_lookup_elem as u64, + &[map_ptr, key_ptr], + ptr_type.into(), + "bt_unwind_row_lookup", + )?; + match result { + BasicValueEnum::PointerValue(ptr) => Ok(ptr), + _ => Err(CodeGenError::LLVMError( + "bt_unwind_rows lookup did not return pointer".to_string(), + )), + } + } + + fn lookup_bt_state_ptr(&mut self, key_const: u32) -> Result> { + self.lookup_percpu_value_ptr("bt_state", key_const) + } + + fn lookup_bt_state_ptr_dynamic( + &mut self, + state_index: IntValue<'ctx>, + ) -> Result> { + let ptr_type = self.context.ptr_type(AddressSpace::default()); + let i32_type = self.context.i32_type(); + let map_global = self + .map_manager + .get_map(&self.module, "bt_state") + .map_err(|e| CodeGenError::LLVMError(format!("Map not found bt_state: {e}")))?; + let map_ptr = self + .builder + .build_bit_cast(map_global, ptr_type, "bt_state_map_ptr") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let key = match state_index + .get_type() + .get_bit_width() + .cmp(&i32_type.get_bit_width()) + { + std::cmp::Ordering::Equal => state_index, + std::cmp::Ordering::Less => self + .builder + .build_int_z_extend(state_index, i32_type, "bt_state_key_i32") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?, + std::cmp::Ordering::Greater => self + .builder + .build_int_truncate(state_index, i32_type, "bt_state_key_i32") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?, + }; + + let key_arr_ty = i32_type.array_type(4); + let key_alloca = self.pm_key_alloca.ok_or_else(|| { + CodeGenError::LLVMError("pm_key not allocated in entry block".to_string()) + })?; + let zero = i32_type.const_zero(); + // SAFETY: pm_key_alloca is a [4 x i32] entry-block alloca and [0, 0] + // addresses its first element, which is sufficient for an Array u32 key. + let key_ptr = unsafe { + self.builder + .build_gep(key_arr_ty, key_alloca, &[zero, zero], "bt_state_key_ptr") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + }; + self.builder + .build_store(key_ptr, key) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let key_ptr = self + .builder + .build_bit_cast(key_ptr, ptr_type, "bt_state_key_void") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let result = self.create_bpf_helper_call( + BPF_FUNC_map_lookup_elem as u64, + &[map_ptr, key_ptr], + ptr_type.into(), + "bt_state_lookup", + )?; + match result { + BasicValueEnum::PointerValue(ptr) => Ok(ptr), + _ => Err(CodeGenError::LLVMError( + "bt_state lookup did not return pointer".to_string(), + )), + } + } + + fn lookup_bt_prog_array_ptr(&mut self) -> Result> { + let ptr_type = self.context.ptr_type(AddressSpace::default()); + let map_global = self + .module + .get_global("bt_prog_array") + .ok_or_else(|| CodeGenError::LLVMError("bt_prog_array map not found".to_string()))?; + let map_ptr = self + .builder + .build_bit_cast(map_global.as_pointer_value(), ptr_type, "bt_prog_array_ptr") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + match map_ptr { + BasicValueEnum::PointerValue(ptr) => Ok(ptr), + _ => Err(CodeGenError::LLVMError( + "bt_prog_array cast did not return pointer".to_string(), + )), + } + } + + fn get_or_create_backtrace_tail_enabled_flag(&mut self) -> Result> { + if let Some(ptr) = self.backtrace_tail_enabled_alloca { + return Ok(ptr); + } + + let current_block = self.builder.get_insert_block().ok_or_else(|| { + CodeGenError::LLVMError("no current block for bt tail flag allocation".to_string()) + })?; + let current_fn = self.current_function("allocate bt tail flag")?; + let entry_block = current_fn.get_first_basic_block().ok_or_else(|| { + CodeGenError::LLVMError("no entry block for bt tail flag allocation".to_string()) + })?; + + if let Some(first_instruction) = entry_block.get_first_instruction() { + self.builder.position_before(&first_instruction); + } else { + self.builder.position_at_end(entry_block); + } + let alloca = self + .builder + .build_alloca(self.context.i8_type(), "bt_tail_enabled") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(alloca, self.context.i8_type().const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder.position_at_end(current_block); + self.backtrace_tail_enabled_alloca = Some(alloca); + Ok(alloca) + } + + fn get_or_create_backtrace_tail_last_slot(&mut self) -> Result> { + if let Some(ptr) = self.backtrace_tail_last_slot_alloca { + return Ok(ptr); + } + + let current_block = self.builder.get_insert_block().ok_or_else(|| { + CodeGenError::LLVMError("no current block for bt tail slot allocation".to_string()) + })?; + let current_fn = self.current_function("allocate bt tail slot")?; + let entry_block = current_fn.get_first_basic_block().ok_or_else(|| { + CodeGenError::LLVMError("no entry block for bt tail slot allocation".to_string()) + })?; + + if let Some(first_instruction) = entry_block.get_first_instruction() { + self.builder.position_before(&first_instruction); + } else { + self.builder.position_at_end(entry_block); + } + let alloca = self + .builder + .build_alloca(self.context.i8_type(), "bt_tail_last_slot") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store( + alloca, + self.context + .i8_type() + .const_int(crate::BACKTRACE_TAIL_NO_NEXT_SLOT as u64, false), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder.position_at_end(current_block); + self.backtrace_tail_last_slot_alloca = Some(alloca); + Ok(alloca) + } + + fn link_backtrace_tail_slot( + &mut self, + tail_slot: u8, + offsets_found_u8: IntValue<'ctx>, + done_block: BasicBlock<'ctx>, + ) -> Result<()> { + let current_fn = self.current_function("link bt tail slot")?; + let offsets_found = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + offsets_found_u8, + self.context.i8_type().const_zero(), + "bt_tail_link_offsets_found", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let link_block = self + .context + .append_basic_block(current_fn, "bt_tail_link_slot"); + self.builder + .build_conditional_branch(offsets_found, link_block, done_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(link_block); + let state0_ptr = self.lookup_bt_state_ptr(0)?; + let state0_is_null = self + .builder + .build_is_null(state0_ptr, "bt_tail_link_state0_null") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let state0_ok_block = self + .context + .append_basic_block(current_fn, "bt_tail_link_state0_ok"); + self.builder + .build_conditional_branch(state0_is_null, done_block, state0_ok_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(state0_ok_block); + let tail_enabled_ptr = self.get_or_create_backtrace_tail_enabled_flag()?; + let last_slot_ptr = self.get_or_create_backtrace_tail_last_slot()?; + let enabled_value = self + .builder + .build_load( + self.context.i8_type(), + tail_enabled_ptr, + "bt_tail_link_enabled_value", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let has_prev_slot = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + enabled_value, + self.context.i8_type().const_zero(), + "bt_tail_link_has_prev", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let first_slot_block = self + .context + .append_basic_block(current_fn, "bt_tail_link_first"); + let append_slot_block = self + .context + .append_basic_block(current_fn, "bt_tail_link_append"); + let linked_block = self + .context + .append_basic_block(current_fn, "bt_tail_linked"); + self.builder + .build_conditional_branch(has_prev_slot, append_slot_block, first_slot_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(first_slot_block); + self.store_u8_const( + state0_ptr, + crate::BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, + tail_slot, + "bt_tail_link_active_slot", + )?; + self.builder + .build_unconditional_branch(linked_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(append_slot_block); + let prev_slot = self.load_i8(last_slot_ptr, "bt_tail_link_prev_slot")?; + let prev_state_ptr = self.lookup_bt_state_ptr_dynamic(prev_slot)?; + let prev_state_is_null = self + .builder + .build_is_null(prev_state_ptr, "bt_tail_link_prev_null") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let prev_state_ok_block = self + .context + .append_basic_block(current_fn, "bt_tail_link_prev_ok"); + self.builder + .build_conditional_branch(prev_state_is_null, first_slot_block, prev_state_ok_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(prev_state_ok_block); + self.store_u8_const( + prev_state_ptr, + crate::BACKTRACE_TAIL_STATE_NEXT_SLOT_OFFSET, + tail_slot, + "bt_tail_link_prev_next_slot", + )?; + self.builder + .build_unconditional_branch(linked_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(linked_block); + self.builder + .build_store( + last_slot_ptr, + self.context.i8_type().const_int(tail_slot as u64, false), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(tail_enabled_ptr, self.context.i8_type().const_int(1, false)) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_unconditional_branch(done_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(()) + } + + fn store_state_i64( + &self, + base: PointerValue<'ctx>, + offset: usize, + value: IntValue<'ctx>, + name: &str, + ) -> Result<()> { + self.store_u64_value(base, offset, value, name) + } + + fn store_state_i32( + &self, + base: PointerValue<'ctx>, + offset: usize, + value: IntValue<'ctx>, + name: &str, + ) -> Result<()> { + let ptr = self.byte_gep(base, offset, name)?; + let ptr = self + .builder + .build_pointer_cast( + ptr, + self.context.ptr_type(AddressSpace::default()), + &format!("{name}_u32_ptr"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(ptr, value) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(()) + } + + fn load_state_i32( + &self, + base: PointerValue<'ctx>, + offset: usize, + name: &str, + ) -> Result> { + let ptr = self.byte_gep(base, offset, name)?; + let ptr = self + .builder + .build_pointer_cast( + ptr, + self.context.ptr_type(AddressSpace::default()), + &format!("{name}_u32_ptr"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(self + .builder + .build_load(self.context.i32_type(), ptr, name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value()) + } + + fn load_row_i64( + &self, + row_ptr: PointerValue<'ctx>, + offset: usize, + name: &str, + ) -> Result> { + let ptr = self.byte_gep(row_ptr, offset, name)?; + let ptr = self + .builder + .build_pointer_cast( + ptr, + self.context.ptr_type(AddressSpace::default()), + &format!("{name}_i64_ptr"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(self + .builder + .build_load(self.context.i64_type(), ptr, name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value()) + } + + fn load_row_i16( + &self, + row_ptr: PointerValue<'ctx>, + offset: usize, + name: &str, + ) -> Result> { + let ptr = self.byte_gep(row_ptr, offset, name)?; + Ok(self + .builder + .build_load(self.context.i16_type(), ptr, name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value()) + } + + fn load_row_i8( + &self, + row_ptr: PointerValue<'ctx>, + offset: usize, + name: &str, + ) -> Result> { + let ptr = self.byte_gep(row_ptr, offset, name)?; + Ok(self + .builder + .build_load(self.context.i8_type(), ptr, name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value()) + } + + fn load_i8(&self, ptr: PointerValue<'ctx>, name: &str) -> Result> { + Ok(self + .builder + .build_load(self.context.i8_type(), ptr, name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value()) + } + + fn load_i32(&self, ptr: PointerValue<'ctx>, name: &str) -> Result> { + Ok(self + .builder + .build_load(self.context.i32_type(), ptr, name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value()) + } + + fn load_i16(&self, ptr: PointerValue<'ctx>, name: &str) -> Result> { + Ok(self + .builder + .build_load(self.context.i16_type(), ptr, name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value()) + } + + fn load_i64(&self, ptr: PointerValue<'ctx>, name: &str) -> Result> { + Ok(self + .builder + .build_load(self.context.i64_type(), ptr, name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value()) + } + + fn load_dwarf_register_i64( + &mut self, + reg: u16, + pt_regs: PointerValue<'ctx>, + ) -> Result> { + let value = self.load_register_value(reg, pt_regs)?; + match value { + BasicValueEnum::IntValue(value) => Ok(value), + _ => Err(CodeGenError::RegisterMappingError(format!( + "DWARF register {reg} did not load as integer" + ))), + } + } + + fn add_signed_offset( + &self, + base: IntValue<'ctx>, + offset: i64, + name: &str, + ) -> Result> { + if offset == 0 { + return Ok(base); + } + self.builder + .build_int_add( + base, + self.context.i64_type().const_int(offset as u64, true), + name, + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + } + + fn normalized_pc_from_raw( + &self, + raw_ip: IntValue<'ctx>, + module_bias: IntValue<'ctx>, + offsets_found: IntValue<'ctx>, + ) -> Result> { + let rebased = self + .builder + .build_int_sub(raw_ip, module_bias, "bt_normalized_pc") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_select::, _>( + offsets_found, + rebased.into(), + raw_ip.into(), + "bt_pc_or_raw", + ) + .map(|value| value.into_int_value()) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + } + + fn backtrace_lookup_pc_from_raw( + &self, + raw_ip: IntValue<'ctx>, + module_bias: IntValue<'ctx>, + offsets_found: IntValue<'ctx>, + ) -> Result> { + if self.backtrace_unwind_rows_use_runtime_pcs { + Ok(raw_ip) + } else { + self.normalized_pc_from_raw(raw_ip, module_bias, offsets_found) + } + } + + fn status_or_offsets_unavailable( + &self, + status: BacktraceStatus, + offsets_found: IntValue<'ctx>, + ) -> Result> { + self.builder + .build_select::, _>( + offsets_found, + self.context + .i8_type() + .const_int(status as u64, false) + .into(), + self.context + .i8_type() + .const_int(BacktraceStatus::OffsetsUnavailable as u64, false) + .into(), + "bt_status_or_offsets", + ) + .map(|value| value.into_int_value()) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + } + + fn status_for_backtrace_stop( + &self, + complete: IntValue<'ctx>, + error_code: IntValue<'ctx>, + offsets_found: IntValue<'ctx>, + ) -> Result> { + let i8_type = self.context.i8_type(); + let i16_type = self.context.i16_type(); + let ra_read_error = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + error_code, + i16_type.const_int(BACKTRACE_ERROR_RETURN_ADDRESS_READ as u64, false), + "bt_status_ra_read_error", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let rbp_read_error = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + error_code, + i16_type.const_int(BACKTRACE_ERROR_FRAME_POINTER_READ as u64, false), + "bt_status_rbp_read_error", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let read_error = self + .builder + .build_or(ra_read_error, rbp_read_error, "bt_status_read_error_flag") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let error_status = self + .builder + .build_select::, _>( + read_error, + i8_type + .const_int(BacktraceStatus::ReadError as u64, false) + .into(), + i8_type + .const_int(BacktraceStatus::InvalidFrame as u64, false) + .into(), + "bt_status_for_error_code", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let backtrace_status = self + .builder + .build_select::, _>( + complete, + i8_type + .const_int(BacktraceStatus::Complete as u64, false) + .into(), + error_status, + "bt_status_for_stop", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_select::, _>( + offsets_found, + backtrace_status, + i8_type + .const_int(BacktraceStatus::OffsetsUnavailable as u64, false) + .into(), + "bt_status_or_offsets_for_error_code", + ) + .map(|value| value.into_int_value()) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + } + + fn store_backtrace_frame( + &self, + inst_buffer: PointerValue<'ctx>, + frame_index: usize, + module_cookie: u64, + pc: IntValue<'ctx>, + raw_ip: IntValue<'ctx>, + flags: u16, + ) -> Result<()> { + let frame_base = + INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_SIZE + frame_index * BACKTRACE_FRAME_DATA_SIZE; + self.store_u64_const( + inst_buffer, + frame_base + BACKTRACE_FRAME_MODULE_COOKIE_OFFSET, + module_cookie, + "bt_frame_cookie", + )?; + self.store_u64_value( + inst_buffer, + frame_base + BACKTRACE_FRAME_PC_OFFSET, + pc, + "bt_frame_pc", + )?; + self.store_u64_value( + inst_buffer, + frame_base + BACKTRACE_FRAME_RAW_IP_OFFSET, + raw_ip, + "bt_frame_raw_ip", + )?; + self.store_u16_const( + inst_buffer, + frame_base + BACKTRACE_FRAME_FLAGS_OFFSET, + flags, + "bt_frame_flags", + ) + } + + fn store_backtrace_frame_dynamic( + &self, + inst_buffer: PointerValue<'ctx>, + frame_index: IntValue<'ctx>, + max_frame_index: u8, + module_cookie: u64, + pc: IntValue<'ctx>, + raw_ip: IntValue<'ctx>, + ) -> Result<()> { + let i64_type = self.context.i64_type(); + let frame_index_i64 = if frame_index.get_type().get_bit_width() == i64_type.get_bit_width() + { + frame_index + } else { + self.builder + .build_int_z_extend(frame_index, i64_type, "bt_frame_index_i64") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + }; + let max_frame_index = i64_type.const_int(max_frame_index as u64, false); + let frame_index_in_bounds = self + .builder + .build_int_compare( + inkwell::IntPredicate::ULE, + frame_index_i64, + max_frame_index, + "bt_frame_index_in_bounds", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let frame_index_i64 = self + .builder + .build_select::, _>( + frame_index_in_bounds, + frame_index_i64.into(), + max_frame_index.into(), + "bt_frame_index_bounded", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let frame_stride = i64_type.const_int(BACKTRACE_FRAME_DATA_SIZE as u64, false); + let frame_offset = self + .builder + .build_int_mul(frame_index_i64, frame_stride, "bt_dynamic_frame_offset") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let base_offset = i64_type.const_int( + (INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_SIZE) as u64, + false, + ); + let frame_base_offset = self + .builder + .build_int_add(base_offset, frame_offset, "bt_dynamic_frame_base_offset") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let frame_base = + self.dynamic_byte_gep(inst_buffer, frame_base_offset, "bt_dynamic_frame")?; + + self.store_u64_const( + frame_base, + BACKTRACE_FRAME_MODULE_COOKIE_OFFSET, + module_cookie, + "bt_frame_cookie", + )?; + self.store_u64_value(frame_base, BACKTRACE_FRAME_PC_OFFSET, pc, "bt_frame_pc")?; + self.store_u64_value( + frame_base, + BACKTRACE_FRAME_RAW_IP_OFFSET, + raw_ip, + "bt_frame_raw_ip", + )?; + self.store_u16_const( + frame_base, + BACKTRACE_FRAME_FLAGS_OFFSET, + 0, + "bt_frame_flags", + ) + } + + fn store_u8_const( + &self, + base: PointerValue<'ctx>, + offset: usize, + value: u8, + name: &str, + ) -> Result<()> { + self.store_u8_value( + base, + offset, + self.context.i8_type().const_int(value as u64, false), + name, + ) + } + + fn store_u8_value( + &self, + base: PointerValue<'ctx>, + offset: usize, + value: IntValue<'ctx>, + name: &str, + ) -> Result<()> { + let ptr = self.byte_gep(base, offset, name)?; + self.builder + .build_store(ptr, value) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(()) + } + + fn store_u16_const( + &self, + base: PointerValue<'ctx>, + offset: usize, + value: u16, + name: &str, + ) -> Result<()> { + self.store_u16_value( + base, + offset, + self.context.i16_type().const_int(value as u64, false), + name, + ) + } + + fn store_u16_value( + &self, + base: PointerValue<'ctx>, + offset: usize, + value: IntValue<'ctx>, + name: &str, + ) -> Result<()> { + let ptr = self.byte_gep(base, offset, name)?; + let ptr = self + .builder + .build_pointer_cast( + ptr, + self.context.ptr_type(AddressSpace::default()), + &format!("{name}_u16_ptr"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(ptr, value) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(()) + } + + fn store_u64_const( + &self, + base: PointerValue<'ctx>, + offset: usize, + value: u64, + name: &str, + ) -> Result<()> { + self.store_u64_value( + base, + offset, + self.context.i64_type().const_int(value, false), + name, + ) + } + + fn store_u64_value( + &self, + base: PointerValue<'ctx>, + offset: usize, + value: IntValue<'ctx>, + name: &str, + ) -> Result<()> { + let ptr = self.byte_gep(base, offset, name)?; + let ptr = self + .builder + .build_pointer_cast( + ptr, + self.context.ptr_type(AddressSpace::default()), + &format!("{name}_u64_ptr"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(ptr, value) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(()) + } + + fn byte_gep( + &self, + base: PointerValue<'ctx>, + offset: usize, + name: &str, + ) -> Result> { + // SAFETY: callers pass offsets within the instruction region reserved for + // this Backtrace instruction. + unsafe { + self.builder + .build_gep( + self.context.i8_type(), + base, + &[self.context.i32_type().const_int(offset as u64, false)], + &format!("{name}_ptr"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + } + } + + fn dynamic_byte_gep( + &self, + base: PointerValue<'ctx>, + offset: IntValue<'ctx>, + name: &str, + ) -> Result> { + // SAFETY: callers guard the dynamic offset against the per-CPU buffer + // size before using the returned pointer. + unsafe { + self.builder + .build_gep( + self.context.i8_type(), + base, + &[offset], + &format!("{name}_ptr"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + } + } + + fn build_entry_alloca(&self, ty: T, name: &str) -> Result> + where + T: inkwell::types::BasicType<'ctx>, + { + let current_block = self.builder.get_insert_block().ok_or_else(|| { + CodeGenError::LLVMError("no current block for bt stack allocation".to_string()) + })?; + let current_fn = self.current_function("allocate bt scratch")?; + let entry_block = current_fn.get_first_basic_block().ok_or_else(|| { + CodeGenError::LLVMError("no entry block for bt stack allocation".to_string()) + })?; + + if let Some(first_instruction) = entry_block.get_first_instruction() { + self.builder.position_before(&first_instruction); + } else { + self.builder.position_at_end(entry_block); + } + let alloca = self + .builder + .build_alloca(ty, name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder.position_at_end(current_block); + Ok(alloca) + } +} + +fn statements_have_backtrace(statements: &[Statement]) -> bool { + count_backtrace_statements(statements) > 0 +} + +fn backtrace_row_binary_search_steps(row_count: usize) -> usize { + if row_count <= 1 { + 1 + } else { + (usize::BITS - (row_count - 1).leading_zeros()) as usize + 1 + } +} + +fn count_backtrace_statements(statements: &[Statement]) -> usize { + statements.iter().map(count_statement_backtraces).sum() +} + +fn count_statement_backtraces(statement: &Statement) -> usize { + match statement { + Statement::Backtrace(_) => 1, + Statement::TracePoint { body, .. } | Statement::Block(body) => { + count_backtrace_statements(body) + } + Statement::If { + then_body, + else_body, + .. + } => { + count_backtrace_statements(then_body) + + else_body + .as_deref() + .map(count_statement_backtraces) + .unwrap_or(0) + } + _ => 0, + } +} + +fn backtrace_unwind_row_from_compact( + row: &CompactUnwindRow, +) -> Option { + if !row.bpf_supported { + return None; + } + let CfaRulePlan::RegPlusOffset { + register, + offset: cfa_offset, + } = &row.cfa + else { + return None; + }; + if !backtrace_supported_state_register(*register) { + return None; + } + + let mut wire = ghostscope_protocol::BacktraceUnwindRow { + pc_start: row.pc_start, + pc_end: row.pc_end, + cfa_offset: *cfa_offset, + cfa_register: *register, + ..Default::default() + }; + + match &row.return_address { + RegisterRecoveryPlan::AtCfaOffset { offset } => { + wire.ra_kind = crate::BACKTRACE_RECOVERY_AT_CFA_OFFSET; + wire.ra_offset = *offset; + wire.ra_register = row.return_address_register; + } + _ => return None, + } + + match row.rbp.as_ref() { + Some(RegisterRecoveryPlan::AtCfaOffset { offset }) => { + wire.rbp_kind = crate::BACKTRACE_RECOVERY_AT_CFA_OFFSET; + wire.rbp_offset = *offset; + wire.rbp_register = X86_64_DWARF_RBP; + } + Some(RegisterRecoveryPlan::ValCfaOffset { offset }) => { + wire.rbp_kind = crate::BACKTRACE_RECOVERY_VAL_CFA_OFFSET; + wire.rbp_offset = *offset; + wire.rbp_register = X86_64_DWARF_RBP; + } + Some(RegisterRecoveryPlan::Register { register }) => { + if !backtrace_supported_state_register(*register) { + return None; + } + wire.rbp_kind = crate::BACKTRACE_RECOVERY_REGISTER; + wire.rbp_register = *register; + } + Some(RegisterRecoveryPlan::SameValue { register }) => { + if !backtrace_supported_state_register(*register) { + return None; + } + wire.rbp_kind = crate::BACKTRACE_RECOVERY_SAME_VALUE; + wire.rbp_register = *register; + } + Some(RegisterRecoveryPlan::Undefined) | None => { + wire.rbp_kind = crate::BACKTRACE_RECOVERY_SAME_VALUE; + wire.rbp_register = X86_64_DWARF_RBP; + } + _ => return None, + } + + Some(wire) +} + +fn backtrace_supported_state_register(register: u16) -> bool { + matches!( + register, + X86_64_DWARF_RIP | X86_64_DWARF_RBP | X86_64_DWARF_RSP + ) +} + +fn backtrace_flags(stmt: &BacktraceStatement) -> u8 { + let mut flags = 0u8; + if stmt.raw { + flags |= BACKTRACE_FLAG_RAW; + } + if stmt.full { + flags |= BACKTRACE_FLAG_FULL; + } + if stmt.inline { + flags |= BACKTRACE_FLAG_INLINE; + } + flags +} + +#[cfg(test)] +mod tests { + use super::backtrace_row_binary_search_steps; + + #[test] + fn binary_search_steps_cover_power_of_two_row_counts() { + assert_eq!(backtrace_row_binary_search_steps(0), 1); + assert_eq!(backtrace_row_binary_search_steps(1), 1); + assert_eq!(backtrace_row_binary_search_steps(2), 2); + assert_eq!(backtrace_row_binary_search_steps(4), 3); + assert_eq!(backtrace_row_binary_search_steps(8), 4); + } + + #[test] + fn binary_search_steps_cover_non_power_of_two_row_counts() { + assert_eq!(backtrace_row_binary_search_steps(3), 3); + assert_eq!(backtrace_row_binary_search_steps(5), 4); + assert_eq!(backtrace_row_binary_search_steps(9), 5); } } diff --git a/ghostscope-compiler/src/ebpf/codegen/mod.rs b/ghostscope-compiler/src/ebpf/codegen/mod.rs index 47d7b04c..a6e34fce 100644 --- a/ghostscope-compiler/src/ebpf/codegen/mod.rs +++ b/ghostscope-compiler/src/ebpf/codegen/mod.rs @@ -7,8 +7,16 @@ use super::context::{CodeGenError, EbpfContext, Result, RuntimeAddress}; use crate::script::{PrintStatement, Program, Statement}; use aya_ebpf_bindings::bindings::bpf_func_id::BPF_FUNC_probe_read_user; use ghostscope_protocol::trace_event::{ - BacktraceData, EndInstructionData, InstructionHeader, PrintComplexFormatData, - PrintComplexVariableData, PrintStringIndexData, PrintVariableIndexData, VariableStatus, + BacktraceFrameData, BacktraceStatus, EndInstructionData, InstructionHeader, + PrintComplexFormatData, PrintComplexVariableData, PrintStringIndexData, PrintVariableIndexData, + VariableStatus, BACKTRACE_DATA_ERROR_CODE_OFFSET, BACKTRACE_DATA_FLAGS_OFFSET, + BACKTRACE_DATA_FRAME_COUNT_OFFSET, BACKTRACE_DATA_REQUESTED_DEPTH_OFFSET, BACKTRACE_DATA_SIZE, + BACKTRACE_DATA_STATUS_OFFSET, BACKTRACE_ERROR_FRAME_POINTER_READ, + BACKTRACE_ERROR_NEXT_CFA_NOT_ADVANCING, BACKTRACE_ERROR_NEXT_CFA_ZERO, + BACKTRACE_ERROR_NEXT_IP_BELOW_USER, BACKTRACE_ERROR_NEXT_IP_KERNEL_LIKE, BACKTRACE_ERROR_NONE, + BACKTRACE_ERROR_RETURN_ADDRESS_READ, BACKTRACE_FLAG_FULL, BACKTRACE_FLAG_INLINE, + BACKTRACE_FLAG_RAW, BACKTRACE_FRAME_DATA_SIZE, BACKTRACE_FRAME_FLAGS_OFFSET, + BACKTRACE_FRAME_MODULE_COOKIE_OFFSET, BACKTRACE_FRAME_PC_OFFSET, BACKTRACE_FRAME_RAW_IP_OFFSET, EXPR_ERROR_DATA_ERROR_CODE_OFFSET, EXPR_ERROR_DATA_FAILING_ADDR_OFFSET, EXPR_ERROR_DATA_FLAGS_OFFSET, EXPR_ERROR_DATA_SIZE, EXPR_ERROR_DATA_STRING_INDEX_OFFSET, INSTRUCTION_HEADER_DATA_LENGTH_OFFSET, INSTRUCTION_HEADER_SIZE, diff --git a/ghostscope-compiler/src/ebpf/codegen/statements.rs b/ghostscope-compiler/src/ebpf/codegen/statements.rs index 04b914d7..cc71fd29 100644 --- a/ghostscope-compiler/src/ebpf/codegen/statements.rs +++ b/ghostscope-compiler/src/ebpf/codegen/statements.rs @@ -28,8 +28,10 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { instruction_count += self.compile_statement(statement)?; } - // Step 4: Send EndInstruction to mark completion - self.send_end_instruction(instruction_count)?; + // Step 4: Write EndInstruction and either emit immediately or hand off + // emission to the bt tail-call finalizer. + self.write_end_instruction(instruction_count)?; + self.finish_event_after_instructions()?; info!( "Sent EndInstruction with {} total instructions", instruction_count @@ -103,6 +105,10 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { } } Statement::Print(print_stmt) => self.compile_print_statement(print_stmt), + Statement::Backtrace(backtrace_stmt) => { + self.generate_backtrace_instruction(backtrace_stmt)?; + Ok(1) + } Statement::If { condition, then_body, diff --git a/ghostscope-compiler/src/ebpf/context.rs b/ghostscope-compiler/src/ebpf/context.rs index e429dd67..b615a5a0 100644 --- a/ghostscope-compiler/src/ebpf/context.rs +++ b/ghostscope-compiler/src/ebpf/context.rs @@ -26,6 +26,19 @@ pub struct CompileTimeContext { pub module_path: String, } +#[derive(Debug, Clone)] +pub struct BacktraceTailCallProgram { + pub step_program_name: String, +} + +#[derive(Debug, Clone)] +pub(crate) struct PendingBacktraceTailCall { + pub step_program_name: String, + pub module_cookie: u64, + pub depth: u8, + pub instruction_size: usize, +} + #[derive(Error, Debug)] pub enum CodeGenError { #[error("LLVM compilation error: {0}")] @@ -147,6 +160,15 @@ pub struct EbpfContext<'ctx, 'dw> { // we keep its bytes (including optional NUL) here for content printing via ImmediateBytes. pub string_vars: HashMap>, + // === DWARF compact unwind rows for bt === + pub backtrace_unwind_rows: Vec, + pub(crate) backtrace_unwind_rows_use_runtime_pcs: bool, + pub(crate) backtrace_tail_call_slots: u8, + pub(crate) next_backtrace_tail_call_slot: u8, + pub(crate) pending_backtrace_tail_call: Option, + pub(crate) backtrace_tail_enabled_alloca: Option>, + pub(crate) backtrace_tail_last_slot_alloca: Option>, + // === Lexical scoping for immutable variables === // Each scope frame records names declared in that scope. pub scope_stack: Vec>, @@ -249,6 +271,14 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { alias_vars: HashMap::new(), // String variables string_vars: HashMap::new(), + // Backtrace compact unwind rows + backtrace_unwind_rows: Vec::new(), + backtrace_unwind_rows_use_runtime_pcs: false, + backtrace_tail_call_slots: 1, + next_backtrace_tail_call_slot: 0, + pending_backtrace_tail_call: None, + backtrace_tail_enabled_alloca: None, + backtrace_tail_last_slot_alloca: None, // Scopes scope_stack: Vec::new(), @@ -425,6 +455,14 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { self.trace_context.clone() } + pub fn backtrace_tail_call_program(&self) -> Option { + self.pending_backtrace_tail_call + .as_ref() + .map(|plan| BacktraceTailCallProgram { + step_program_name: plan.step_program_name.clone(), + }) + } + pub(crate) fn current_insert_block(&self, op: &str) -> Result> { self.builder .get_insert_block() @@ -474,6 +512,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { } else { None }; + self.prepare_backtrace_unwind_rows(trace_statements); // Create required maps - critical for eBPF loader // Create event output map based on compile options @@ -560,6 +599,44 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { CodeGenError::LLVMError(format!("Failed to create pid_aliases map: {e}")) })?; + if !self.backtrace_unwind_rows.is_empty() { + self.map_manager + .create_array_map( + &self.module, + &self.di_builder, + &self.compile_unit, + "bt_unwind_rows", + self.backtrace_unwind_rows.len() as u64, + crate::BACKTRACE_UNWIND_ROW_SIZE as u64, + ) + .map_err(|e| { + CodeGenError::LLVMError(format!("Failed to create bt_unwind_rows map: {e}")) + })?; + self.map_manager + .create_percpu_array_map( + &self.module, + &self.di_builder, + &self.compile_unit, + "bt_state", + self.backtrace_tail_call_slots.max(1) as u64, + crate::BACKTRACE_TAIL_STATE_SIZE as u64, + ) + .map_err(|e| { + CodeGenError::LLVMError(format!("Failed to create bt_state map: {e}")) + })?; + self.map_manager + .create_program_array_map( + &self.module, + &self.di_builder, + &self.compile_unit, + "bt_prog_array", + 1, + ) + .map_err(|e| { + CodeGenError::LLVMError(format!("Failed to create bt_prog_array map: {e}")) + })?; + } + // Variables are now queried on-demand when accessed in expressions // No need to pre-populate DWARF variables @@ -648,6 +725,30 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { Ok(function) } + pub(crate) fn create_tail_call_function( + &mut self, + function_name: &str, + ) -> Result> { + let i32_type = self.context.i32_type(); + let ptr_type = self.context.ptr_type(AddressSpace::default()); + let fn_type = i32_type.fn_type(&[ptr_type.into()], false); + let function = self.module.add_function(function_name, fn_type, None); + function.set_section(Some("uprobe")); + + let basic_block = self.context.append_basic_block(function, "entry"); + self.builder.position_at_end(basic_block); + + let key_arr_ty = i32_type.array_type(4); + let key_alloca = self + .builder + .build_alloca(key_arr_ty, "pm_key") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.pm_key_alloca = Some(key_alloca); + + info!("Created tail-call eBPF function: {}", function_name); + Ok(function) + } + /// Add PID filtering logic to the current function. /// This generates LLVM IR to check PID and early-return if not matching. fn add_pid_filter(&mut self, spec: crate::PidFilterSpec) -> Result<()> { diff --git a/ghostscope-compiler/src/ebpf/dwarf_bridge.rs b/ghostscope-compiler/src/ebpf/dwarf_bridge.rs index e4f8bb9b..1a87eab7 100644 --- a/ghostscope-compiler/src/ebpf/dwarf_bridge.rs +++ b/ghostscope-compiler/src/ebpf/dwarf_bridge.rs @@ -40,7 +40,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { } /// Compute cookie for module using coordinator policy. - fn cookie_for_module_or_fallback(&mut self, module_path: &str) -> u64 { + pub(crate) fn cookie_for_module_or_fallback(&mut self, module_path: &str) -> u64 { self.fallback_cookie_from_module_path(module_path) } fn planned_value_to_llvm_value( diff --git a/ghostscope-compiler/src/ebpf/expression.rs b/ghostscope-compiler/src/ebpf/expression.rs index ffa4c278..963fd56d 100644 --- a/ghostscope-compiler/src/ebpf/expression.rs +++ b/ghostscope-compiler/src/ebpf/expression.rs @@ -2993,9 +2993,19 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let ts = self.get_current_timestamp()?; Ok(ts.into()) } + "pc" => self.load_special_register_value(16), + "sp" => self.load_special_register_value(7), _ => { - let supported = - ["$pid", "$tid", "$host_pid", "$input_pid", "$timestamp"].join(", "); + let supported = [ + "$pid", + "$tid", + "$host_pid", + "$input_pid", + "$timestamp", + "$pc", + "$sp", + ] + .join(", "); Err(CodeGenError::NotImplemented(format!( "Unknown special variable '${name}'. Supported: {supported}" ))) @@ -3003,6 +3013,11 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { } } + fn load_special_register_value(&mut self, dwarf_reg: u16) -> Result> { + let pt_regs = self.get_pt_regs_parameter()?; + self.load_register_value(dwarf_reg, pt_regs) + } + /// Compile binary operations pub fn compile_binary_op( &mut self, diff --git a/ghostscope-compiler/src/ebpf/helper_functions.rs b/ghostscope-compiler/src/ebpf/helper_functions.rs index b9abf6b5..b18ad69c 100644 --- a/ghostscope-compiler/src/ebpf/helper_functions.rs +++ b/ghostscope-compiler/src/ebpf/helper_functions.rs @@ -454,16 +454,6 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let key_alloca = self.pm_key_alloca.ok_or_else(|| { CodeGenError::LLVMError("pm_key not allocated in entry block".to_string()) })?; - // Get i32* to the first element (&key[0]) - let zero = i32_type.const_zero(); - // SAFETY: key_alloca is the [4 x i32] pm_key stack slot and [0, 0] - // addresses the first element. - let base_i32_ptr = unsafe { - self.builder - .build_gep(key_arr_ty, key_alloca, &[zero, zero], "pm_key_i32_ptr") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - }; - // Resolve the pid key used for proc_module_offsets lookup. // Host mode uses host TGID, while NamespaceTgid mode uses namespace TGID // from bpf_get_ns_current_pid_tgid(). @@ -569,60 +559,52 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { }; let pid = self.lookup_proc_pid_alias(runtime_pid, "offset_pid")?; - // Store pid at key[0] - self.builder - .build_store(base_i32_ptr, pid) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - // Zero padding at key[1] for deterministic key bytes - let idx1 = i32_type.const_int(1, false); - // SAFETY: base_i32_ptr points to key[0] of a four-element i32 array; index - // 1 is the padding field. - let pad_ptr = unsafe { - self.builder - .build_gep(self.context.i32_type(), base_i32_ptr, &[idx1], "pad_ptr") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + let store_key_u32 = |offset: usize, + value: IntValue<'ctx>, + name: &str, + ctx: &mut EbpfContext<'ctx, 'dw>| + -> Result<()> { + let offset_i32 = ctx.context.i32_type().const_int(offset as u64, false); + // SAFETY: key_alloca is the ProcModuleKey stack slot. `offset` + // comes from ghostscope_protocol::bpf_abi field offsets. + let ptr = unsafe { + ctx.builder + .build_gep(ctx.context.i8_type(), key_alloca, &[offset_i32], name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + }; + ctx.builder + .build_store(ptr, value) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(()) }; - self.builder - .build_store(pad_ptr, self.context.i32_type().const_zero()) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - // Store cookie_lo at key[2] and cookie_hi at key[3] (key[1] is padding for 8-byte alignment) + store_key_u32( + ghostscope_protocol::PROC_MODULE_KEY_PID_OFFSET, + pid, + "pm_key_pid_ptr", + self, + )?; + store_key_u32( + ghostscope_protocol::PROC_MODULE_KEY_PAD_OFFSET, + self.context.i32_type().const_zero(), + "pm_key_pad_ptr", + self, + )?; + let cookie_lo = i32_type.const_int(module_cookie & 0xffff_ffff, false); let cookie_hi = i32_type.const_int(module_cookie >> 32, false); - let idx2 = i32_type.const_int(2, false); - let idx3 = i32_type.const_int(3, false); - // key[1] left as padding = 0 by default - // SAFETY: base_i32_ptr points to key[0] of a four-element i32 array; index - // 2 is the low cookie word. - let cookie_lo_ptr = unsafe { - self.builder - .build_gep( - self.context.i32_type(), - base_i32_ptr, - &[idx2], - "cookie_lo_ptr", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - }; - // SAFETY: base_i32_ptr points to key[0] of a four-element i32 array; index - // 3 is the high cookie word. - let cookie_hi_ptr = unsafe { - self.builder - .build_gep( - self.context.i32_type(), - base_i32_ptr, - &[idx3], - "cookie_hi_ptr", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - }; - self.builder - .build_store(cookie_lo_ptr, cookie_lo) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(cookie_hi_ptr, cookie_hi) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + store_key_u32( + ghostscope_protocol::PROC_MODULE_KEY_COOKIE_LO_OFFSET, + cookie_lo, + "pm_key_cookie_lo_ptr", + self, + )?; + store_key_u32( + ghostscope_protocol::PROC_MODULE_KEY_COOKIE_HI_OFFSET, + cookie_hi, + "pm_key_cookie_hi_ptr", + self, + )?; // Call bpf_map_lookup_elem(map, &key) let lookup_id = i64_type.const_int(BPF_FUNC_map_lookup_elem as u64, false); @@ -672,26 +654,20 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .build_conditional_branch(is_null, miss_block, found_block) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - // Found: load appropriate offset based on section_type + // Found: load the requested field from ProcModuleOffsetsValue. The + // byte offsets are shared through ghostscope_protocol::bpf_abi because + // this map is an ABI between generated eBPF and userspace. self.builder.position_at_end(found_block); - // Cast value pointer (void*) to i64* for loading 64-bit offsets - // Use opaque pointer type (LLVM15+): model as generic pointer - let i64_ptr_ty = self.context.ptr_type(AddressSpace::default()); - let val_u64_ptr = self - .builder - .build_pointer_cast(val_ptr, i64_ptr_ty, "val_u64_ptr") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let load_field = |idx: u64, + let load_field = |offset: usize, ctx: &mut EbpfContext<'ctx, 'dw>, base: PointerValue<'ctx>| -> Result> { - // GEP in i64 element space - let idx_i32 = ctx.context.i32_type().const_int(idx, false); - // SAFETY: val_u64_ptr points at ProcModuleOffsetsValue, represented as - // four contiguous u64 fields; idx is one of 0..=3. + let offset_i32 = ctx.context.i32_type().const_int(offset as u64, false); + // SAFETY: `base` points at ProcModuleOffsetsValue returned by + // bpf_map_lookup_elem. `offset` is one of its u64 field offsets. let field_ptr = unsafe { ctx.builder - .build_gep(ctx.context.i64_type(), base, &[idx_i32], "field_ptr_i64") + .build_gep(ctx.context.i8_type(), base, &[offset_i32], "field_ptr") .map_err(|e| CodeGenError::LLVMError(e.to_string()))? }; let loaded = ctx @@ -705,10 +681,26 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { } }; let st = section_type; - let off_text = load_field(0, self, val_u64_ptr)?; - let off_rodata = load_field(1, self, val_u64_ptr)?; - let off_data = load_field(2, self, val_u64_ptr)?; - let off_bss = load_field(3, self, val_u64_ptr)?; + let off_text = load_field( + ghostscope_protocol::PROC_MODULE_OFFSETS_VALUE_TEXT_OFFSET, + self, + val_ptr, + )?; + let off_rodata = load_field( + ghostscope_protocol::PROC_MODULE_OFFSETS_VALUE_RODATA_OFFSET, + self, + val_ptr, + )?; + let off_data = load_field( + ghostscope_protocol::PROC_MODULE_OFFSETS_VALUE_DATA_OFFSET, + self, + val_ptr, + )?; + let off_bss = load_field( + ghostscope_protocol::PROC_MODULE_OFFSETS_VALUE_BSS_OFFSET, + self, + val_ptr, + )?; // Build a bottom-up cascade to preserve earlier choices: // tmp = (section==data) ? off_data : off_bss // tmp2 = (section==rodata) ? off_rodata : tmp @@ -1086,6 +1078,36 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .map_err(|e| CodeGenError::LLVMError(e.to_string())) } + /// Generate a user memory read and return both the zero-on-failure value and failure flag. + pub(crate) fn generate_memory_read_with_fail_flag( + &mut self, + address: RuntimeAddress<'ctx>, + size: MemoryAccessSize, + name_suffix: &str, + ) -> Result<(BasicValueEnum<'ctx>, IntValue<'ctx>)> { + let zero_const = self.context.i64_type().const_zero(); + let ProbeReadResult { + loaded_i64, + combined_fail, + .. + } = self.probe_read_user_core(address, size, name_suffix)?; + + self.update_any_fail_flag(combined_fail, name_suffix)?; + + let zero_bv: BasicValueEnum = zero_const.into(); + let val_bv: BasicValueEnum = loaded_i64.into(); + let value = self + .builder + .build_select::, _>( + combined_fail, + zero_bv, + val_bv, + &format!("{name_suffix}_value_or_zero"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok((value, combined_fail)) + } + /// Generate memory read with runtime status capture (for control-flow conditions). /// On helper failure, sets condition error code (if active) and returns zero value. pub(crate) fn generate_memory_read_with_status( diff --git a/ghostscope-compiler/src/ebpf/instruction.rs b/ghostscope-compiler/src/ebpf/instruction.rs index a3f6a547..d76e5be9 100644 --- a/ghostscope-compiler/src/ebpf/instruction.rs +++ b/ghostscope-compiler/src/ebpf/instruction.rs @@ -457,10 +457,10 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { Ok(()) } - /// Send EndInstruction as final segment - pub fn send_end_instruction(&mut self, total_instructions: u16) -> Result<()> { + /// Write EndInstruction as final segment into the accumulation buffer. + pub(crate) fn write_end_instruction(&mut self, total_instructions: u16) -> Result<()> { info!( - "Sending EndInstruction segment with {} total instructions", + "Writing EndInstruction segment with {} total instructions", total_instructions ); @@ -634,70 +634,78 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { // Already accumulated in per-CPU buffer; no extra copy needed - // End: send the entire accumulated buffer once (RingBuf or PerfEventArray) - { - let accum_buffer = self - .get_or_create_perf_accumulation_buffer_or_return_zero()? - .into_value_after_runtime_returns(); - let offset_ptr = self.get_or_create_perf_buffer_offset()?; + Ok(()) + } - // Load total accumulated size (u32) - let total_accumulated_size = self - .builder - .build_load(self.context.i32_type(), offset_ptr, "total_size") - .map_err(|e| CodeGenError::LLVMError(format!("Failed to load total size: {e}")))? - .into_int_value(); - - // Clamp size to [0, max_trace_event_size] to satisfy verifier bounded access - let max_size_i32 = self - .context - .i32_type() - .const_int(self.compile_options.max_trace_event_size as u64, false); - let size_le_max = self - .builder - .build_int_compare( - inkwell::IntPredicate::ULE, - total_accumulated_size, - max_size_i32, - "size_le_max", - ) - .map_err(|e| CodeGenError::LLVMError(format!("Failed to compare end size: {e}")))?; - let clamped_size_i32 = self - .builder - .build_select( - size_le_max, - total_accumulated_size, - max_size_i32, - "clamped_size_i32", - ) - .map_err(|e| CodeGenError::LLVMError(format!("Failed to select clamp size: {e}")))? - .into_int_value(); - - // Convert i32 to i64 for size parameter - let total_size_i64 = self - .builder - .build_int_z_extend(clamped_size_i32, self.context.i64_type(), "total_size_i64") - .map_err(|e| CodeGenError::LLVMError(format!("Failed to extend size: {e}")))?; - - match self.compile_options.event_map_type { - crate::EventMapType::PerfEventArray => { - // Single-shot perf send - self.create_perf_event_output_dynamic(accum_buffer, total_size_i64)?; - } - crate::EventMapType::RingBuf => { - // Single-shot ringbuf send - self.create_ringbuf_output_dynamic(accum_buffer, total_size_i64)?; - } - } + /// Send EndInstruction and immediately output the accumulated event. + pub fn send_end_instruction(&mut self, total_instructions: u16) -> Result<()> { + self.write_end_instruction(total_instructions)?; + self.emit_accumulated_event_output_from_stack_offset() + } - // Reset offset to 0 after sending to ensure clean state for next trace event - self.builder - .build_store(offset_ptr, self.context.i32_type().const_zero()) - .map_err(|e| { - CodeGenError::LLVMError(format!("Failed to reset offset after send: {e}")) - })?; - } + pub(crate) fn emit_accumulated_event_output_from_stack_offset(&mut self) -> Result<()> { + let accum_buffer = self + .get_or_create_perf_accumulation_buffer_or_return_zero()? + .into_value_after_runtime_returns(); + let offset_ptr = self.get_or_create_perf_buffer_offset()?; + let total_accumulated_size = self + .builder + .build_load(self.context.i32_type(), offset_ptr, "total_size") + .map_err(|e| CodeGenError::LLVMError(format!("Failed to load total size: {e}")))? + .into_int_value(); + self.emit_accumulated_event_output(accum_buffer, total_accumulated_size)?; + + self.builder + .build_store(offset_ptr, self.context.i32_type().const_zero()) + .map_err(|e| { + CodeGenError::LLVMError(format!("Failed to reset offset after send: {e}")) + })?; + Ok(()) + } + + pub(crate) fn emit_accumulated_event_output( + &mut self, + accum_buffer: PointerValue<'ctx>, + total_accumulated_size: inkwell::values::IntValue<'ctx>, + ) -> Result<()> { + let max_size_i32 = self + .context + .i32_type() + .const_int(self.compile_options.max_trace_event_size as u64, false); + let size_le_max = self + .builder + .build_int_compare( + inkwell::IntPredicate::ULE, + total_accumulated_size, + max_size_i32, + "size_le_max", + ) + .map_err(|e| CodeGenError::LLVMError(format!("Failed to compare end size: {e}")))?; + let clamped_size_i32 = self + .builder + .build_select( + size_le_max, + total_accumulated_size, + max_size_i32, + "clamped_size_i32", + ) + .map_err(|e| CodeGenError::LLVMError(format!("Failed to select clamp size: {e}")))? + .into_int_value(); + + let total_size_i64 = self + .builder + .build_int_z_extend(clamped_size_i32, self.context.i64_type(), "total_size_i64") + .map_err(|e| CodeGenError::LLVMError(format!("Failed to extend size: {e}")))?; + + match self.compile_options.event_map_type { + crate::EventMapType::PerfEventArray => { + self.create_perf_event_output_dynamic(accum_buffer, total_size_i64)?; + } + crate::EventMapType::RingBuf => { + self.create_ringbuf_output_dynamic(accum_buffer, total_size_i64)?; + } + } Ok(()) } } diff --git a/ghostscope-compiler/src/ebpf/maps.rs b/ghostscope-compiler/src/ebpf/maps.rs index 504530ce..dc6414e4 100644 --- a/ghostscope-compiler/src/ebpf/maps.rs +++ b/ghostscope-compiler/src/ebpf/maps.rs @@ -16,6 +16,7 @@ pub enum BpfMapType { PerCpuArray, Hash, PerfEventArray, + ProgramArray, } impl BpfMapType { @@ -26,6 +27,7 @@ impl BpfMapType { BpfMapType::Array => bpf_map_type::BPF_MAP_TYPE_ARRAY, BpfMapType::Hash => bpf_map_type::BPF_MAP_TYPE_HASH, BpfMapType::PerfEventArray => bpf_map_type::BPF_MAP_TYPE_PERF_EVENT_ARRAY, + BpfMapType::ProgramArray => bpf_map_type::BPF_MAP_TYPE_PROG_ARRAY, } } } @@ -527,6 +529,49 @@ impl<'ctx> MapManager<'ctx> { SizedType::integer(value_size_bytes * 8), ) } + + /// Create a regular Array map (key=u32, value arbitrary size). + pub fn create_array_map( + &mut self, + module: &Module<'ctx>, + di_builder: &DebugInfoBuilder<'ctx>, + compile_unit: &inkwell::debug_info::DICompileUnit<'ctx>, + name: &str, + max_entries: u64, + value_size_bytes: u64, + ) -> Result<()> { + self.create_map_definition( + module, + di_builder, + compile_unit, + name, + BpfMapType::Array, + max_entries, + SizedType::integer(32), + SizedType::integer(value_size_bytes * 8), + ) + } + + /// Create a ProgramArray map for eBPF tail calls. + pub fn create_program_array_map( + &mut self, + module: &Module<'ctx>, + di_builder: &DebugInfoBuilder<'ctx>, + compile_unit: &inkwell::debug_info::DICompileUnit<'ctx>, + name: &str, + max_entries: u64, + ) -> Result<()> { + self.create_map_definition( + module, + di_builder, + compile_unit, + name, + BpfMapType::ProgramArray, + max_entries, + SizedType::integer(32), + SizedType::integer(32), + ) + } } #[cfg(test)] @@ -555,5 +600,9 @@ mod tests { MapManager::map_definition_field_count("ringbuf", BpfMapType::Ringbuf), 2 ); + assert_eq!( + MapManager::map_definition_field_count("bt_prog_array", BpfMapType::ProgramArray), + 4 + ); } } diff --git a/ghostscope-compiler/src/lib.rs b/ghostscope-compiler/src/lib.rs index 46d3d170..dc3a838a 100644 --- a/ghostscope-compiler/src/lib.rs +++ b/ghostscope-compiler/src/lib.rs @@ -15,6 +15,29 @@ pub fn hello() -> &'static str { "Hello from ghostscope-compiler!" } +pub use ghostscope_protocol::bpf_abi::{ + BacktraceTailCallState, BacktraceUnwindRow, BACKTRACE_RA_AT_CFA_OFFSET, BACKTRACE_RA_REGISTER, + BACKTRACE_RA_SAME_VALUE, BACKTRACE_RA_UNDEFINED, BACKTRACE_RA_VAL_CFA_OFFSET, + BACKTRACE_RECOVERY_AT_CFA_OFFSET, BACKTRACE_RECOVERY_REGISTER, BACKTRACE_RECOVERY_SAME_VALUE, + BACKTRACE_RECOVERY_UNDEFINED, BACKTRACE_RECOVERY_VAL_CFA_OFFSET, BACKTRACE_TAIL_NO_NEXT_SLOT, + BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, BACKTRACE_TAIL_STATE_CURRENT_IP_OFFSET, + BACKTRACE_TAIL_STATE_CURRENT_RBP_OFFSET, BACKTRACE_TAIL_STATE_CURRENT_RSP_OFFSET, + BACKTRACE_TAIL_STATE_ERROR_CODE_OFFSET, BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, + BACKTRACE_TAIL_STATE_FLAGS_OFFSET, BACKTRACE_TAIL_STATE_FRAME_COUNT_OFFSET, + BACKTRACE_TAIL_STATE_INST_OFFSET_OFFSET, BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, + BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET, BACKTRACE_TAIL_STATE_NEXT_SLOT_OFFSET, + BACKTRACE_TAIL_STATE_OFFSETS_FOUND_OFFSET, BACKTRACE_TAIL_STATE_REQUESTED_DEPTH_OFFSET, + BACKTRACE_TAIL_STATE_SIZE, BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, + BACKTRACE_UNWIND_ROW_CFA_OFFSET_OFFSET, BACKTRACE_UNWIND_ROW_CFA_REGISTER_OFFSET, + BACKTRACE_UNWIND_ROW_PC_END_OFFSET, BACKTRACE_UNWIND_ROW_PC_START_OFFSET, + BACKTRACE_UNWIND_ROW_RA_KIND_OFFSET, BACKTRACE_UNWIND_ROW_RA_OFFSET_OFFSET, + BACKTRACE_UNWIND_ROW_RA_REGISTER_OFFSET, BACKTRACE_UNWIND_ROW_RBP_KIND_OFFSET, + BACKTRACE_UNWIND_ROW_RBP_OFFSET_OFFSET, BACKTRACE_UNWIND_ROW_RBP_REGISTER_OFFSET, + BACKTRACE_UNWIND_ROW_SIZE, BACKTRACE_UNWIND_WORDS_PER_ROW, BACKTRACE_UNWIND_WORD_CFA_OFFSET, + BACKTRACE_UNWIND_WORD_PC_END, BACKTRACE_UNWIND_WORD_PC_START, BACKTRACE_UNWIND_WORD_RA_OFFSET, + BACKTRACE_UNWIND_WORD_RBP_OFFSET, BACKTRACE_UNWIND_WORD_REGISTERS, +}; + #[derive(Debug, thiserror::Error)] pub enum CompileError { #[error("Parse error: {0}")] @@ -95,6 +118,8 @@ pub struct CompileOptions { pub compare_cap: u32, /// Max total bytes in a single trace event (used for PerfEventArray accumulation buffer size). pub max_trace_event_size: u32, + /// Max DWARF-unwound frames captured by each `bt`/`backtrace` instruction. + pub backtrace_depth: u8, /// Optional single-address filter: if set, only the Nth (1-based) address /// resolved for a target will be compiled. When None, compile all. pub selected_index: Option, @@ -137,6 +162,7 @@ impl Default for CompileOptions { mem_dump_cap: 256, // Default per-arg dump cap (bytes) compare_cap: 64, // Default compare cap for strncmp/memcmp (bytes) max_trace_event_size: 32768, // Default event size cap (32KB) + backtrace_depth: DEFAULT_BACKTRACE_DEPTH, selected_index: None, pid_filter_spec: None, special_pid_ns: None, @@ -147,6 +173,9 @@ impl Default for CompileOptions { } } +pub const DEFAULT_BACKTRACE_DEPTH: u8 = 128; +pub const MAX_BACKTRACE_DEPTH: u8 = 128; + /// Main compilation interface with DwarfAnalyzer (multi-module support) /// /// This is the new multi-module interface that uses DwarfAnalyzer @@ -279,4 +308,39 @@ mod tests { "Use of variable 'x' outside of its scope" ); } + + #[test] + fn backtrace_unwind_row_layout_matches_bpf_map_value() { + assert_eq!(BACKTRACE_UNWIND_ROW_SIZE, 56); + assert_eq!(BACKTRACE_UNWIND_ROW_PC_START_OFFSET, 0); + assert_eq!(BACKTRACE_UNWIND_ROW_PC_END_OFFSET, 8); + assert_eq!(BACKTRACE_UNWIND_ROW_CFA_OFFSET_OFFSET, 16); + assert_eq!(BACKTRACE_UNWIND_ROW_RA_OFFSET_OFFSET, 24); + assert_eq!(BACKTRACE_UNWIND_ROW_RBP_OFFSET_OFFSET, 32); + assert_eq!(BACKTRACE_UNWIND_ROW_CFA_REGISTER_OFFSET, 40); + assert_eq!(BACKTRACE_UNWIND_ROW_RA_REGISTER_OFFSET, 42); + assert_eq!(BACKTRACE_UNWIND_ROW_RBP_REGISTER_OFFSET, 44); + assert_eq!(BACKTRACE_UNWIND_ROW_RA_KIND_OFFSET, 46); + assert_eq!(BACKTRACE_UNWIND_ROW_RBP_KIND_OFFSET, 47); + } + + #[test] + fn backtrace_tail_call_state_layout_matches_bpf_accessors() { + assert_eq!(BACKTRACE_TAIL_STATE_SIZE, 64); + assert_eq!(BACKTRACE_TAIL_STATE_CURRENT_IP_OFFSET, 0); + assert_eq!(BACKTRACE_TAIL_STATE_CURRENT_RSP_OFFSET, 8); + assert_eq!(BACKTRACE_TAIL_STATE_CURRENT_RBP_OFFSET, 16); + assert_eq!(BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, 24); + assert_eq!(BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET, 32); + assert_eq!(BACKTRACE_TAIL_STATE_INST_OFFSET_OFFSET, 40); + assert_eq!(BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, 44); + assert_eq!(BACKTRACE_TAIL_STATE_FRAME_COUNT_OFFSET, 48); + assert_eq!(BACKTRACE_TAIL_STATE_REQUESTED_DEPTH_OFFSET, 49); + assert_eq!(BACKTRACE_TAIL_STATE_OFFSETS_FOUND_OFFSET, 50); + assert_eq!(BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, 51); + assert_eq!(BACKTRACE_TAIL_STATE_FLAGS_OFFSET, 52); + assert_eq!(BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, 53); + assert_eq!(BACKTRACE_TAIL_STATE_ERROR_CODE_OFFSET, 54); + assert_eq!(BACKTRACE_TAIL_STATE_NEXT_SLOT_OFFSET, 56); + } } diff --git a/ghostscope-compiler/src/script/ast.rs b/ghostscope-compiler/src/script/ast.rs index 4059237f..1c1cd49b 100644 --- a/ghostscope-compiler/src/script/ast.rs +++ b/ghostscope-compiler/src/script/ast.rs @@ -65,7 +65,7 @@ pub enum VarType { #[derive(Debug, Clone)] pub enum Statement { Print(PrintStatement), // Updated to use new PrintStatement - Backtrace, + Backtrace(BacktraceStatement), Expr(Expr), VarDeclaration { name: String, @@ -89,6 +89,23 @@ pub enum Statement { Block(Vec), } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BacktraceStatement { + pub raw: bool, + pub full: bool, + pub inline: bool, +} + +impl Default for BacktraceStatement { + fn default() -> Self { + Self { + raw: false, + full: false, + inline: true, + } + } +} + /// Print statement variants for new instruction system #[derive(Debug, Clone)] pub enum PrintStatement { diff --git a/ghostscope-compiler/src/script/compiler.rs b/ghostscope-compiler/src/script/compiler.rs index ca066888..8789bc78 100644 --- a/ghostscope-compiler/src/script/compiler.rs +++ b/ghostscope-compiler/src/script/compiler.rs @@ -52,6 +52,12 @@ pub struct UProbeConfig { /// Trace context containing all strings, types, and variable names used in this uprobe pub trace_context: ghostscope_protocol::TraceContext, + /// BPF-facing compact DWARF CFI rows used by the `bt` unwinder. + pub backtrace_unwind_rows: Vec, + + /// Optional eBPF tail-call step program used by the `bt` unwinder. + pub backtrace_tail_call_program: Option, + /// Global 1-based index of this address within the resolved target list (if applicable) pub resolved_address_index: Option, } @@ -273,7 +279,7 @@ impl<'a> AstCompiler<'a> { fn top_level_statement_error(statement: &Statement) -> String { let kind = match statement { Statement::Print(_) => "print", - Statement::Backtrace => "backtrace", + Statement::Backtrace(_) => "backtrace", Statement::Expr(_) => "expression", Statement::VarDeclaration { .. } | Statement::AliasDeclaration { .. } => "let", Statement::If { .. } => "if", @@ -778,6 +784,8 @@ impl<'a> AstCompiler<'a> { ebpf_function_name, assigned_trace_id, trace_context, + backtrace_unwind_rows: codegen.backtrace_unwind_rows.clone(), + backtrace_tail_call_program: codegen.backtrace_tail_call_program(), resolved_address_index, }) } @@ -896,16 +904,6 @@ impl<'a> AstCompiler<'a> { use inkwell::targets::{FileType, Target, TargetTriple}; use inkwell::OptimizationLevel; - // Get LLVM IR string for logging and saving - let llvm_ir = module.print_to_string().to_string(); - let llvm_ir = llvm_ir.trim_end().to_string(); - info!( - "Successfully generated LLVM IR for {}, length: {}", - function_name, - llvm_ir.len() - ); - - // Save LLVM IR file if requested if compile_options.save_llvm_ir { let filename = Self::generate_filename_with_hint( target, @@ -913,12 +911,13 @@ impl<'a> AstCompiler<'a> { "ll", binary_path_hint, ); - if let Err(e) = std::fs::write(&filename, &llvm_ir) { + if let Err(e) = module.print_to_file(&filename) { warn!("Failed to save LLVM IR to {}: {}", filename, e); } else { info!("Saved LLVM IR to: {}", filename); } } + info!("Successfully generated LLVM module for {}", function_name); // Get target triple let triple = TargetTriple::create("bpf-pc-linux"); @@ -967,13 +966,6 @@ impl<'a> AstCompiler<'a> { info!("About to call LLVM write_to_memory_buffer..."); let object_code = { - // Print module for debugging before compilation - let module_string = module.print_to_string(); - debug!( - "Module IR before compilation:\n{}", - module_string.to_string() - ); - // Add a flush to ensure logs are written before potential crash use std::io::Write; let _ = std::io::stderr().flush(); diff --git a/ghostscope-compiler/src/script/grammar.pest b/ghostscope-compiler/src/script/grammar.pest index 058ec3b4..441748e1 100644 --- a/ghostscope-compiler/src/script/grammar.pest +++ b/ghostscope-compiler/src/script/grammar.pest @@ -38,7 +38,9 @@ print_content = { expr // print expression (covers variable, member, array, pointer, etc.) } format_expr = { string ~ "," ~ expr ~ ("," ~ expr)* } -backtrace_stmt = { ("backtrace" | "bt") ~ ";" } +backtrace_stmt = { ("backtrace" | "bt") ~ backtrace_arg* ~ ";" } +backtrace_arg = _{ backtrace_flag } +backtrace_flag = { "raw" | "full" | "inline" | "noinline" } expr_stmt = { expr ~ ";" } var_decl_stmt = { "let" ~ identifier ~ "=" ~ expr ~ ";" } assign_stmt = { identifier ~ "=" ~ expr ~ ";" } diff --git a/ghostscope-compiler/src/script/parser.rs b/ghostscope-compiler/src/script/parser.rs index 5c3a9cac..ebcc5212 100644 --- a/ghostscope-compiler/src/script/parser.rs +++ b/ghostscope-compiler/src/script/parser.rs @@ -4,7 +4,8 @@ use pest::RuleType; use pest_derive::Parser; use crate::script::ast::{ - infer_type, BinaryOp, Expr, PrintStatement, Program, Statement, TracePattern, + infer_type, BacktraceStatement, BinaryOp, Expr, PrintStatement, Program, Statement, + TracePattern, }; use crate::script::format_validator::FormatValidator; use tracing::{debug, info}; @@ -67,6 +68,9 @@ pub fn parse(input: &str) -> Result { if let Some(msg) = detect_unclosed_print_string(input) { return Err(ParseError::SyntaxError(msg)); } + if let Some(msg) = detect_backtrace_depth_argument(input) { + return Err(ParseError::SyntaxError(msg)); + } // Heuristic: detect likely misspelled or unknown keywords and suggest fixes if let Some(msg) = detect_unknown_keyword(input) { return Err(ParseError::SyntaxError(msg)); @@ -129,6 +133,41 @@ fn detect_unclosed_print_string(input: &str) -> Option { None } +fn detect_backtrace_depth_argument(input: &str) -> Option { + fn boundary_before(line: &str, idx: usize) -> bool { + idx == 0 + || line[..idx] + .chars() + .next_back() + .is_some_and(|ch| ch.is_whitespace() || matches!(ch, '{' | ';' | '}')) + } + + for (line_idx, raw_line) in input.lines().enumerate() { + let line = raw_line.split("//").next().unwrap_or(raw_line); + for command in ["bt", "backtrace"] { + for (idx, _) in line.match_indices(command) { + if !boundary_before(line, idx) { + continue; + } + let after = &line[idx + command.len()..]; + if !after.starts_with(char::is_whitespace) { + continue; + } + let arg = after.trim_start(); + if arg.starts_with("depth") + || arg.chars().next().is_some_and(|ch| ch.is_ascii_digit()) + { + return Some(format!( + "bt depth is no longer a script option at line {}. Set the global limit with --backtrace-depth or [ebpf] backtrace_depth = N.", + line_idx + 1 + )); + } + } + } + } + None +} + // Try to detect lines that start with an unknown/misspelled keyword and suggest known ones. fn detect_unknown_keyword(input: &str) -> Option { // Suggest only currently supported top-level keywords. @@ -276,6 +315,28 @@ fn detect_unknown_keyword(input: &str) -> Option { None } +fn parse_backtrace_stmt(pair: Pair) -> Result { + let mut stmt = BacktraceStatement::default(); + + for arg in pair.into_inner() { + if arg.as_rule() == Rule::backtrace_flag { + match arg.as_str() { + "raw" => stmt.raw = true, + "full" => stmt.full = true, + "inline" => stmt.inline = true, + "noinline" => stmt.inline = false, + other => { + return Err(ParseError::SyntaxError(format!( + "Unknown bt option '{other}'" + ))) + } + } + } + } + + Ok(stmt) +} + fn parse_statement(pair: Pair) -> Result { debug!( "parse_statement: {:?} = '{}'", @@ -326,7 +387,7 @@ fn parse_statement(pair: Pair) -> Result { let print_stmt = parse_print_content(print_content)?; Ok(Statement::Print(print_stmt)) } - Rule::backtrace_stmt => Ok(Statement::Backtrace), + Rule::backtrace_stmt => Ok(Statement::Backtrace(parse_backtrace_stmt(inner)?)), Rule::assign_stmt => { // Friendly error for immutable variables (no assignment supported) let mut it = inner.into_inner(); @@ -2692,13 +2753,65 @@ trace foo { #[test] fn parse_backtrace_and_bt_statements() { let s = r#" -trace foo { - backtrace; - bt; -} -"#; + trace foo { + backtrace; + bt; + bt raw; + bt full noinline; + } + "#; + let program = parse(s).expect("parse ok"); + let Statement::TracePoint { body, .. } = &program.statements[0] else { + panic!("expected trace"); + }; + assert_eq!(body.len(), 4); + match &body[2] { + Statement::Backtrace(bt) => { + assert!(bt.raw); + assert!(bt.inline); + } + other => panic!("expected backtrace, got {other:?}"), + } + match &body[3] { + Statement::Backtrace(bt) => { + assert!(bt.full); + assert!(!bt.inline); + } + other => panic!("expected backtrace, got {other:?}"), + } + } + + #[test] + fn parse_backtrace_rejects_named_depth_option() { + let s = r#" + trace foo { + bt depth=8; + } + "#; let r = parse(s); - assert!(r.is_ok(), "parse failed: {:?}", r.err()); + match r { + Err(ParseError::SyntaxError(msg)) => { + assert!(msg.contains("no longer a script option"), "{msg}"); + assert!(msg.contains("--backtrace-depth"), "{msg}"); + } + other => panic!("expected SyntaxError, got {other:?}"), + } + } + + #[test] + fn parse_backtrace_rejects_positional_depth_option() { + let s = r#" + trace foo { + bt 4 raw; + } + "#; + let r = parse(s); + match r { + Err(ParseError::SyntaxError(msg)) => { + assert!(msg.contains("no longer a script option"), "{msg}"); + } + other => panic!("expected SyntaxError, got {other:?}"), + } } #[test] diff --git a/ghostscope-dwarf/src/analyzer/mod.rs b/ghostscope-dwarf/src/analyzer/mod.rs index d6e8cb5e..5fe72d07 100644 --- a/ghostscope-dwarf/src/analyzer/mod.rs +++ b/ghostscope-dwarf/src/analyzer/mod.rs @@ -10,9 +10,9 @@ use crate::{ }; use ghostscope_debuginfod::DebuginfodClient; use object::{Object, ObjectSection}; -use std::collections::HashMap; +use std::collections::{HashMap, VecDeque}; use std::path::{Path, PathBuf}; -use std::sync::Arc; +use std::sync::{Arc, RwLock}; mod module_resolution; mod plan_global; @@ -87,6 +87,15 @@ pub struct AddressQueryResult { pub parameters: Vec, } +/// Runtime mapping metadata for a loaded module. +#[derive(Debug, Clone)] +pub struct LoadedModuleRuntimeInfo { + pub module_path: PathBuf, + pub loaded_address: Option, + pub load_bias: Option, + pub size: u64, +} + /// Rich query result for a function lookup across modules. #[derive(Debug, Clone)] pub struct FunctionQueryResult { @@ -101,6 +110,74 @@ pub struct DwarfAnalyzer { pid: u32, /// Module path -> module data mapping modules: HashMap, + /// Cached PC semantic contexts for repeated symbol/source lookups. + pc_context_cache: RwLock, +} + +const PC_CONTEXT_CACHE_MAX_ENTRIES: usize = 8192; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct PcContextCacheKey { + module_path: PathBuf, + address: u64, +} + +#[derive(Debug)] +struct PcContextCache { + entries: HashMap>, + insertion_order: VecDeque, + len: usize, + max_entries: usize, +} + +impl Default for PcContextCache { + fn default() -> Self { + Self { + entries: HashMap::new(), + insertion_order: VecDeque::new(), + len: 0, + max_entries: PC_CONTEXT_CACHE_MAX_ENTRIES, + } + } +} + +impl PcContextCache { + fn get(&self, module_path: &Path, address: u64) -> Option { + self.entries + .get(module_path) + .and_then(|entries| entries.get(&address)) + .cloned() + } + + fn insert(&mut self, module_path: PathBuf, address: u64, context: PcContext) { + if self.max_entries == 0 { + return; + } + + let key = PcContextCacheKey { + module_path, + address, + }; + let module_entries = self.entries.entry(key.module_path.clone()).or_default(); + if module_entries.insert(address, context).is_none() { + self.insertion_order.push_back(key.clone()); + self.len += 1; + } + + while self.len > self.max_entries { + let Some(expired) = self.insertion_order.pop_front() else { + break; + }; + if let Some(module_entries) = self.entries.get_mut(&expired.module_path) { + if module_entries.remove(&expired.address).is_some() { + self.len -= 1; + } + if module_entries.is_empty() { + self.entries.remove(&expired.module_path); + } + } + } + } } impl DwarfAnalyzer { @@ -382,6 +459,7 @@ impl DwarfAnalyzer { std::path::PathBuf::from(&e.module_path), ); mm.loaded_address = Some(e.base); + mm.load_bias = Some(e.offsets.text); mm.size = e.size; module_mappings.push(mm); } @@ -506,6 +584,7 @@ impl DwarfAnalyzer { let mut analyzer = Self { pid: 0, // No specific PID in exec mode modules: HashMap::new(), + pc_context_cache: RwLock::new(PcContextCache::default()), }; // Create a single module mapping for the executable @@ -513,7 +592,8 @@ impl DwarfAnalyzer { let module_mapping = ModuleMapping { path: exec_path.clone(), loaded_address: None, // No process mapping in exec path mode - size: 0, // Will be determined from file size if needed + load_bias: None, + size: 0, // Will be determined from file size if needed }; let module_path = exec_path.to_string_lossy().to_string(); @@ -586,6 +666,7 @@ impl DwarfAnalyzer { let mut analyzer = Self { pid, modules: HashMap::new(), + pc_context_cache: RwLock::new(PcContextCache::default()), }; for module in modules { @@ -698,7 +779,7 @@ impl DwarfAnalyzer { pub fn compact_unwind_table_for_context( &self, ctx: &PcContext, - ) -> Result> { + ) -> Result>> { let module_path = self .module_path_for_id(ctx.module) .ok_or_else(|| anyhow::anyhow!("Semantic module id {:?} is not loaded", ctx.module))?; @@ -726,7 +807,7 @@ impl DwarfAnalyzer { pub fn compact_unwind_table_for_module( &self, module: crate::ModuleId, - ) -> Result> { + ) -> Result>> { let module_path = self .module_path_for_id(module) .ok_or_else(|| anyhow::anyhow!("Semantic module id {:?} is not loaded", module))?; @@ -741,6 +822,32 @@ impl DwarfAnalyzer { self.modules.keys().collect() } + /// Get loaded module paths with process mapping metadata, when available. + pub fn loaded_module_runtime_info(&self) -> Vec { + let mut modules: Vec<_> = self + .modules + .values() + .map(|module| { + let mapping = module.module_mapping(); + LoadedModuleRuntimeInfo { + module_path: mapping.path.clone(), + loaded_address: mapping.loaded_address, + load_bias: mapping.load_bias, + size: mapping.size, + } + }) + .collect(); + modules.sort_by(|left, right| left.module_path.cmp(&right.module_path)); + modules + } + + /// Get the ELF entry address for a loaded module, when present. + pub fn module_entry_address>(&self, module_path: P) -> Option { + self.modules + .get(module_path.as_ref()) + .and_then(|module| module.entry_address()) + } + /// Classify the section type for a link-time virtual address in a specific module pub fn classify_section_for_address>( &self, diff --git a/ghostscope-dwarf/src/analyzer/plan_pc.rs b/ghostscope-dwarf/src/analyzer/plan_pc.rs index 535c1408..8fba6bca 100644 --- a/ghostscope-dwarf/src/analyzer/plan_pc.rs +++ b/ghostscope-dwarf/src/analyzer/plan_pc.rs @@ -2,8 +2,9 @@ use super::DwarfAnalyzer; use crate::{ core::{ModuleAddress, Provenance, Result}, semantics::{ - AddressSpaceInfo, PcContext, PcLineInfo, PcRange, PlanError, VariableAccessPath, - VariableAccessSegment, VariableReadPlan, VisibleVariable, VisibleVariablesResult, + AddressSpaceInfo, FunctionParameter, PcContext, PcLineInfo, PcRange, PlanError, + VariableAccessPath, VariableAccessSegment, VariableReadPlan, VisibleVariable, + VisibleVariablesResult, }, }; use std::path::Path; @@ -15,6 +16,28 @@ impl DwarfAnalyzer { /// query APIs, so `pc` and `normalized_pc` intentionally match. Runtime /// rebasing details are preserved in `address_space` for future lowering. pub fn resolve_pc(&self, module_address: &ModuleAddress) -> Result { + if let Some(context) = self + .pc_context_cache + .read() + .expect("PC context cache lock poisoned") + .get(&module_address.module_path, module_address.address) + { + return Ok(context); + } + + let context = self.resolve_pc_uncached(module_address)?; + self.pc_context_cache + .write() + .expect("PC context cache lock poisoned") + .insert( + module_address.module_path.clone(), + module_address.address, + context.clone(), + ); + Ok(context) + } + + fn resolve_pc_uncached(&self, module_address: &ModuleAddress) -> Result { let module_data = self .modules .get(&module_address.module_path) @@ -76,6 +99,25 @@ impl DwarfAnalyzer { Ok(self.visible_variables_with_diagnostics(ctx)?.variables) } + /// Return only function formal parameters for display. + /// + /// This does not evaluate locations, compute frame-base/CFA, or resolve + /// visible lexical variables. It is intended for inexpensive function + /// signature rendering in hot paths such as backtrace output. + pub fn function_parameters(&self, ctx: &PcContext) -> Result> { + let Some(function) = ctx.function else { + return Ok(Vec::new()); + }; + let module_address = self.module_address_for_context(ctx)?; + + self.modules + .get(&module_address.module_path) + .ok_or_else(|| { + anyhow::anyhow!("Module {} not loaded", module_address.module_display()) + })? + .function_parameters(function) + } + /// Return variables visible at a PC context plus non-fatal DWARF diagnostics. pub fn visible_variables_with_diagnostics( &self, diff --git a/ghostscope-dwarf/src/core/mapping.rs b/ghostscope-dwarf/src/core/mapping.rs index 7c87bfd8..4d9da32b 100644 --- a/ghostscope-dwarf/src/core/mapping.rs +++ b/ghostscope-dwarf/src/core/mapping.rs @@ -4,6 +4,7 @@ use std::path::PathBuf; pub struct ModuleMapping { pub path: PathBuf, pub loaded_address: Option, + pub load_bias: Option, pub size: u64, } @@ -12,6 +13,7 @@ impl ModuleMapping { Self { path, loaded_address: None, + load_bias: None, size: 0, } } diff --git a/ghostscope-dwarf/src/index/cfi_index.rs b/ghostscope-dwarf/src/index/cfi_index.rs index 4f336d44..7d8b33bd 100644 --- a/ghostscope-dwarf/src/index/cfi_index.rs +++ b/ghostscope-dwarf/src/index/cfi_index.rs @@ -17,7 +17,7 @@ use gimli::{ Register, RegisterRule, UnwindContext, UnwindSection, }; use object::{Object, ObjectSection}; -use std::{collections::BTreeMap, sync::Arc}; +use std::{collections::BTreeMap, sync::Arc, time::Instant}; use tracing::{debug, info, warn}; /// CFI index for fast CFA rule lookup @@ -218,13 +218,16 @@ impl CfiIndex { /// Compile all FDE rows into a compact unwind table for userspace/BPF planning. pub fn compact_unwind_table(&self, module: ModuleId) -> Result { + let started_at = Instant::now(); let mut rows = Vec::new(); let mut diagnostics = Vec::new(); let mut entries = self.eh_frame.entries(&self.bases); + let mut fde_count = 0usize; while let Some(entry) = entries.next().context("Failed to iterate FDE entries")? { match entry { CieOrFde::Fde(partial_fde) => { + fde_count += 1; let fde = partial_fde .parse(|_, bases, offset| self.eh_frame.cie_from_offset(bases, offset)) .context("Failed to parse FDE")?; @@ -235,6 +238,14 @@ impl CfiIndex { } rows.sort_by_key(|row| (row.pc_start, row.pc_end)); + info!( + ?module, + fdes = fde_count, + rows = rows.len(), + diagnostics = diagnostics.len(), + elapsed_ms = started_at.elapsed().as_millis(), + "Built compact DWARF unwind table for bt" + ); Ok(CompactUnwindTable { module, rows, diff --git a/ghostscope-dwarf/src/lib.rs b/ghostscope-dwarf/src/lib.rs index a6d2a76e..af923668 100644 --- a/ghostscope-dwarf/src/lib.rs +++ b/ghostscope-dwarf/src/lib.rs @@ -22,9 +22,9 @@ pub(crate) mod analyzer; // Re-export main public API only pub use analyzer::{ AddressQueryResult, AnalyzerStats, DwarfAnalyzer, ExecutableFileInfo, FunctionQueryResult, - MainExecutableInfo, ModuleDefaultPolicy, ModuleLoadingEvent, ModuleLoadingStats, ModuleStats, - SectionInfo, SharedLibraryInfo, SimpleFileInfo, SourceLineAddressSearch, SourceLineQuerySearch, - TypeLookupAmbiguity, + LoadedModuleRuntimeInfo, MainExecutableInfo, ModuleDefaultPolicy, ModuleLoadingEvent, + ModuleLoadingStats, ModuleStats, SectionInfo, SharedLibraryInfo, SimpleFileInfo, + SourceLineAddressSearch, SourceLineQuerySearch, TypeLookupAmbiguity, }; // Re-export essential core and semantic support types. @@ -43,12 +43,13 @@ pub use semantics::{ is_c_pointer_or_array_type, is_c_signed_integer_type, member_layout, strip_type_aliases, usual_c_arithmetic_comparison_plan, AddressOrigin, AddressSpaceInfo, CIntegerComparisonPlan, CIntegerComparisonType, CfaRulePlan, CompactUnwindRow, CompactUnwindStats, CompactUnwindTable, - IndexableElementLayout, InlineFrame, LvalueAddressPlan, MemberLayout, PcContext, PcLineInfo, - PcRange, PlannedAddress, PlannedAddressKind, PlannedValue, RegisterRecoveryPlan, - RuntimeComputedExpr, RuntimeComputedKind, TypeLayoutError, UnwindDiagnostic, - UnwindDiagnosticKind, VariableAccessPath, VariableAccessSegment, VariableLoweringKind, - VariableLoweringPlan, VariableMaterialization, VariableMaterializationPlan, VariablePlan, - VariableQueryDiagnostic, VariableReadPlan, VisibleVariable, VisibleVariablesResult, + FunctionParameter, IndexableElementLayout, InlineFrame, LvalueAddressPlan, MemberLayout, + PcContext, PcLineInfo, PcRange, PlannedAddress, PlannedAddressKind, PlannedValue, + RegisterRecoveryPlan, RuntimeComputedExpr, RuntimeComputedKind, TypeLayoutError, + UnwindDiagnostic, UnwindDiagnosticKind, VariableAccessPath, VariableAccessSegment, + VariableLoweringKind, VariableLoweringPlan, VariableMaterialization, + VariableMaterializationPlan, VariablePlan, VariableQueryDiagnostic, VariableReadPlan, + VisibleVariable, VisibleVariablesResult, }; // Re-export type definitions from protocol (avoiding circular dependencies) diff --git a/ghostscope-dwarf/src/objfile/function_lookup.rs b/ghostscope-dwarf/src/objfile/function_lookup.rs index 4c60886c..0c019edc 100644 --- a/ghostscope-dwarf/src/objfile/function_lookup.rs +++ b/ghostscope-dwarf/src/objfile/function_lookup.rs @@ -163,6 +163,16 @@ impl LoadedObjfile { return Ok(Vec::new()); } + let cache_key = (entry.unit_offset.0 as u64, entry.die_offset.0 as u64); + if let Some(ranges) = self + .function_ranges_cache + .read() + .expect("function range cache lock poisoned") + .get(&cache_key) + { + return Ok(ranges.clone()); + } + let dwarf = self.dwarf(); let header = dwarf .unit_header(entry.unit_offset) @@ -174,8 +184,13 @@ impl LoadedObjfile { .entry(entry.die_offset) .map_err(|e| anyhow::anyhow!("entry load error: {}", e))?; - RangeExtractor::extract_all_ranges(&die, &unit, dwarf) - .map_err(|e| anyhow::anyhow!("range extraction error: {}", e)) + let ranges = RangeExtractor::extract_all_ranges(&die, &unit, dwarf) + .map_err(|e| anyhow::anyhow!("range extraction error: {}", e))?; + self.function_ranges_cache + .write() + .expect("function range cache lock poisoned") + .insert(cache_key, ranges.clone()); + Ok(ranges) } pub(super) fn find_function_index_entry_by_address( diff --git a/ghostscope-dwarf/src/objfile/loaded.rs b/ghostscope-dwarf/src/objfile/loaded.rs index aa3e70ca..6c12a588 100644 --- a/ghostscope-dwarf/src/objfile/loaded.rs +++ b/ghostscope-dwarf/src/objfile/loaded.rs @@ -4,9 +4,9 @@ use crate::{ binary::{DwarfReader, MappedFile}, core::{mapping::ModuleMapping, Result}, index::{ - BlockIndex, CfiIndex, LightweightIndex, LineMappingTable, ScopedFileIndexManager, - TypeNameIndex, + BlockIndex, LightweightIndex, LineMappingTable, ScopedFileIndexManager, TypeNameIndex, }, + objfile::ModuleUnwindInfo, parser::{CompilationUnit, DetailedParser}, }; use object::{Object, ObjectSegment}; @@ -16,6 +16,8 @@ use std::{ sync::{Arc, RwLock}, }; +type FunctionRangeCacheKey = (u64, u64); + /// Complete DWARF data for a single loaded object file. #[derive(Debug)] pub(crate) struct LoadedObjfile { @@ -24,12 +26,14 @@ pub(crate) struct LoadedObjfile { pub(super) line_mapping: LineMappingTable, pub(super) scoped_file_manager: ScopedFileIndexManager, pub(super) compilation_units: HashMap, - pub(super) cfi_index: Option, + pub(super) unwind_info: ModuleUnwindInfo, pub(super) dwarf: gimli::Dwarf, pub(super) detailed_parser: DetailedParser, pub(super) _dwarf_mapped_file: Arc, pub(super) _binary_mapped_file: Arc, + pub(super) entry_address: Option, pub(super) text_symbol_starts_by_name: HashMap>, + pub(super) function_ranges_cache: RwLock>>, pub(super) block_index: RwLock, pub(super) type_name_index: Arc, pub(super) load_parse_ms: u64, @@ -46,6 +50,10 @@ impl LoadedObjfile { &self.module_mapping } + pub(crate) fn entry_address(&self) -> Option { + self.entry_address + } + pub(crate) fn get_function_names(&self) -> Vec<&String> { self.lightweight_index.get_function_names() } @@ -86,10 +94,7 @@ impl LoadedObjfile { } pub(crate) fn get_cfa_result(&self, pc: u64) -> Result> { - match &self.cfi_index { - Some(cfi) => Ok(Some(cfi.get_cfa_result(pc)?)), - None => Ok(None), - } + self.unwind_info.get_cfa_result(pc) } pub(crate) fn recover_caller_frame( @@ -97,20 +102,14 @@ impl LoadedObjfile { pc: u64, registers: &[u16], ) -> Result> { - match &self.cfi_index { - Some(cfi) => Ok(Some(cfi.recover_caller_frame(pc, registers)?)), - None => Ok(None), - } + self.unwind_info.recover_caller_frame(pc, registers) } pub(crate) fn compact_unwind_table( &self, module: crate::ModuleId, - ) -> Result> { - match &self.cfi_index { - Some(cfi) => Ok(Some(cfi.compact_unwind_table(module)?)), - None => Ok(None), - } + ) -> Result>> { + self.unwind_info.compact_unwind_table(module) } pub(crate) fn compact_unwind_row( @@ -118,9 +117,7 @@ impl LoadedObjfile { module: crate::ModuleId, pc: u64, ) -> Result> { - Ok(self - .compact_unwind_table(module)? - .and_then(|table| table.row_for_pc(pc).cloned())) + self.unwind_info.compact_unwind_row(module, pc) } pub(crate) fn vaddr_to_file_offset(&self, vaddr: u64) -> Option { diff --git a/ghostscope-dwarf/src/objfile/loading.rs b/ghostscope-dwarf/src/objfile/loading.rs index 0e28871d..7116fa1f 100644 --- a/ghostscope-dwarf/src/objfile/loading.rs +++ b/ghostscope-dwarf/src/objfile/loading.rs @@ -5,7 +5,8 @@ use crate::{ empty_dwarf_reader_with_endian, try_load_debug_file, DwarfData, MappedFile, }, core::{mapping::ModuleMapping, Result}, - index::{BlockIndex, CfiIndex, TypeNameIndex}, + index::{BlockIndex, TypeNameIndex}, + objfile::ModuleUnwindInfo, parser::DetailedParser, }; use ghostscope_debuginfod::{build_id_to_hex, DebuginfodClient}; @@ -13,7 +14,7 @@ use object::{Object, ObjectSection, ObjectSymbol, SymbolKind}; use std::{borrow::Cow, collections::HashMap, path::Path, sync::Arc, time::Instant}; impl LoadedObjfile { - /// Parallel loading: debug_info || debug_line || CFI simultaneously + /// Parallel loading: debug_info || debug_line || CFI simultaneously. pub(crate) async fn load_parallel( module_mapping: ModuleMapping, debug_search_paths: &[String], @@ -112,7 +113,7 @@ impl LoadedObjfile { "Starting parallel DWARF parsing with true debug_line || debug_info parallelism..." ); - let (pair_result, cfi_index_result) = tokio::try_join!( + let (pair_result, unwind_info) = tokio::try_join!( tokio::task::spawn_blocking({ let dwarf = Arc::clone(&dwarf); let module_path = module_mapping.path.to_string_lossy().to_string(); @@ -137,30 +138,11 @@ impl LoadedObjfile { tokio::task::spawn_blocking({ let binary_for_cfi = Arc::clone(&binary_mapped); let module_path = module_mapping.path.clone(); - move || -> Result> { - match CfiIndex::from_mapped_file(binary_for_cfi) { - Ok(cfi) => { - tracing::info!( - "CFI index initialized successfully for {}", - module_path.display() - ); - Ok(Some(cfi)) - } - Err(e) => { - tracing::warn!( - "Failed to initialize CFI index for {}: {}", - module_path.display(), - e - ); - Ok(None) - } - } - } + move || ModuleUnwindInfo::from_mapped_file(binary_for_cfi, &module_path) }) )?; let (line_result, info_result) = pair_result?; - let cfi_index = cfi_index_result?; let parse_result = crate::parser::DwarfParser::combine_parallel_results( line_result, @@ -169,7 +151,7 @@ impl LoadedObjfile { ); let parse_elapsed_ms = load_started_at.elapsed().as_millis(); - if let Some(ref cfi) = cfi_index { + if let Some(cfi) = unwind_info.cfi_index() { let stats = cfi.get_stats(); tracing::info!( "CFI stats: has_eh_frame_hdr={}, fast_lookup={}", @@ -199,10 +181,11 @@ impl LoadedObjfile { Arc::try_unwrap(dwarf).map_err(|_| anyhow::anyhow!("Failed to unwrap DWARF Arc"))?; let mut detailed_parser = DetailedParser::new(); detailed_parser.set_type_name_index(Arc::clone(&type_name_index)); + let entry_address = Self::read_entry_address(&binary_mapped); let text_symbol_starts_by_name = Self::collect_text_symbol_starts(&binary_mapped); let mut warnings = Vec::new(); - if cfi_index.is_none() { + if !unwind_info.has_cfi() { warnings.push("CFI index failed to initialize".to_string()); } @@ -230,14 +213,16 @@ impl LoadedObjfile { line_mapping, scoped_file_manager, compilation_units, - cfi_index, + unwind_info, dwarf, detailed_parser, block_index: std::sync::RwLock::new(BlockIndex::new()), type_name_index, _dwarf_mapped_file: mapped_file, _binary_mapped_file: binary_mapped, + entry_address, text_symbol_starts_by_name, + function_ranges_cache: std::sync::RwLock::new(HashMap::new()), load_parse_ms: parse_elapsed_ms as u64, load_index_ms: index_elapsed_ms as u64, load_total_ms, @@ -259,6 +244,11 @@ impl LoadedObjfile { Ok(module) } + fn read_entry_address(binary_mapped: &MappedFile) -> Option { + let address = binary_mapped.parse_object().ok()?.entry(); + (address != 0).then_some(address) + } + fn has_debug_info(dwarf: &DwarfData) -> bool { matches!(dwarf.units().next(), Ok(Some(_))) } diff --git a/ghostscope-dwarf/src/objfile/mod.rs b/ghostscope-dwarf/src/objfile/mod.rs index 5e667b32..c257d28a 100644 --- a/ghostscope-dwarf/src/objfile/mod.rs +++ b/ghostscope-dwarf/src/objfile/mod.rs @@ -5,6 +5,8 @@ pub(crate) mod globals; pub(crate) mod loaded; pub(crate) mod loading; pub(crate) mod source_location; +pub(crate) mod unwind; pub(crate) mod variables; pub(crate) use loaded::LoadedObjfile; +pub(crate) use unwind::ModuleUnwindInfo; diff --git a/ghostscope-dwarf/src/objfile/unwind.rs b/ghostscope-dwarf/src/objfile/unwind.rs new file mode 100644 index 00000000..d51650ec --- /dev/null +++ b/ghostscope-dwarf/src/objfile/unwind.rs @@ -0,0 +1,111 @@ +//! Module-local unwind data derived from `.eh_frame`. +//! +//! This intentionally stays separate from `.debug_info`/`.debug_line` data: +//! CFI is enough for stack unwinding even when source DWARF is unavailable. + +use crate::{ + binary::MappedFile, + core::{CallerFrameRecovery, CfaResult, Result}, + index::CfiIndex, + CompactUnwindRow, CompactUnwindTable, ModuleId, +}; +use std::{ + collections::HashMap, + path::Path, + sync::{Arc, RwLock}, +}; + +#[derive(Debug)] +pub(crate) struct ModuleUnwindInfo { + cfi_index: Option, + compact_unwind_tables: RwLock>>, +} + +impl ModuleUnwindInfo { + pub(crate) fn from_mapped_file(file_data: Arc, module_path: &Path) -> Self { + let cfi_index = match CfiIndex::from_mapped_file(file_data) { + Ok(cfi) => { + tracing::info!( + "CFI index initialized successfully for {}", + module_path.display() + ); + Some(cfi) + } + Err(error) => { + tracing::warn!( + "Failed to initialize CFI index for {}: {}", + module_path.display(), + error + ); + None + } + }; + + Self { + cfi_index, + compact_unwind_tables: RwLock::new(HashMap::new()), + } + } + + pub(crate) fn has_cfi(&self) -> bool { + self.cfi_index.is_some() + } + + pub(crate) fn cfi_index(&self) -> Option<&CfiIndex> { + self.cfi_index.as_ref() + } + + pub(crate) fn get_cfa_result(&self, pc: u64) -> Result> { + match &self.cfi_index { + Some(cfi) => Ok(Some(cfi.get_cfa_result(pc)?)), + None => Ok(None), + } + } + + pub(crate) fn recover_caller_frame( + &self, + pc: u64, + registers: &[u16], + ) -> Result> { + match &self.cfi_index { + Some(cfi) => Ok(Some(cfi.recover_caller_frame(pc, registers)?)), + None => Ok(None), + } + } + + pub(crate) fn compact_unwind_table( + &self, + module: ModuleId, + ) -> Result>> { + let Some(cfi) = &self.cfi_index else { + return Ok(None); + }; + + if let Some(table) = self + .compact_unwind_tables + .read() + .expect("compact unwind table cache lock poisoned") + .get(&module) + .cloned() + { + return Ok(Some(table)); + } + + let table = Arc::new(cfi.compact_unwind_table(module)?); + self.compact_unwind_tables + .write() + .expect("compact unwind table cache lock poisoned") + .insert(module, Arc::clone(&table)); + Ok(Some(table)) + } + + pub(crate) fn compact_unwind_row( + &self, + module: ModuleId, + pc: u64, + ) -> Result> { + Ok(self + .compact_unwind_table(module)? + .and_then(|table| table.row_for_pc(pc).cloned())) + } +} diff --git a/ghostscope-dwarf/src/objfile/variables.rs b/ghostscope-dwarf/src/objfile/variables.rs index 0793c562..9ef1df03 100644 --- a/ghostscope-dwarf/src/objfile/variables.rs +++ b/ghostscope-dwarf/src/objfile/variables.rs @@ -7,7 +7,8 @@ use crate::{ parser::ExpressionEvaluator, semantics::{ resolve_attr_with_unit_origins, resolve_name_with_origins, resolve_origin_entry, - resolve_type_ref_with_origins, InlineFrame, PcLineInfo, VariableQueryDiagnostic, + resolve_type_ref_with_origins, FunctionParameter, InlineFrame, PcLineInfo, + VariableQueryDiagnostic, }, }; use gimli::Reader; @@ -390,7 +391,7 @@ impl LoadedObjfile { let fb_result = self.compute_frame_base_for_pc(&func, address); let cfa_result = if fb_result.is_none() { - if self.cfi_index.is_some() { + if self.unwind_info.has_cfi() { match self.get_cfa_result(address) { Ok(Some(cfa)) => Some(cfa), _ => None, @@ -412,7 +413,7 @@ impl LoadedObjfile { }; let var_refs = func.variables_at_pc_with_scope_depth(address); - let cfi_index = self.cfi_index.clone(); + let cfi_index = self.unwind_info.cfi_index().cloned(); let dwarf_ref = self.dwarf(); let mut variables = Vec::with_capacity(var_refs.len()); let mut diagnostics = Vec::new(); @@ -483,6 +484,95 @@ impl LoadedObjfile { Ok((variables, diagnostics)) } + pub(crate) fn function_parameters( + &self, + function: FunctionId, + ) -> Result> { + let cu_off = gimli::DebugInfoOffset(function.declaration.cu.0 as usize); + let die_off = gimli::UnitOffset(function.declaration.offset as usize); + let dwarf = self.dwarf(); + let header = dwarf.unit_header(cu_off)?; + let unit = dwarf.unit(header)?; + let entry = unit.entry(die_off)?; + + let params = self.direct_function_parameters(dwarf, &unit, &entry)?; + if !params.is_empty() { + return Ok(params); + } + + for origin_attr in [ + gimli::constants::DW_AT_abstract_origin, + gimli::constants::DW_AT_specification, + ] { + if let Some(value) = entry.attr_value(origin_attr) { + if let Some((_, origin_unit, origin_entry)) = + resolve_origin_entry(dwarf, &unit, value)? + { + let params = + self.direct_function_parameters(dwarf, &origin_unit, &origin_entry)?; + if !params.is_empty() { + return Ok(params); + } + } + } + } + + Ok(Vec::new()) + } + + fn direct_function_parameters( + &self, + dwarf: &gimli::Dwarf, + unit: &gimli::Unit, + entry: &gimli::DebuggingInformationEntry, + ) -> Result> { + let mut params = Vec::new(); + let mut tree = unit.entries_tree(Some(entry.offset()))?; + let root = tree.root()?; + let mut children = root.children(); + + while let Some(child) = children.next()? { + let child_entry = child.entry(); + if child_entry.tag() != gimli::constants::DW_TAG_formal_parameter { + continue; + } + + params.push(self.function_parameter_from_entry(dwarf, unit, child_entry)?); + } + + Ok(params) + } + + fn function_parameter_from_entry( + &self, + dwarf: &gimli::Dwarf, + unit: &gimli::Unit, + entry: &gimli::DebuggingInformationEntry, + ) -> Result { + let name = resolve_name_with_origins(dwarf, unit, entry)?.unwrap_or_default(); + let type_name = resolve_type_ref_with_origins(dwarf, entry, unit)? + .and_then(|loc| self.detailed_shallow_type(loc.cu_off, loc.die_off)) + .map(|ty| ty.type_name()) + .unwrap_or_else(|| "unknown".to_string()); + let is_artificial = + resolve_attr_with_unit_origins(entry, unit, gimli::constants::DW_AT_artificial)? + .is_some_and(|value| match value { + gimli::AttributeValue::Flag(v) => v, + gimli::AttributeValue::Data1(v) => v != 0, + gimli::AttributeValue::Data2(v) => v != 0, + gimli::AttributeValue::Data4(v) => v != 0, + gimli::AttributeValue::Data8(v) => v != 0, + gimli::AttributeValue::Udata(v) => v != 0, + _ => false, + }); + + Ok(FunctionParameter { + name, + type_name, + is_artificial, + }) + } + fn variable_name_for_ref(&self, var_ref: &VarRef) -> Option { let dwarf = self.dwarf(); let header = dwarf.unit_header(var_ref.cu_offset).ok()?; diff --git a/ghostscope-dwarf/src/semantics/mod.rs b/ghostscope-dwarf/src/semantics/mod.rs index c71a8c56..b222df8c 100644 --- a/ghostscope-dwarf/src/semantics/mod.rs +++ b/ghostscope-dwarf/src/semantics/mod.rs @@ -17,7 +17,9 @@ pub(crate) use origins::{ resolve_attr_with_unit_origins, resolve_name_with_origins, resolve_origin_entry, }; pub(crate) use pc::{range_contains_pc, ranges_contain_pc}; -pub use pc_context::{AddressSpaceInfo, InlineFrame, PcContext, PcLineInfo, PcRange}; +pub use pc_context::{ + AddressSpaceInfo, FunctionParameter, InlineFrame, PcContext, PcLineInfo, PcRange, +}; pub(crate) use types::{resolve_type_ref_in_same_unit_with_origins, resolve_type_ref_with_origins}; pub use unwind_plan::{ CfaRulePlan, CompactUnwindRow, CompactUnwindStats, CompactUnwindTable, RegisterRecoveryPlan, diff --git a/ghostscope-dwarf/src/semantics/pc_context.rs b/ghostscope-dwarf/src/semantics/pc_context.rs index 28fa604a..76525d1e 100644 --- a/ghostscope-dwarf/src/semantics/pc_context.rs +++ b/ghostscope-dwarf/src/semantics/pc_context.rs @@ -22,6 +22,13 @@ pub struct PcContext { pub address_space: AddressSpaceInfo, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FunctionParameter { + pub name: String, + pub type_name: String, + pub is_artificial: bool, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct InlineFrame { pub context: Option, diff --git a/ghostscope-loader/Cargo.toml b/ghostscope-loader/Cargo.toml index 6b9e94f5..6ddd07de 100644 --- a/ghostscope-loader/Cargo.toml +++ b/ghostscope-loader/Cargo.toml @@ -15,7 +15,7 @@ description = "Loads compiled GhostScope programs, manages sessions, and orchest [dependencies] ghostscope-compiler = { version = "0.1.5", path = "../ghostscope-compiler" } -ghostscope-protocol = { version = "0.1.5", path = "../ghostscope-protocol" } +ghostscope-protocol = { version = "0.1.5", path = "../ghostscope-protocol", features = ["aya-pod"] } ghostscope-process = { version = "0.1.5", path = "../ghostscope-process" } aya = { workspace = true } aya-obj = { workspace = true } diff --git a/ghostscope-loader/src/lib.rs b/ghostscope-loader/src/lib.rs index 38da8f27..d29eb640 100644 --- a/ghostscope-loader/src/lib.rs +++ b/ghostscope-loader/src/lib.rs @@ -18,7 +18,7 @@ use aya::{ maps::{ perf::{PerfEvent, PerfEventArray}, - MapData, PerCpuArray, RingBuf, + Array, MapData, PerCpuArray, ProgramArray, RingBuf, }, programs::{ uprobe::{UProbeLinkId, UProbeScope}, @@ -26,9 +26,13 @@ use aya::{ }, Ebpf, EbpfLoader, VerifierLogLevel, }; -use ghostscope_protocol::{ParsedTraceEvent, StreamingTraceParser, TraceContext}; +use ghostscope_protocol::{ + BacktraceUnwindRow, ParsedTraceEvent, StreamingTraceParser, TraceContext, + BACKTRACE_UNWIND_ROW_SIZE, +}; use log::log_enabled; use log::Level as LogLevel; +use std::borrow::Borrow; use std::convert::TryInto; use std::future::poll_fn; use std::num::NonZeroU32; @@ -36,6 +40,7 @@ use std::os::unix::io::AsRawFd; use std::os::unix::io::RawFd; use std::path::Path; use std::task::Poll; +use std::time::Instant; use std::{io, ops::ControlFlow}; use tokio::io::unix::AsyncFd; use tokio::io::Interest; @@ -101,6 +106,40 @@ impl AsRawFd for PerfBufferFd { } } +fn log_backtrace_unwind_row_samples>( + array: &Array, + rows: &[BacktraceUnwindRow], +) -> Result<()> { + fn read_row>( + array: &Array, + row_index: usize, + ) -> Result { + let key = row_index as u32; + array.get(&key, 0).map_err(|e| { + LoaderError::Generic(format!("Failed to read back unwind row {row_index}: {e}")) + }) + } + + let mut sample_indices = vec![0usize, rows.len() / 2, rows.len().saturating_sub(1)]; + sample_indices.sort_unstable(); + sample_indices.dedup(); + + for index in sample_indices { + let stored = read_row(array, index)?; + if stored == rows[index] { + debug!(index, row = ?stored, "bt unwind row readback sample"); + } else { + warn!( + index, + expected = ?rows[index], + stored = ?stored, + "bt unwind row readback mismatch" + ); + } + } + Ok(()) +} + struct PerfEventCpuBuffer { cpu_id: u32, buffer: aya::maps::perf::PerfEventArrayBuffer, @@ -240,6 +279,8 @@ pub struct GhostScopeLoader { event_map: Option, /// eBPF-side output helper failure counters. event_loss_counters: Option>, + /// ProgramArray holding bt tail-call targets. + bt_prog_array: Option>, /// Active uprobe link uprobe_link: Option, /// Stored parameters for re-attaching uprobe @@ -258,6 +299,7 @@ impl std::fmt::Debug for GhostScopeLoader { .field("bpf", &"") .field("event_map", &self.event_map.is_some()) .field("event_loss_counters", &self.event_loss_counters.is_some()) + .field("bt_prog_array", &self.bt_prog_array.is_some()) .field("uprobe_attached", &self.uprobe_link.is_some()) .field("attachment_params", &self.attachment_params.is_some()) .finish() @@ -335,6 +377,7 @@ impl GhostScopeLoader { bpf, event_map: None, event_loss_counters: None, + bt_prog_array: None, uprobe_link: None, attachment_params: None, parser: StreamingTraceParser::new(), @@ -381,6 +424,50 @@ impl GhostScopeLoader { self.perf_page_count = Some(pages as usize); } + /// Load and register optional bt tail-call programs before the entry uprobe is attached. + pub fn register_backtrace_tail_call_program( + &mut self, + program_name: Option<&str>, + ) -> Result<()> { + let Some(program_name) = program_name else { + return Ok(()); + }; + + info!("Registering bt tail-call step program: {}", program_name); + let program_ref = self.bpf.program_mut(program_name).ok_or_else(|| { + LoaderError::Generic(format!("bt tail-call program '{program_name}' not found")) + })?; + let program: &mut UProbe = program_ref.try_into().map_err(|e| { + LoaderError::Generic(format!( + "bt tail-call program '{program_name}' is not a UProbe: {e:?}" + )) + })?; + program.load().map_err(LoaderError::Program)?; + let step_fd = program + .fd() + .map_err(LoaderError::Program)? + .try_clone() + .map_err(|e| { + LoaderError::Generic(format!( + "Failed to clone bt tail-call program fd for '{program_name}': {e}" + )) + })?; + + let map = self + .bpf + .take_map("bt_prog_array") + .ok_or_else(|| LoaderError::MapNotFound("bt_prog_array".to_string()))?; + let mut prog_array: ProgramArray<_> = map.try_into().map_err(|e| { + LoaderError::Generic(format!("Failed to convert bt_prog_array map: {e}")) + })?; + prog_array.set(0, &step_fd, 0).map_err(|e| { + LoaderError::Generic(format!("Failed to set bt tail-call program fd: {e}")) + })?; + self.bt_prog_array = Some(prog_array); + info!("Registered bt tail-call step program at bt_prog_array[0]"); + Ok(()) + } + /// Attach to a uprobe with a specific eBPF program name pub fn attach_uprobe_with_program_name( &mut self, @@ -1020,6 +1107,35 @@ impl GhostScopeLoader { self.trace_context = Some(trace_context); } + pub fn populate_backtrace_unwind_rows(&mut self, rows: &[BacktraceUnwindRow]) -> Result<()> { + if rows.is_empty() { + return Ok(()); + } + + let Some(map) = self.bpf.map_mut("bt_unwind_rows") else { + return Err(LoaderError::MapNotFound("bt_unwind_rows".to_string())); + }; + let mut array: Array<_, BacktraceUnwindRow> = map.try_into().map_err(|e| { + LoaderError::Generic(format!("Failed to convert bt_unwind_rows map: {e}")) + })?; + let populate_started_at = Instant::now(); + for (row_index, row) in rows.iter().copied().enumerate() { + array.set(row_index as u32, row, 0).map_err(|e| { + LoaderError::Generic(format!("Failed to set unwind row {row_index}: {e}")) + })?; + } + info!( + rows = rows.len(), + row_size = BACKTRACE_UNWIND_ROW_SIZE, + elapsed_ms = populate_started_at.elapsed().as_millis(), + "Loaded DWARF unwind rows for bt" + ); + if log_enabled!(LogLevel::Debug) { + log_backtrace_unwind_row_samples(&array, rows)?; + } + Ok(()) + } + // ============================================================================ // Information and Debugging // ============================================================================ diff --git a/ghostscope-process/Cargo.toml b/ghostscope-process/Cargo.toml index c2ef502f..9210f7e6 100644 --- a/ghostscope-process/Cargo.toml +++ b/ghostscope-process/Cargo.toml @@ -20,6 +20,7 @@ log = { workspace = true } object = { workspace = true } aya = { workspace = true } aya-obj = { workspace = true } +ghostscope-protocol = { version = "0.1.4", path = "../ghostscope-protocol", features = ["aya-pod"] } libc = "0.2" bytes = "1" memmap2 = "0.9" diff --git a/ghostscope-process/src/module_probe.rs b/ghostscope-process/src/module_probe.rs index be29c4a7..22b6e9f5 100644 --- a/ghostscope-process/src/module_probe.rs +++ b/ghostscope-process/src/module_probe.rs @@ -93,13 +93,15 @@ fn validate_module_path(path: &str) -> Result { // `/proc//maps` is not a trustworthy module list. Reject procfs/sysfs // paths up front, then resolve symlinks to the final target and insist that // the resolved object is a regular file before it reaches the ELF path. - if is_filtered_module_prefix(path) { + let input_is_proc_root = is_safe_proc_root_path(path); + if is_filtered_module_prefix(path) && !input_is_proc_root { anyhow::bail!("refusing to read pseudo-filesystem path {}", path); } let resolved_path = fs::canonicalize(path)?; let resolved_str = resolved_path.to_string_lossy(); - if is_filtered_module_prefix(&resolved_str) { + let resolved_is_safe_proc_root = input_is_proc_root && is_safe_proc_root_path(&resolved_str); + if is_filtered_module_prefix(&resolved_str) && !resolved_is_safe_proc_root { anyhow::bail!("refusing to read pseudo-filesystem path {}", resolved_str); } let meta = fs::metadata(&resolved_path)?; @@ -113,6 +115,23 @@ fn validate_module_path(path: &str) -> Result { }) } +fn is_safe_proc_root_path(path: &str) -> bool { + let Some(rest) = path.strip_prefix("/proc/") else { + return false; + }; + let Some((pid, path)) = rest.split_once('/') else { + return false; + }; + let Some(inner_path) = path.strip_prefix("root/") else { + return false; + }; + !pid.is_empty() + && pid.bytes().all(|byte| byte.is_ascii_digit()) + && !matches!(inner_path, "proc" | "sys") + && !inner_path.starts_with("proc/") + && !inner_path.starts_with("sys/") +} + fn normalize_cookie_path(module_path: &str) -> String { normalize_mapped_module_path(module_path).replace("/./", "/") } @@ -144,6 +163,39 @@ mod tests { assert!(err .to_string() .contains("refusing to read non-regular file /dev/null")); + + let err = match ModuleProbe::open("/proc/self/maps") { + Ok(_) => panic!("expected /proc/self/maps probe to be rejected"), + Err(err) => err, + }; + assert!(err + .to_string() + .contains("refusing to read pseudo-filesystem path /proc/self/maps")); + + let err = match ModuleProbe::open("/proc/self/root/proc/self/maps") { + Ok(_) => panic!("expected /proc/self/root/proc/self/maps probe to be rejected"), + Err(err) => err, + }; + assert!(err.to_string().contains("pseudo-filesystem path")); + } + + #[test] + fn allows_proc_root_regular_files() { + let base = std::env::temp_dir().join(format!( + "ghostscope-module-probe-proc-root-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::write(&base, b"not an elf but still a regular file").unwrap(); + let pid = std::process::id(); + let proc_root_path = format!("/proc/{pid}/root{}", base.display()); + + ModuleProbe::open(&proc_root_path).unwrap(); + + let _ = std::fs::remove_file(&base); } #[test] diff --git a/ghostscope-process/src/offsets.rs b/ghostscope-process/src/offsets.rs index ecf80731..271680a3 100644 --- a/ghostscope-process/src/offsets.rs +++ b/ghostscope-process/src/offsets.rs @@ -156,6 +156,14 @@ impl ModulePathSummaries { } } } + + fn merged_summary(&self) -> Option { + let mut merged = ModuleMapSummary::default(); + for summary in self.by_identity.values() { + merged.merge(summary); + } + (!merged.candidates.is_empty()).then_some(merged) + } } impl ProcessManager { @@ -291,11 +299,13 @@ impl ProcessManager { .observe(entry); } let mut list: Vec = Vec::new(); - for (module_path, summaries) in module_summaries { - let Some(summary) = summaries.summary_for_path(&module_path) else { + for (mapped_path, summaries) in module_summaries { + let Some((module_path, summary)) = + accessible_module_path_for_pid(pid, &mapped_path, &summaries) + else { tracing::debug!( - "ProcessManager: skip module {} for pid {}: no maps matched current file identity", - module_path, + "ProcessManager: skip module {} for pid {}: no accessible file matched current maps identity", + mapped_path, pid ); continue; @@ -527,6 +537,39 @@ impl ProcessManager { } } +fn accessible_module_path_for_pid( + pid: u32, + mapped_path: &str, + summaries: &ModulePathSummaries, +) -> Option<(String, ModuleMapSummary)> { + let mapped_path = normalize_mapped_module_path(mapped_path).replace("/./", "/"); + if path_is_regular_file(&mapped_path) { + if let Some(summary) = summaries.summary_for_path(&mapped_path) { + return Some((mapped_path.clone(), summary)); + } + } + + let proc_root_path = proc_root_module_path(pid, &mapped_path)?; + if !path_is_regular_file(&proc_root_path) { + return None; + } + + let summary = summaries + .summary_for_path(&proc_root_path) + .or_else(|| summaries.merged_summary())?; + Some((proc_root_path, summary)) +} + +fn proc_root_module_path(pid: u32, mapped_path: &str) -> Option { + mapped_path + .starts_with('/') + .then(|| format!("/proc/{pid}/root{mapped_path}")) +} + +fn path_is_regular_file(path: &str) -> bool { + matches!(fs::metadata(path), Ok(meta) if meta.file_type().is_file()) +} + fn is_same_executable_as_current(pid: u32) -> bool { // Strongest signal: dev+ino equality on /proc/*/exe let self_meta = fs::metadata("/proc/self/exe"); diff --git a/ghostscope-process/src/pinned_bpf_maps.rs b/ghostscope-process/src/pinned_bpf_maps.rs index 160c674d..e2fb0bc1 100644 --- a/ghostscope-process/src/pinned_bpf_maps.rs +++ b/ghostscope-process/src/pinned_bpf_maps.rs @@ -204,50 +204,7 @@ pub fn bpffs_mount_hint_for_pin_path(pin_path: &Path) -> Option { ) } -/// Key for proc_module_offsets map: { pid:u32, pad:u32, cookie_lo:u32, cookie_hi:u32 } -#[repr(C)] -#[derive(Debug, Clone, Copy)] -pub struct ProcModuleKey { - pub pid: u32, - pub pad: u32, - pub cookie_lo: u32, - pub cookie_hi: u32, -} - -/// Value for proc_module_offsets map - section offsets for a module -#[repr(C)] -#[derive(Debug, Clone, Copy)] -pub struct ProcModuleOffsetsValue { - pub text: u64, - pub rodata: u64, - pub data: u64, - pub bss: u64, -} - -// SAFETY: ProcModuleKey is repr(C), Copy, and contains only plain integer fields. -unsafe impl aya::Pod for ProcModuleKey {} -// SAFETY: ProcModuleOffsetsValue is repr(C), Copy, and contains only plain integer fields. -unsafe impl aya::Pod for ProcModuleOffsetsValue {} - -#[repr(C)] -#[derive(Debug, Clone, Copy)] -pub struct PidAliasValue { - pub proc_pid: u32, -} - -// SAFETY: PidAliasValue is repr(C), Copy, and contains only a plain integer field. -unsafe impl aya::Pod for PidAliasValue {} - -impl ProcModuleOffsetsValue { - pub fn new(text: u64, rodata: u64, data: u64, bss: u64) -> Self { - Self { - text, - rodata, - data, - bss, - } - } -} +pub use ghostscope_protocol::{PidAliasValue, ProcModuleKey, ProcModuleOffsetsValue}; fn ensure_pin_dir(path: &Path) -> std::io::Result<()> { if let Some(dir) = path.parent() { @@ -296,8 +253,8 @@ pub fn ensure_pinned_proc_offsets_exists(max_entries: u32) -> anyhow::Result<()> symbol_index: None, def: bpf_map_def { map_type: BPF_MAP_TYPE_HASH as u32, - key_size: 16, // pid:u32, pad:u32, cookie:u64 - value_size: 32, // text, rodata, data, bss + key_size: ghostscope_protocol::PROC_MODULE_KEY_SIZE as u32, + value_size: ghostscope_protocol::PROC_MODULE_OFFSETS_VALUE_SIZE as u32, max_entries, map_flags: 0, id: 0, @@ -490,8 +447,8 @@ pub fn ensure_pinned_pid_aliases_exists(max_entries: u32) -> anyhow::Result<()> symbol_index: None, def: bpf_map_def { map_type: BPF_MAP_TYPE_HASH as u32, - key_size: 4, - value_size: 4, + key_size: std::mem::size_of::() as u32, + value_size: ghostscope_protocol::PID_ALIAS_VALUE_SIZE as u32, max_entries, map_flags: 0, id: 0, diff --git a/ghostscope-protocol/Cargo.toml b/ghostscope-protocol/Cargo.toml index 1f1a8636..e7e0a2c7 100644 --- a/ghostscope-protocol/Cargo.toml +++ b/ghostscope-protocol/Cargo.toml @@ -19,6 +19,10 @@ serde_json = { workspace = true } tracing = { workspace = true } chrono = { workspace = true } aya-ebpf-bindings = { workspace = true } +aya = { workspace = true, optional = true } gimli = { workspace = true } ghostscope-platform = { version = "0.1.5", path = "../ghostscope-platform" } zerocopy = { version = "0.8", features = ["derive"] } + +[features] +aya-pod = ["dep:aya"] diff --git a/ghostscope-protocol/src/bpf_abi.rs b/ghostscope-protocol/src/bpf_abi.rs new file mode 100644 index 00000000..2ee01828 --- /dev/null +++ b/ghostscope-protocol/src/bpf_abi.rs @@ -0,0 +1,334 @@ +//! Shared eBPF map key/value ABI. +//! +//! These layouts are consumed by generated eBPF bytecode and by userspace map +//! writers. Keep them numeric, `repr(C)`, and free of compiler/loader state. + +/// Key for the pinned `proc_module_offsets` map. +#[repr(C)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +pub struct ProcModuleKey { + pub pid: u32, + pub pad: u32, + pub cookie_lo: u32, + pub cookie_hi: u32, +} + +impl ProcModuleKey { + pub fn new(pid: u32, module_cookie: u64) -> Self { + Self { + pid, + pad: 0, + cookie_lo: module_cookie as u32, + cookie_hi: (module_cookie >> 32) as u32, + } + } +} + +pub const PROC_MODULE_KEY_PID_OFFSET: usize = std::mem::offset_of!(ProcModuleKey, pid); +pub const PROC_MODULE_KEY_PAD_OFFSET: usize = std::mem::offset_of!(ProcModuleKey, pad); +pub const PROC_MODULE_KEY_COOKIE_LO_OFFSET: usize = std::mem::offset_of!(ProcModuleKey, cookie_lo); +pub const PROC_MODULE_KEY_COOKIE_HI_OFFSET: usize = std::mem::offset_of!(ProcModuleKey, cookie_hi); +pub const PROC_MODULE_KEY_SIZE: usize = std::mem::size_of::(); + +/// Value for the pinned `proc_module_offsets` map. +#[repr(C)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ProcModuleOffsetsValue { + pub text: u64, + pub rodata: u64, + pub data: u64, + pub bss: u64, +} + +impl ProcModuleOffsetsValue { + pub fn new(text: u64, rodata: u64, data: u64, bss: u64) -> Self { + Self { + text, + rodata, + data, + bss, + } + } +} + +pub const PROC_MODULE_OFFSETS_VALUE_TEXT_OFFSET: usize = + std::mem::offset_of!(ProcModuleOffsetsValue, text); +pub const PROC_MODULE_OFFSETS_VALUE_RODATA_OFFSET: usize = + std::mem::offset_of!(ProcModuleOffsetsValue, rodata); +pub const PROC_MODULE_OFFSETS_VALUE_DATA_OFFSET: usize = + std::mem::offset_of!(ProcModuleOffsetsValue, data); +pub const PROC_MODULE_OFFSETS_VALUE_BSS_OFFSET: usize = + std::mem::offset_of!(ProcModuleOffsetsValue, bss); +pub const PROC_MODULE_OFFSETS_VALUE_SIZE: usize = std::mem::size_of::(); + +/// Value for the pinned `pid_aliases` map. +#[repr(C)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct PidAliasValue { + pub proc_pid: u32, +} + +pub const PID_ALIAS_VALUE_PROC_PID_OFFSET: usize = std::mem::offset_of!(PidAliasValue, proc_pid); +pub const PID_ALIAS_VALUE_SIZE: usize = std::mem::size_of::(); + +/// BPF-facing compact DWARF CFI row used by the `bt` unwinder. +/// +/// The row is intentionally numeric and path-free: userspace owns module +/// identity, DWARF parsing, and symbolization, while eBPF receives only the +/// compact rules it can execute safely. +#[repr(C)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct BacktraceUnwindRow { + pub pc_start: u64, + pub pc_end: u64, + pub cfa_offset: i64, + pub ra_offset: i64, + pub rbp_offset: i64, + pub cfa_register: u16, + pub ra_register: u16, + pub rbp_register: u16, + pub ra_kind: u8, + pub rbp_kind: u8, + pub reserved: [u8; 2], +} + +pub const BACKTRACE_UNWIND_ROW_PC_START_OFFSET: usize = + std::mem::offset_of!(BacktraceUnwindRow, pc_start); +pub const BACKTRACE_UNWIND_ROW_PC_END_OFFSET: usize = + std::mem::offset_of!(BacktraceUnwindRow, pc_end); +pub const BACKTRACE_UNWIND_ROW_CFA_OFFSET_OFFSET: usize = + std::mem::offset_of!(BacktraceUnwindRow, cfa_offset); +pub const BACKTRACE_UNWIND_ROW_RA_OFFSET_OFFSET: usize = + std::mem::offset_of!(BacktraceUnwindRow, ra_offset); +pub const BACKTRACE_UNWIND_ROW_RBP_OFFSET_OFFSET: usize = + std::mem::offset_of!(BacktraceUnwindRow, rbp_offset); +pub const BACKTRACE_UNWIND_ROW_CFA_REGISTER_OFFSET: usize = + std::mem::offset_of!(BacktraceUnwindRow, cfa_register); +pub const BACKTRACE_UNWIND_ROW_RA_REGISTER_OFFSET: usize = + std::mem::offset_of!(BacktraceUnwindRow, ra_register); +pub const BACKTRACE_UNWIND_ROW_RBP_REGISTER_OFFSET: usize = + std::mem::offset_of!(BacktraceUnwindRow, rbp_register); +pub const BACKTRACE_UNWIND_ROW_RA_KIND_OFFSET: usize = + std::mem::offset_of!(BacktraceUnwindRow, ra_kind); +pub const BACKTRACE_UNWIND_ROW_RBP_KIND_OFFSET: usize = + std::mem::offset_of!(BacktraceUnwindRow, rbp_kind); +pub const BACKTRACE_UNWIND_ROW_SIZE: usize = std::mem::size_of::(); + +pub const BACKTRACE_UNWIND_WORDS_PER_ROW: usize = 6; +pub const BACKTRACE_UNWIND_WORD_PC_START: usize = 0; +pub const BACKTRACE_UNWIND_WORD_PC_END: usize = 1; +pub const BACKTRACE_UNWIND_WORD_CFA_OFFSET: usize = 2; +pub const BACKTRACE_UNWIND_WORD_RA_OFFSET: usize = 3; +pub const BACKTRACE_UNWIND_WORD_RBP_OFFSET: usize = 4; +pub const BACKTRACE_UNWIND_WORD_REGISTERS: usize = 5; + +pub fn backtrace_unwind_row_register_word(row: BacktraceUnwindRow) -> u64 { + u64::from(row.cfa_register) + | (u64::from(row.ra_register) << 16) + | (u64::from(row.rbp_register) << 32) + | (u64::from(row.ra_kind) << 48) + | (u64::from(row.rbp_kind) << 56) +} + +pub fn backtrace_unwind_row_word(row: BacktraceUnwindRow, word: usize) -> u64 { + match word { + BACKTRACE_UNWIND_WORD_PC_START => row.pc_start, + BACKTRACE_UNWIND_WORD_PC_END => row.pc_end, + BACKTRACE_UNWIND_WORD_CFA_OFFSET => row.cfa_offset as u64, + BACKTRACE_UNWIND_WORD_RA_OFFSET => row.ra_offset as u64, + BACKTRACE_UNWIND_WORD_RBP_OFFSET => row.rbp_offset as u64, + BACKTRACE_UNWIND_WORD_REGISTERS => backtrace_unwind_row_register_word(row), + _ => 0, + } +} + +pub fn backtrace_unwind_row_from_words( + words: [u64; BACKTRACE_UNWIND_WORDS_PER_ROW], +) -> BacktraceUnwindRow { + let registers = words[BACKTRACE_UNWIND_WORD_REGISTERS]; + BacktraceUnwindRow { + pc_start: words[BACKTRACE_UNWIND_WORD_PC_START], + pc_end: words[BACKTRACE_UNWIND_WORD_PC_END], + cfa_offset: words[BACKTRACE_UNWIND_WORD_CFA_OFFSET] as i64, + ra_offset: words[BACKTRACE_UNWIND_WORD_RA_OFFSET] as i64, + rbp_offset: words[BACKTRACE_UNWIND_WORD_RBP_OFFSET] as i64, + cfa_register: registers as u16, + ra_register: (registers >> 16) as u16, + rbp_register: (registers >> 32) as u16, + ra_kind: (registers >> 48) as u8, + rbp_kind: (registers >> 56) as u8, + reserved: [0, 0], + } +} + +/// Per-CPU state carried across `bt` tail-call unwind programs. +#[repr(C)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct BacktraceTailCallState { + pub current_ip: u64, + pub current_rsp: u64, + pub current_rbp: u64, + pub module_bias: u64, + pub module_cookie: u64, + pub inst_offset: u32, + pub event_size: u32, + pub frame_count: u8, + pub requested_depth: u8, + pub offsets_found: u8, + pub tail_calls: u8, + pub flags: u8, + pub active_slot: u8, + pub error_code: u16, + pub next_slot: u8, +} + +pub const BACKTRACE_TAIL_STATE_CURRENT_IP_OFFSET: usize = + std::mem::offset_of!(BacktraceTailCallState, current_ip); +pub const BACKTRACE_TAIL_STATE_CURRENT_RSP_OFFSET: usize = + std::mem::offset_of!(BacktraceTailCallState, current_rsp); +pub const BACKTRACE_TAIL_STATE_CURRENT_RBP_OFFSET: usize = + std::mem::offset_of!(BacktraceTailCallState, current_rbp); +pub const BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET: usize = + std::mem::offset_of!(BacktraceTailCallState, module_bias); +pub const BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET: usize = + std::mem::offset_of!(BacktraceTailCallState, module_cookie); +pub const BACKTRACE_TAIL_STATE_INST_OFFSET_OFFSET: usize = + std::mem::offset_of!(BacktraceTailCallState, inst_offset); +pub const BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET: usize = + std::mem::offset_of!(BacktraceTailCallState, event_size); +pub const BACKTRACE_TAIL_STATE_FRAME_COUNT_OFFSET: usize = + std::mem::offset_of!(BacktraceTailCallState, frame_count); +pub const BACKTRACE_TAIL_STATE_REQUESTED_DEPTH_OFFSET: usize = + std::mem::offset_of!(BacktraceTailCallState, requested_depth); +pub const BACKTRACE_TAIL_STATE_OFFSETS_FOUND_OFFSET: usize = + std::mem::offset_of!(BacktraceTailCallState, offsets_found); +pub const BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET: usize = + std::mem::offset_of!(BacktraceTailCallState, tail_calls); +pub const BACKTRACE_TAIL_STATE_FLAGS_OFFSET: usize = + std::mem::offset_of!(BacktraceTailCallState, flags); +pub const BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET: usize = + std::mem::offset_of!(BacktraceTailCallState, active_slot); +pub const BACKTRACE_TAIL_STATE_ERROR_CODE_OFFSET: usize = + std::mem::offset_of!(BacktraceTailCallState, error_code); +pub const BACKTRACE_TAIL_STATE_NEXT_SLOT_OFFSET: usize = + std::mem::offset_of!(BacktraceTailCallState, next_slot); +pub const BACKTRACE_TAIL_STATE_SIZE: usize = std::mem::size_of::(); +pub const BACKTRACE_TAIL_NO_NEXT_SLOT: u8 = u8::MAX; + +pub const BACKTRACE_RECOVERY_UNDEFINED: u8 = 0; +pub const BACKTRACE_RECOVERY_AT_CFA_OFFSET: u8 = 1; +pub const BACKTRACE_RECOVERY_VAL_CFA_OFFSET: u8 = 2; +pub const BACKTRACE_RECOVERY_REGISTER: u8 = 3; +pub const BACKTRACE_RECOVERY_SAME_VALUE: u8 = 4; + +pub const BACKTRACE_RA_UNDEFINED: u8 = BACKTRACE_RECOVERY_UNDEFINED; +pub const BACKTRACE_RA_AT_CFA_OFFSET: u8 = BACKTRACE_RECOVERY_AT_CFA_OFFSET; +pub const BACKTRACE_RA_VAL_CFA_OFFSET: u8 = BACKTRACE_RECOVERY_VAL_CFA_OFFSET; +pub const BACKTRACE_RA_REGISTER: u8 = BACKTRACE_RECOVERY_REGISTER; +pub const BACKTRACE_RA_SAME_VALUE: u8 = BACKTRACE_RECOVERY_SAME_VALUE; + +#[cfg(feature = "aya-pod")] +mod aya_pod { + use super::{ + BacktraceTailCallState, BacktraceUnwindRow, PidAliasValue, ProcModuleKey, + ProcModuleOffsetsValue, + }; + + // SAFETY: ProcModuleKey is repr(C), Copy, 'static, and contains only + // integer fields with no invalid bit patterns. + unsafe impl aya::Pod for ProcModuleKey {} + // SAFETY: ProcModuleOffsetsValue is repr(C), Copy, 'static, and contains + // only integer fields with no invalid bit patterns. + unsafe impl aya::Pod for ProcModuleOffsetsValue {} + // SAFETY: PidAliasValue is repr(C), Copy, 'static, and contains only an + // integer field with no invalid bit patterns. + unsafe impl aya::Pod for PidAliasValue {} + // SAFETY: BacktraceUnwindRow is repr(C), Copy, 'static, and contains only + // integer fields with no invalid bit patterns. + unsafe impl aya::Pod for BacktraceUnwindRow {} + // SAFETY: BacktraceTailCallState is repr(C), Copy, 'static, and contains + // only integer fields with no invalid bit patterns. + unsafe impl aya::Pod for BacktraceTailCallState {} +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn proc_module_offsets_layout_matches_bpf_maps() { + assert_eq!(PROC_MODULE_KEY_SIZE, 16); + assert_eq!(PROC_MODULE_KEY_PID_OFFSET, 0); + assert_eq!(PROC_MODULE_KEY_PAD_OFFSET, 4); + assert_eq!(PROC_MODULE_KEY_COOKIE_LO_OFFSET, 8); + assert_eq!(PROC_MODULE_KEY_COOKIE_HI_OFFSET, 12); + + assert_eq!(PROC_MODULE_OFFSETS_VALUE_SIZE, 32); + assert_eq!(PROC_MODULE_OFFSETS_VALUE_TEXT_OFFSET, 0); + assert_eq!(PROC_MODULE_OFFSETS_VALUE_RODATA_OFFSET, 8); + assert_eq!(PROC_MODULE_OFFSETS_VALUE_DATA_OFFSET, 16); + assert_eq!(PROC_MODULE_OFFSETS_VALUE_BSS_OFFSET, 24); + + assert_eq!(PID_ALIAS_VALUE_SIZE, 4); + assert_eq!(PID_ALIAS_VALUE_PROC_PID_OFFSET, 0); + } + + #[test] + fn backtrace_unwind_row_layout_matches_bpf_map_value() { + assert_eq!(BACKTRACE_UNWIND_ROW_SIZE, 56); + assert_eq!(BACKTRACE_UNWIND_ROW_PC_START_OFFSET, 0); + assert_eq!(BACKTRACE_UNWIND_ROW_PC_END_OFFSET, 8); + assert_eq!(BACKTRACE_UNWIND_ROW_CFA_OFFSET_OFFSET, 16); + assert_eq!(BACKTRACE_UNWIND_ROW_RA_OFFSET_OFFSET, 24); + assert_eq!(BACKTRACE_UNWIND_ROW_RBP_OFFSET_OFFSET, 32); + assert_eq!(BACKTRACE_UNWIND_ROW_CFA_REGISTER_OFFSET, 40); + assert_eq!(BACKTRACE_UNWIND_ROW_RA_REGISTER_OFFSET, 42); + assert_eq!(BACKTRACE_UNWIND_ROW_RBP_REGISTER_OFFSET, 44); + assert_eq!(BACKTRACE_UNWIND_ROW_RA_KIND_OFFSET, 46); + assert_eq!(BACKTRACE_UNWIND_ROW_RBP_KIND_OFFSET, 47); + assert_eq!(BACKTRACE_UNWIND_WORDS_PER_ROW, 6); + + let row = BacktraceUnwindRow { + pc_start: 0x10, + pc_end: 0x20, + cfa_offset: 128, + ra_offset: -8, + rbp_offset: -16, + cfa_register: 7, + ra_register: 16, + rbp_register: 6, + ra_kind: BACKTRACE_RA_AT_CFA_OFFSET, + rbp_kind: BACKTRACE_RECOVERY_SAME_VALUE, + reserved: [0, 0], + }; + let words = [ + backtrace_unwind_row_word(row, BACKTRACE_UNWIND_WORD_PC_START), + backtrace_unwind_row_word(row, BACKTRACE_UNWIND_WORD_PC_END), + backtrace_unwind_row_word(row, BACKTRACE_UNWIND_WORD_CFA_OFFSET), + backtrace_unwind_row_word(row, BACKTRACE_UNWIND_WORD_RA_OFFSET), + backtrace_unwind_row_word(row, BACKTRACE_UNWIND_WORD_RBP_OFFSET), + backtrace_unwind_row_word(row, BACKTRACE_UNWIND_WORD_REGISTERS), + ]; + assert_eq!(backtrace_unwind_row_from_words(words), row); + } + + #[test] + fn backtrace_tail_call_state_layout_matches_bpf_accessors() { + assert_eq!(BACKTRACE_TAIL_STATE_SIZE, 64); + assert_eq!(BACKTRACE_TAIL_STATE_CURRENT_IP_OFFSET, 0); + assert_eq!(BACKTRACE_TAIL_STATE_CURRENT_RSP_OFFSET, 8); + assert_eq!(BACKTRACE_TAIL_STATE_CURRENT_RBP_OFFSET, 16); + assert_eq!(BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, 24); + assert_eq!(BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET, 32); + assert_eq!(BACKTRACE_TAIL_STATE_INST_OFFSET_OFFSET, 40); + assert_eq!(BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, 44); + assert_eq!(BACKTRACE_TAIL_STATE_FRAME_COUNT_OFFSET, 48); + assert_eq!(BACKTRACE_TAIL_STATE_REQUESTED_DEPTH_OFFSET, 49); + assert_eq!(BACKTRACE_TAIL_STATE_OFFSETS_FOUND_OFFSET, 50); + assert_eq!(BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, 51); + assert_eq!(BACKTRACE_TAIL_STATE_FLAGS_OFFSET, 52); + assert_eq!(BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, 53); + assert_eq!(BACKTRACE_TAIL_STATE_ERROR_CODE_OFFSET, 54); + assert_eq!(BACKTRACE_TAIL_STATE_NEXT_SLOT_OFFSET, 56); + } +} diff --git a/ghostscope-protocol/src/lib.rs b/ghostscope-protocol/src/lib.rs index a075c18e..1ac81807 100644 --- a/ghostscope-protocol/src/lib.rs +++ b/ghostscope-protocol/src/lib.rs @@ -3,6 +3,7 @@ //! Provides types and functionality for the GhostScope tracing protocol. // Core modules +pub mod bpf_abi; mod type_kind; pub mod format_printer; @@ -32,12 +33,43 @@ pub use trace_event::{ VARIABLE_READ_ERROR_PAYLOAD_ERRNO_OFFSET, VARIABLE_READ_ERROR_PAYLOAD_LEN, }; +pub use bpf_abi::{ + backtrace_unwind_row_from_words, backtrace_unwind_row_register_word, backtrace_unwind_row_word, + BacktraceTailCallState, BacktraceUnwindRow, PidAliasValue, ProcModuleKey, + ProcModuleOffsetsValue, BACKTRACE_RA_AT_CFA_OFFSET, BACKTRACE_RA_REGISTER, + BACKTRACE_RA_SAME_VALUE, BACKTRACE_RA_UNDEFINED, BACKTRACE_RA_VAL_CFA_OFFSET, + BACKTRACE_RECOVERY_AT_CFA_OFFSET, BACKTRACE_RECOVERY_REGISTER, BACKTRACE_RECOVERY_SAME_VALUE, + BACKTRACE_RECOVERY_UNDEFINED, BACKTRACE_RECOVERY_VAL_CFA_OFFSET, BACKTRACE_TAIL_NO_NEXT_SLOT, + BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, BACKTRACE_TAIL_STATE_CURRENT_IP_OFFSET, + BACKTRACE_TAIL_STATE_CURRENT_RBP_OFFSET, BACKTRACE_TAIL_STATE_CURRENT_RSP_OFFSET, + BACKTRACE_TAIL_STATE_ERROR_CODE_OFFSET, BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, + BACKTRACE_TAIL_STATE_FLAGS_OFFSET, BACKTRACE_TAIL_STATE_FRAME_COUNT_OFFSET, + BACKTRACE_TAIL_STATE_INST_OFFSET_OFFSET, BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, + BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET, BACKTRACE_TAIL_STATE_NEXT_SLOT_OFFSET, + BACKTRACE_TAIL_STATE_OFFSETS_FOUND_OFFSET, BACKTRACE_TAIL_STATE_REQUESTED_DEPTH_OFFSET, + BACKTRACE_TAIL_STATE_SIZE, BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, + BACKTRACE_UNWIND_ROW_CFA_OFFSET_OFFSET, BACKTRACE_UNWIND_ROW_CFA_REGISTER_OFFSET, + BACKTRACE_UNWIND_ROW_PC_END_OFFSET, BACKTRACE_UNWIND_ROW_PC_START_OFFSET, + BACKTRACE_UNWIND_ROW_RA_KIND_OFFSET, BACKTRACE_UNWIND_ROW_RA_OFFSET_OFFSET, + BACKTRACE_UNWIND_ROW_RA_REGISTER_OFFSET, BACKTRACE_UNWIND_ROW_RBP_KIND_OFFSET, + BACKTRACE_UNWIND_ROW_RBP_OFFSET_OFFSET, BACKTRACE_UNWIND_ROW_RBP_REGISTER_OFFSET, + BACKTRACE_UNWIND_ROW_SIZE, BACKTRACE_UNWIND_WORDS_PER_ROW, BACKTRACE_UNWIND_WORD_CFA_OFFSET, + BACKTRACE_UNWIND_WORD_PC_END, BACKTRACE_UNWIND_WORD_PC_START, BACKTRACE_UNWIND_WORD_RA_OFFSET, + BACKTRACE_UNWIND_WORD_RBP_OFFSET, BACKTRACE_UNWIND_WORD_REGISTERS, + PID_ALIAS_VALUE_PROC_PID_OFFSET, PID_ALIAS_VALUE_SIZE, PROC_MODULE_KEY_COOKIE_HI_OFFSET, + PROC_MODULE_KEY_COOKIE_LO_OFFSET, PROC_MODULE_KEY_PAD_OFFSET, PROC_MODULE_KEY_PID_OFFSET, + PROC_MODULE_KEY_SIZE, PROC_MODULE_OFFSETS_VALUE_BSS_OFFSET, + PROC_MODULE_OFFSETS_VALUE_DATA_OFFSET, PROC_MODULE_OFFSETS_VALUE_RODATA_OFFSET, + PROC_MODULE_OFFSETS_VALUE_SIZE, PROC_MODULE_OFFSETS_VALUE_TEXT_OFFSET, +}; + pub use trace_context::TraceContext; pub use format_printer::FormatPrinter; pub use streaming_parser::{ - EventSource, ParseState, ParsedInstruction, ParsedTraceEvent, StreamingTraceParser, + EventSource, ParseState, ParsedBacktraceFrame, ParsedInstruction, ParsedTraceEvent, + StreamingTraceParser, }; pub use type_info::{EnumVariant, StructMember, TypeCache, TypeInfo, TypeQualifier}; diff --git a/ghostscope-protocol/src/streaming_parser.rs b/ghostscope-protocol/src/streaming_parser.rs index f4b20859..4e0967a3 100644 --- a/ghostscope-protocol/src/streaming_parser.rs +++ b/ghostscope-protocol/src/streaming_parser.rs @@ -47,7 +47,11 @@ pub enum ParsedInstruction { raw_data: Vec, }, Backtrace { - depth: u8, + requested_depth: u8, + flags: u8, + status: BacktraceStatus, + error_code: u16, + frames: Vec, }, EndInstruction { total_instructions: u16, @@ -55,6 +59,14 @@ pub enum ParsedInstruction { }, } +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct ParsedBacktraceFrame { + pub module_cookie: u64, + pub pc: u64, + pub raw_ip: u64, + pub flags: u16, +} + /// Parsed trace event containing header, message, and instructions #[derive(Debug, Clone)] pub struct ParsedTraceEvent { @@ -567,12 +579,43 @@ impl StreamingTraceParser { } t if t == InstructionType::Backtrace as u8 => { - if inst_data.is_empty() { + let (data_struct, _) = BacktraceData::read_from_prefix(inst_data) + .map_err(|_| "Invalid Backtrace data".to_string())?; + + let requested_depth = data_struct.requested_depth; + let frame_count = data_struct.frame_count; + let flags = data_struct.flags; + let status = BacktraceStatus::from_u8(data_struct.status); + let error_code = data_struct.error_code; + let frame_offset = BACKTRACE_DATA_SIZE; + let available_frames = + inst_data.len().saturating_sub(frame_offset) / BACKTRACE_FRAME_DATA_SIZE; + let parsed_frames = frame_count as usize; + if frame_count > requested_depth || parsed_frames > available_frames { return Err("Invalid Backtrace data".to_string()); } - let depth = inst_data[0]; - ParsedInstruction::Backtrace { depth } + let mut frames = Vec::with_capacity(parsed_frames); + for index in 0..parsed_frames { + let start = frame_offset + index * BACKTRACE_FRAME_DATA_SIZE; + let end = start + BACKTRACE_FRAME_DATA_SIZE; + let (frame, _) = BacktraceFrameData::read_from_prefix(&inst_data[start..end]) + .map_err(|_| "Invalid Backtrace frame data".to_string())?; + frames.push(ParsedBacktraceFrame { + module_cookie: frame.module_cookie, + pc: frame.pc, + raw_ip: frame.raw_ip, + flags: frame.flags, + }); + } + + ParsedInstruction::Backtrace { + requested_depth, + flags, + status, + error_code, + frames, + } } t if t == InstructionType::PrintComplexVariable as u8 => { @@ -756,8 +799,33 @@ impl ParsedInstruction { // formatted_value already contains "name = ..." or "name.access = ..." formatted_value.clone() } - ParsedInstruction::Backtrace { depth } => { - format!("backtrace({depth})") + ParsedInstruction::Backtrace { + requested_depth, + status, + error_code, + frames, + .. + } => { + let mut lines = vec![format!( + "backtrace(max_depth={requested_depth}, frames={}, status={})", + frames.len(), + status.label() + )]; + for (index, frame) in frames.iter().enumerate() { + lines.push(format!( + " #{index} cookie=0x{:016x} pc=0x{:x} raw=0x{:x}", + frame.module_cookie, frame.pc, frame.raw_ip + )); + } + if *status != BacktraceStatus::Complete { + let suffix = match backtrace_error_label(*error_code) { + Some("unknown") => format!(" (code={error_code})"), + Some(label) => format!(" ({label}, code={error_code})"), + None => String::new(), + }; + lines.push(format!(" stopped: {}{}", status.label(), suffix)); + } + lines.join("\n") } ParsedInstruction::EndInstruction { total_instructions, @@ -919,6 +987,78 @@ mod tests { } } + #[test] + fn test_parse_backtrace_instruction_with_frames() { + let trace_context = TraceContext::new(); + let mut parser = StreamingTraceParser::new(); + + let header = TraceEventHeader { + magic: crate::consts::MAGIC, + }; + parser + .process_segment(zerocopy::IntoBytes::as_bytes(&header), &trace_context) + .unwrap(); + + let message = TraceEventMessage { + trace_id: 7, + timestamp: 0, + pid: 100, + tid: 101, + }; + parser + .process_segment(zerocopy::IntoBytes::as_bytes(&message), &trace_context) + .unwrap(); + + let mut inst = Vec::new(); + inst.push(InstructionType::Backtrace as u8); + inst.extend_from_slice( + &((BACKTRACE_DATA_SIZE + BACKTRACE_FRAME_DATA_SIZE) as u16).to_le_bytes(), + ); + inst.push(0); + inst.push(8); // requested_depth + inst.push(1); // frame_count + inst.push(BACKTRACE_FLAG_INLINE); + inst.push(BacktraceStatus::UnsupportedCfi as u8); + inst.extend_from_slice(&0u16.to_le_bytes()); + inst.extend_from_slice(&0u16.to_le_bytes()); + inst.extend_from_slice(&0x1122_3344_5566_7788u64.to_le_bytes()); + inst.extend_from_slice(&0x1234u64.to_le_bytes()); + inst.extend_from_slice(&0x7fff_0000_1234u64.to_le_bytes()); + inst.extend_from_slice(&0u16.to_le_bytes()); + inst.extend_from_slice(&0u16.to_le_bytes()); + inst.extend_from_slice(&0u32.to_le_bytes()); + + inst.push(InstructionType::EndInstruction as u8); + inst.extend_from_slice(&(std::mem::size_of::() as u16).to_le_bytes()); + inst.push(0); + inst.extend_from_slice(&1u16.to_le_bytes()); + inst.push(1); + inst.push(0); + + let event = parser + .process_segment(&inst, &trace_context) + .unwrap() + .expect("complete event"); + match &event.instructions[0] { + ParsedInstruction::Backtrace { + requested_depth, + flags, + status, + frames, + .. + } => { + assert_eq!(*requested_depth, 8); + assert_eq!(*flags, BACKTRACE_FLAG_INLINE); + assert_eq!(*status, BacktraceStatus::UnsupportedCfi); + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].module_cookie, 0x1122_3344_5566_7788); + assert_eq!(frames[0].pc, 0x1234); + assert_eq!(frames[0].raw_ip, 0x7fff_0000_1234); + } + other => panic!("unexpected instruction: {other:?}"), + } + } + #[test] fn test_format_string_only_consumes_contiguous_variables() { let event = ParsedTraceEvent { diff --git a/ghostscope-protocol/src/trace_event.rs b/ghostscope-protocol/src/trace_event.rs index 090b9390..15f79c77 100644 --- a/ghostscope-protocol/src/trace_event.rs +++ b/ghostscope-protocol/src/trace_event.rs @@ -174,11 +174,112 @@ pub const PRINT_COMPLEX_FORMAT_ARG_FIXED_HEADER_LEN: usize = #[repr(C, packed)] #[derive(Debug, Clone, Copy, FromBytes, KnownLayout, Immutable, Unaligned)] pub struct BacktraceData { - pub depth: u8, // Maximum backtrace depth to capture - pub flags: u8, // Backtrace options (0 = default) - pub reserved: u16, // Padding for alignment - // Followed by backtrace frame data in the instruction payload + pub requested_depth: u8, + pub frame_count: u8, + pub flags: u8, + pub status: u8, + pub error_code: u16, + pub reserved: u16, + // Followed by BacktraceFrameData[requested_depth]. +} + +pub const BACKTRACE_DATA_SIZE: usize = std::mem::size_of::(); +pub const BACKTRACE_DATA_REQUESTED_DEPTH_OFFSET: usize = + std::mem::offset_of!(BacktraceData, requested_depth); +pub const BACKTRACE_DATA_FRAME_COUNT_OFFSET: usize = + std::mem::offset_of!(BacktraceData, frame_count); +pub const BACKTRACE_DATA_FLAGS_OFFSET: usize = std::mem::offset_of!(BacktraceData, flags); +pub const BACKTRACE_DATA_STATUS_OFFSET: usize = std::mem::offset_of!(BacktraceData, status); +pub const BACKTRACE_DATA_ERROR_CODE_OFFSET: usize = std::mem::offset_of!(BacktraceData, error_code); + +#[repr(C, packed)] +#[derive( + Debug, Clone, Copy, FromBytes, KnownLayout, Immutable, Unaligned, Serialize, Deserialize, +)] +pub struct BacktraceFrameData { + pub module_cookie: u64, + /// Module-normalized DWARF PC / ELF virtual address. + pub pc: u64, + /// Runtime instruction pointer as observed in the target process. + pub raw_ip: u64, + pub flags: u16, + pub reserved: u16, + pub reserved2: u32, +} + +pub const BACKTRACE_FRAME_DATA_SIZE: usize = std::mem::size_of::(); +pub const BACKTRACE_FRAME_MODULE_COOKIE_OFFSET: usize = + std::mem::offset_of!(BacktraceFrameData, module_cookie); +pub const BACKTRACE_FRAME_PC_OFFSET: usize = std::mem::offset_of!(BacktraceFrameData, pc); +pub const BACKTRACE_FRAME_RAW_IP_OFFSET: usize = std::mem::offset_of!(BacktraceFrameData, raw_ip); +pub const BACKTRACE_FRAME_FLAGS_OFFSET: usize = std::mem::offset_of!(BacktraceFrameData, flags); + +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum BacktraceStatus { + Complete = 0, + Truncated = 1, + DwarfUnavailable = 2, + UnsupportedCfi = 3, + OffsetsUnavailable = 4, + ReadError = 5, + InternalError = 6, + InvalidFrame = 7, } + +impl BacktraceStatus { + pub fn from_u8(value: u8) -> Self { + match value { + 0 => Self::Complete, + 1 => Self::Truncated, + 2 => Self::DwarfUnavailable, + 3 => Self::UnsupportedCfi, + 4 => Self::OffsetsUnavailable, + 5 => Self::ReadError, + 6 => Self::InternalError, + 7 => Self::InvalidFrame, + _ => Self::InternalError, + } + } + + pub fn label(self) -> &'static str { + match self { + Self::Complete => "complete", + Self::Truncated => "truncated", + Self::DwarfUnavailable => "dwarf unavailable", + Self::UnsupportedCfi => "unsupported CFI", + Self::OffsetsUnavailable => "offsets unavailable", + Self::ReadError => "read error", + Self::InternalError => "internal error", + Self::InvalidFrame => "invalid frame", + } + } +} + +pub const BACKTRACE_ERROR_NONE: u16 = 0; +pub const BACKTRACE_ERROR_RETURN_ADDRESS_READ: u16 = 1; +pub const BACKTRACE_ERROR_FRAME_POINTER_READ: u16 = 2; +pub const BACKTRACE_ERROR_NEXT_IP_BELOW_USER: u16 = 3; +pub const BACKTRACE_ERROR_NEXT_IP_KERNEL_LIKE: u16 = 4; +pub const BACKTRACE_ERROR_NEXT_CFA_ZERO: u16 = 5; +pub const BACKTRACE_ERROR_NEXT_CFA_NOT_ADVANCING: u16 = 6; + +pub fn backtrace_error_label(error_code: u16) -> Option<&'static str> { + match error_code { + BACKTRACE_ERROR_NONE => None, + BACKTRACE_ERROR_RETURN_ADDRESS_READ => Some("return-address-read-failed"), + BACKTRACE_ERROR_FRAME_POINTER_READ => Some("frame-pointer-read-failed"), + BACKTRACE_ERROR_NEXT_IP_BELOW_USER => Some("next-ip-below-user-range"), + BACKTRACE_ERROR_NEXT_IP_KERNEL_LIKE => Some("next-ip-kernel-like"), + BACKTRACE_ERROR_NEXT_CFA_ZERO => Some("next-cfa-zero"), + BACKTRACE_ERROR_NEXT_CFA_NOT_ADVANCING => Some("next-cfa-not-advancing"), + _ => Some("unknown"), + } +} + +pub const BACKTRACE_FLAG_RAW: u8 = 0x01; +pub const BACKTRACE_FLAG_FULL: u8 = 0x02; +pub const BACKTRACE_FLAG_INLINE: u8 = 0x04; /// ExprError instruction data - structured warning for runtime expression failure #[repr(C, packed)] #[derive(Debug, Clone, Copy, FromBytes, KnownLayout, Immutable, Unaligned)] @@ -233,9 +334,12 @@ pub enum Instruction { failing_addr: u64, }, Backtrace { - depth: u8, + requested_depth: u8, + frame_count: u8, flags: u8, - frames: Vec, // Stack frame addresses + status: BacktraceStatus, + error_code: u16, + frames: Vec, }, EndInstruction { total_instructions: u16, @@ -299,6 +403,17 @@ mod tests { assert_eq!(EXPR_ERROR_DATA_ERROR_CODE_OFFSET, 2); assert_eq!(EXPR_ERROR_DATA_FLAGS_OFFSET, 3); assert_eq!(EXPR_ERROR_DATA_FAILING_ADDR_OFFSET, 4); + assert_eq!(BACKTRACE_DATA_SIZE, 8); + assert_eq!(BACKTRACE_DATA_REQUESTED_DEPTH_OFFSET, 0); + assert_eq!(BACKTRACE_DATA_FRAME_COUNT_OFFSET, 1); + assert_eq!(BACKTRACE_DATA_FLAGS_OFFSET, 2); + assert_eq!(BACKTRACE_DATA_STATUS_OFFSET, 3); + assert_eq!(BACKTRACE_DATA_ERROR_CODE_OFFSET, 4); + assert_eq!(BACKTRACE_FRAME_DATA_SIZE, 32); + assert_eq!(BACKTRACE_FRAME_MODULE_COOKIE_OFFSET, 0); + assert_eq!(BACKTRACE_FRAME_PC_OFFSET, 8); + assert_eq!(BACKTRACE_FRAME_RAW_IP_OFFSET, 16); + assert_eq!(BACKTRACE_FRAME_FLAGS_OFFSET, 24); assert_eq!(PRINT_COMPLEX_FORMAT_DATA_ARG_COUNT_OFFSET, 2); assert_eq!(PRINT_COMPLEX_FORMAT_ARG_VAR_NAME_INDEX_OFFSET, 0); assert_eq!(PRINT_COMPLEX_FORMAT_ARG_TYPE_INDEX_OFFSET, 2); diff --git a/ghostscope-ui/src/action.rs b/ghostscope-ui/src/action.rs index 3b77ae98..87f84a90 100644 --- a/ghostscope-ui/src/action.rs +++ b/ghostscope-ui/src/action.rs @@ -74,7 +74,7 @@ pub enum Action { // Runtime communication SendRuntimeCommand(RuntimeCommand), HandleRuntimeStatus(RuntimeStatus), - HandleTraceEvent(ghostscope_protocol::ParsedTraceEvent), + HandleTraceEvent(crate::events::UiTraceEvent), // Source panel actions LoadSource { diff --git a/ghostscope-ui/src/components/app.rs b/ghostscope-ui/src/components/app.rs index 1f50c0b5..6461f5d1 100644 --- a/ghostscope-ui/src/components/app.rs +++ b/ghostscope-ui/src/components/app.rs @@ -3341,7 +3341,7 @@ impl App { } /// Handle trace events - async fn handle_trace_event(&mut self, trace_event: ghostscope_protocol::ParsedTraceEvent) { + async fn handle_trace_event(&mut self, trace_event: crate::events::UiTraceEvent) { tracing::debug!("Trace event: {:?}", trace_event); // Realtime logging: write eBPF event to file if enabled @@ -3807,7 +3807,7 @@ impl App { /// Write an eBPF event to the output log (realtime) fn write_ebpf_event_to_output_log( &mut self, - event: &ghostscope_protocol::ParsedTraceEvent, + event: &crate::events::UiTraceEvent, ) -> anyhow::Result<()> { if self.state.realtime_output_logger.enabled { // Format timestamp diff --git a/ghostscope-ui/src/components/ebpf_panel/renderer.rs b/ghostscope-ui/src/components/ebpf_panel/renderer.rs index 82ec2973..89129705 100644 --- a/ghostscope-ui/src/components/ebpf_panel/renderer.rs +++ b/ghostscope-ui/src/components/ebpf_panel/renderer.rs @@ -1,5 +1,7 @@ +use crate::events::{BacktraceDisplay, BacktraceDisplayFrame, TraceDisplayItem}; use crate::model::panel_state::{DisplayMode, EbpfPanelState, EbpfViewMode}; use crate::ui::themes::UIThemes; +use ghostscope_protocol::trace_event::BacktraceStatus; use ratatui::{ layout::Rect, style::{Color, Modifier, Style}, @@ -72,17 +74,7 @@ impl EbpfPanelRenderer { for (trace_index, cached_trace) in state.trace_events.iter().enumerate() { let trace = &cached_trace.event; let is_latest = trace_index == total_traces - 1; - let is_error = trace.instructions.last().is_some_and(|inst| { - if let ghostscope_protocol::ParsedInstruction::EndInstruction { - execution_status, - .. - } = inst - { - *execution_status == 1 || *execution_status == 2 - } else { - false - } - }); + let is_error = trace.is_error(); let header_no_bold = String::from("[No:"); let message_id = (trace_index + 1) as u64; @@ -94,30 +86,12 @@ impl EbpfPanelRenderer { ); let mut body_lines: Vec = Vec::new(); - for output_line in trace.to_formatted_output() { - let color = if output_line.contains("ERROR") || output_line.contains("Error") { - Color::Red - } else if output_line.contains("WARN") || output_line.contains("Warning") { - Color::Yellow - } else { - Color::Cyan - }; - // Compute wrapping widths based on card inner width and indent per line - let inner_width = content_width.saturating_sub(2); - let first_width = inner_width.saturating_sub(2); // first line indent " " - let cont_width = inner_width.saturating_sub(4); // continuation indent " " - let wrapped_lines = - Self::wrap_text_with_widths(&output_line, first_width, cont_width); - for (i, seg) in wrapped_lines.into_iter().enumerate() { - let line_indent = if i == 0 { " " } else { " " }; - body_lines.push(Line::from(vec![ - Span::raw(line_indent), - Span::styled( - seg, - Style::default().fg(color).add_modifier(Modifier::empty()), - ), - ])); - } + for item in &trace.items { + body_lines.extend(Self::render_trace_item( + item, + content_width, + state.view_mode, + )); } // In list view: truncate to 3 body lines (with ellipsis) to keep card compact @@ -453,6 +427,349 @@ impl EbpfPanelRenderer { } s[..end].to_string() } + + fn render_trace_item( + item: &TraceDisplayItem, + content_width: usize, + view_mode: EbpfViewMode, + ) -> Vec> { + match item { + TraceDisplayItem::Text { content } => Self::render_text_item(content, content_width), + TraceDisplayItem::Backtrace(backtrace) => Self::render_backtrace_item( + backtrace, + matches!(view_mode, EbpfViewMode::Expanded { .. }), + content_width, + ), + } + } + + fn render_text_item(content: &str, content_width: usize) -> Vec> { + let color = if content.contains("ERROR") || content.contains("Error") { + Color::Red + } else if content.contains("WARN") || content.contains("Warning") { + Color::Yellow + } else { + Color::Cyan + }; + let inner_width = content_width.saturating_sub(2); + let first_width = inner_width.saturating_sub(2); + let cont_width = inner_width.saturating_sub(4); + Self::wrap_text_with_widths(content, first_width, cont_width) + .into_iter() + .enumerate() + .map(|(i, seg)| { + let line_indent = if i == 0 { " " } else { " " }; + Line::from(vec![ + Span::raw(line_indent), + Span::styled(seg, Style::default().fg(color)), + ]) + }) + .collect() + } + + fn render_backtrace_item( + backtrace: &BacktraceDisplay, + expanded: bool, + content_width: usize, + ) -> Vec> { + let line_width = content_width.saturating_sub(2).max(1); + let mut lines = Vec::new(); + lines.push(Self::render_backtrace_header(backtrace)); + + if expanded { + lines.extend(backtrace.frames.iter().map(Self::render_backtrace_frame)); + if let Some(stopped) = backtrace.stopped_text() { + lines.push(Line::from(vec![ + Span::raw(" "), + Span::styled(stopped, Self::status_style(backtrace.status)), + ])); + } + return lines + .into_iter() + .flat_map(|line| Self::wrap_styled_line(line, line_width, " ")) + .collect(); + } else if backtrace.frames.len() > 2 { + if let Some(first) = backtrace.frames.first() { + let first_frame = + Self::wrap_styled_line(Self::render_backtrace_frame(first), line_width, " "); + if let Some(first_line) = first_frame.into_iter().next() { + lines.push(first_line); + } + } + lines.push(Self::render_backtrace_more_line( + backtrace.frames.len().saturating_sub(1), + )); + return lines; + } else { + lines.extend(backtrace.frames.iter().map(Self::render_backtrace_frame)); + if lines.len() < 3 { + if let Some(stopped) = backtrace.stopped_text() { + lines.push(Line::from(vec![ + Span::raw(" "), + Span::styled(stopped, Self::status_style(backtrace.status)), + ])); + } + } + } + + lines + .into_iter() + .flat_map(|line| Self::wrap_styled_line(line, line_width, " ")) + .collect() + } + + fn wrap_styled_line( + line: Line<'static>, + width: usize, + continuation_indent: &'static str, + ) -> Vec> { + let width = width.max(1); + let indent_width = continuation_indent.chars().count(); + if width <= indent_width { + return vec![line]; + } + + let mut wrapped = Vec::new(); + let mut current = Vec::new(); + let mut current_len = 0usize; + let mut continuation = false; + + for span in line.spans { + let style = span.style; + let mut remaining = span.content.into_owned(); + while !remaining.is_empty() { + if current_len >= width { + wrapped.push(Line::from(current)); + current = vec![Span::raw(continuation_indent)]; + current_len = indent_width; + continuation = true; + } + + let available = width.saturating_sub(current_len); + if available == 0 { + wrapped.push(Line::from(current)); + current = vec![Span::raw(continuation_indent)]; + current_len = indent_width; + continuation = true; + continue; + } + + let (segment, rest) = Self::split_prefix_chars(&remaining, available); + current_len += segment.chars().count(); + current.push(Span::styled(segment, style)); + remaining = rest; + } + } + + if current.is_empty() { + if continuation { + wrapped.push(Line::from(vec![Span::raw(continuation_indent)])); + } + } else { + wrapped.push(Line::from(current)); + } + wrapped + } + + fn split_prefix_chars(text: &str, max_chars: usize) -> (String, String) { + if max_chars == 0 { + return (String::new(), text.to_string()); + } + + let mut split = text.len(); + for (count, (idx, _)) in text.char_indices().enumerate() { + if count == max_chars { + split = idx; + break; + } + } + (text[..split].to_string(), text[split..].to_string()) + } + + fn render_backtrace_header(backtrace: &BacktraceDisplay) -> Line<'static> { + let frame_word = if backtrace.physical_frame_count == 1 { + "frame" + } else { + "frames" + }; + let mut spans = vec![ + Span::raw(" "), + Span::styled( + "backtrace", + Style::default() + .fg(Color::LightBlue) + .add_modifier(Modifier::BOLD), + ), + Span::styled(": ", Style::default().fg(Color::Gray)), + Span::styled( + backtrace.status.label().to_string(), + Self::status_style(backtrace.status), + ), + Span::styled(", ", Style::default().fg(Color::Gray)), + Span::styled( + backtrace.physical_frame_count.to_string(), + Style::default() + .fg(Color::White) + .add_modifier(Modifier::BOLD), + ), + Span::styled(format!(" {frame_word}"), Style::default().fg(Color::Gray)), + Span::styled( + format!(" (max {})", backtrace.requested_depth), + Style::default().fg(Color::DarkGray), + ), + ]; + if backtrace.raw { + spans.push(Span::styled(" raw", Style::default().fg(Color::Yellow))); + } + Line::from(spans) + } + + fn render_backtrace_frame(frame: &BacktraceDisplayFrame) -> Line<'static> { + let mut spans = vec![ + Span::raw(" "), + Span::styled( + format!("#{}", frame.index), + Style::default() + .fg(Color::LightBlue) + .add_modifier(Modifier::BOLD), + ), + ]; + if frame.inline { + spans.push(Span::styled( + ".inline", + Style::default().fg(Color::LightBlue), + )); + } + spans.push(Span::raw(" ")); + + if let Some(function) = &frame.function { + spans.push(Span::styled( + function.clone(), + Style::default() + .fg(Color::White) + .add_modifier(Modifier::BOLD), + )); + if !frame.parameters.is_empty() { + spans.push(Span::styled("(", Style::default().fg(Color::Gray))); + for (idx, parameter) in frame.parameters.iter().enumerate() { + if idx > 0 { + spans.push(Span::styled(", ", Style::default().fg(Color::Gray))); + } + spans.extend(Self::render_parameter_spans(parameter)); + } + spans.push(Span::styled(")", Style::default().fg(Color::Gray))); + } + } else { + spans.push(Span::styled( + frame + .address + .clone() + .unwrap_or_else(|| "".to_string()), + Style::default().fg(Color::Yellow), + )); + } + + if let Some(location) = &frame.location { + spans.push(Span::styled(" at ", Style::default().fg(Color::Gray))); + spans.push(Span::styled( + location.clone(), + Style::default().fg(Color::Cyan), + )); + } else if frame.function.is_some() { + spans.push(Span::styled(" at ??", Style::default().fg(Color::DarkGray))); + } + spans.push(Span::styled(" [", Style::default().fg(Color::Gray))); + spans.push(Span::styled( + frame.module.clone(), + Style::default().fg(Color::LightYellow), + )); + spans.push(Span::styled("]", Style::default().fg(Color::Gray))); + + if let Some(raw_ip) = frame.raw_ip { + spans.push(Span::styled( + format!(" raw=0x{raw_ip:x}"), + Style::default().fg(Color::DarkGray), + )); + } + if let Some(cookie) = frame.cookie { + spans.push(Span::styled( + format!(" cookie=0x{cookie:016x}"), + Style::default().fg(Color::DarkGray), + )); + } + if let Some(flags) = frame.flags { + spans.push(Span::styled( + format!(" flags=0x{flags:x}"), + Style::default().fg(Color::DarkGray), + )); + } + + Line::from(spans) + } + + fn render_backtrace_more_line(hidden_frames: usize) -> Line<'static> { + let frame_word = if hidden_frames == 1 { + "frame" + } else { + "frames" + }; + Line::from(vec![ + Span::raw(" "), + Span::styled( + format!("... {hidden_frames} more {frame_word}"), + Style::default().fg(Color::DarkGray), + ), + ]) + } + + fn render_parameter_spans(parameter: &str) -> Vec> { + let parameter = parameter.trim(); + if parameter.is_empty() { + return Vec::new(); + } + + if let Some((type_name, name)) = parameter.rsplit_once(' ') { + if !type_name.trim().is_empty() && !name.trim().is_empty() { + return vec![ + Span::styled( + type_name.trim().to_string(), + Style::default().fg(Color::LightMagenta), + ), + Span::raw(" "), + Span::styled( + name.trim().to_string(), + Style::default() + .fg(Color::Green) + .add_modifier(Modifier::BOLD), + ), + ]; + } + } + + vec![Span::styled( + parameter.to_string(), + Style::default().fg(Color::LightMagenta), + )] + } + + fn status_style(status: BacktraceStatus) -> Style { + match status { + BacktraceStatus::Complete => Style::default() + .fg(Color::Green) + .add_modifier(Modifier::BOLD), + BacktraceStatus::Truncated + | BacktraceStatus::DwarfUnavailable + | BacktraceStatus::UnsupportedCfi + | BacktraceStatus::OffsetsUnavailable => Style::default() + .fg(Color::Yellow) + .add_modifier(Modifier::BOLD), + BacktraceStatus::ReadError + | BacktraceStatus::InternalError + | BacktraceStatus::InvalidFrame => { + Style::default().fg(Color::Red).add_modifier(Modifier::BOLD) + } + } + } } impl Default for EbpfPanelRenderer { @@ -460,3 +777,160 @@ impl Default for EbpfPanelRenderer { Self::new() } } + +#[cfg(test)] +mod tests { + use super::*; + + fn line_text(line: &Line<'_>) -> String { + line.spans + .iter() + .map(|span| span.content.as_ref()) + .collect() + } + + fn sample_backtrace(frame_count: usize) -> BacktraceDisplay { + BacktraceDisplay { + requested_depth: 128, + physical_frame_count: frame_count, + status: BacktraceStatus::Complete, + error_code: 0, + raw: false, + frames: (0..frame_count) + .map(|index| BacktraceDisplayFrame { + index, + inline: false, + function: Some(format!("function_{index}")), + parameters: vec!["ngx_http_request_s* r".to_string()], + address: None, + location: Some(format!("request.c:{}", 100 + index)), + module: format!("nginx+0x{:x}", 0x1000 + index), + raw_ip: None, + cookie: None, + flags: None, + }) + .collect(), + } + } + + #[test] + fn list_mode_renders_backtrace_as_compact_structured_item() { + let item = TraceDisplayItem::Backtrace(sample_backtrace(4)); + let lines = EbpfPanelRenderer::render_trace_item(&item, 120, EbpfViewMode::List); + + assert_eq!(lines.len(), 3); + assert!(line_text(&lines[0]).contains("backtrace: complete, 4 frames")); + assert!(line_text(&lines[1]).contains("#0 function_0")); + assert!(line_text(&lines[2]).contains("... 3 more frames")); + } + + #[test] + fn list_mode_keeps_backtrace_summary_when_first_frame_wraps() { + let mut backtrace = sample_backtrace(4); + backtrace.frames[0].function = + Some("ngx_http_process_request_headers_with_a_long_suffix".to_string()); + backtrace.frames[0].parameters = vec!["ngx_http_request_s* request".to_string()]; + backtrace.frames[0].location = Some( + "/mnt/500g/code/openresty/openresty-1.27.1.1/build/nginx/src/http/ngx_http_request.c:1529:13" + .to_string(), + ); + + let item = TraceDisplayItem::Backtrace(backtrace); + let lines = EbpfPanelRenderer::render_trace_item(&item, 48, EbpfViewMode::List); + + assert_eq!(lines.len(), 3); + assert!(line_text(&lines[0]).contains("backtrace: complete, 4 frames")); + assert!(line_text(&lines[1]).contains("#0 ngx_http_process")); + assert!(line_text(&lines[2]).contains("... 3 more frames")); + } + + #[test] + fn expanded_mode_keeps_backtrace_status_and_parameters_structured() { + let item = TraceDisplayItem::Backtrace(sample_backtrace(1)); + let lines = EbpfPanelRenderer::render_trace_item( + &item, + 120, + EbpfViewMode::Expanded { + index: 0, + scroll: 0, + }, + ); + + let frame = line_text(&lines[1]); + assert!(frame.contains("function_0(")); + assert!(frame.contains("ngx_http_request_s* r")); + assert!(frame.contains("request.c:100")); + assert!(frame.contains("[nginx+0x1000]")); + } + + #[test] + fn expanded_backtrace_lines_wrap_to_panel_width() { + let mut backtrace = sample_backtrace(1); + backtrace.frames[0].function = + Some("ngx_http_process_request_headers_with_a_long_suffix".to_string()); + backtrace.frames[0].parameters = vec![ + "ngx_http_request_s* request".to_string(), + "long unsigned int flags".to_string(), + ]; + backtrace.frames[0].location = Some( + "/mnt/500g/code/openresty/openresty-1.27.1.1/build/nginx/src/http/ngx_http_request.c:1529:13" + .to_string(), + ); + + let item = TraceDisplayItem::Backtrace(backtrace); + let lines = EbpfPanelRenderer::render_trace_item( + &item, + 48, + EbpfViewMode::Expanded { + index: 0, + scroll: 0, + }, + ); + + assert!( + lines.len() > 2, + "narrow backtrace output should wrap long frame lines" + ); + assert!(line_text(&lines[1]).contains("#0 ngx_http_process")); + assert!( + lines + .iter() + .skip(2) + .map(line_text) + .any(|line| line.starts_with(" ") && line.contains("request")), + "wrapped continuation should keep parameter text with indentation" + ); + assert!( + lines + .iter() + .map(line_text) + .any(|line| line.contains("ngx_http_request.c:1529:13")), + "wrapped continuation should retain the source location" + ); + } + + #[test] + fn header_uses_physical_frame_count_for_inline_backtraces() { + let mut backtrace = sample_backtrace(1); + let mut inline_frame = backtrace.frames[0].clone(); + inline_frame.inline = true; + inline_frame.function = Some("inlined_add".to_string()); + backtrace.frames.insert(0, inline_frame); + + let item = TraceDisplayItem::Backtrace(backtrace); + let lines = EbpfPanelRenderer::render_trace_item( + &item, + 120, + EbpfViewMode::Expanded { + index: 0, + scroll: 0, + }, + ); + + let header = line_text(&lines[0]); + assert!(header.contains("backtrace: complete, 1 frame (max 128)")); + assert!(!header.contains("2 frames")); + assert!(line_text(&lines[1]).contains("#0.inline inlined_add")); + assert!(line_text(&lines[2]).contains("#0 function_0")); + } +} diff --git a/ghostscope-ui/src/events.rs b/ghostscope-ui/src/events.rs index a482edef..09002af2 100644 --- a/ghostscope-ui/src/events.rs +++ b/ghostscope-ui/src/events.rs @@ -1,8 +1,221 @@ use crossterm::event::{KeyEvent, MouseEvent}; -use ghostscope_protocol::ParsedTraceEvent; +use ghostscope_protocol::{ + trace_event::{backtrace_error_label, BacktraceStatus}, + ParsedTraceEvent, +}; use tokio::sync::mpsc; use unicode_width::UnicodeWidthStr; +/// Runtime trace event after conversion into TUI display items. +/// +/// This keeps the UI transport structured without changing the eBPF/protocol +/// wire format. Non-backtrace instructions are grouped into text lines, while +/// backtraces keep frame/status fields for dedicated TUI rendering. +#[derive(Debug, Clone)] +pub struct UiTraceEvent { + pub trace_id: u64, + pub timestamp: u64, + pub pid: u32, + pub tid: u32, + pub items: Vec, + pub execution_status: Option, +} + +impl UiTraceEvent { + pub fn from_protocol_event(event: &ParsedTraceEvent) -> Self { + let execution_status = event.instructions.iter().rev().find_map(|instruction| { + if let ghostscope_protocol::ParsedInstruction::EndInstruction { + execution_status, .. + } = instruction + { + Some(*execution_status) + } else { + None + } + }); + + Self { + trace_id: event.trace_id, + timestamp: event.timestamp, + pid: event.pid, + tid: event.tid, + items: event + .to_formatted_output() + .into_iter() + .map(|content| TraceDisplayItem::Text { content }) + .collect(), + execution_status, + } + } + + pub fn text_event( + trace_id: u64, + timestamp: u64, + pid: u32, + tid: u32, + content: String, + execution_status: Option, + ) -> Self { + Self { + trace_id, + timestamp, + pid, + tid, + items: vec![TraceDisplayItem::Text { content }], + execution_status, + } + } + + pub fn to_formatted_output(&self) -> Vec { + self.items + .iter() + .flat_map(TraceDisplayItem::to_formatted_output) + .collect() + } + + pub fn is_error(&self) -> bool { + self.execution_status + .is_some_and(|status| status == 1 || status == 2) + || self.items.iter().any(|item| { + matches!( + item, + TraceDisplayItem::Backtrace(backtrace) + if backtrace.status != BacktraceStatus::Complete + && backtrace.status != BacktraceStatus::Truncated + ) + }) + } +} + +#[derive(Debug, Clone)] +pub enum TraceDisplayItem { + Text { content: String }, + Backtrace(BacktraceDisplay), +} + +impl TraceDisplayItem { + pub fn to_formatted_output(&self) -> Vec { + match self { + Self::Text { content } => vec![content.clone()], + Self::Backtrace(backtrace) => backtrace.to_formatted_output(), + } + } +} + +#[derive(Debug, Clone)] +pub struct BacktraceDisplay { + pub requested_depth: u8, + pub physical_frame_count: usize, + pub status: BacktraceStatus, + pub error_code: u16, + pub raw: bool, + pub frames: Vec, +} + +impl BacktraceDisplay { + pub fn header_text(&self) -> String { + let frame_word = if self.physical_frame_count == 1 { + "frame" + } else { + "frames" + }; + format!( + "backtrace: {}, {} {} (max {})", + self.status.label(), + self.physical_frame_count, + frame_word, + self.requested_depth + ) + } + + pub fn stopped_text(&self) -> Option { + if self.status == BacktraceStatus::Complete { + return None; + } + + let suffix = match backtrace_error_label(self.error_code) { + Some("unknown") => format!(" (code={})", self.error_code), + Some(label) => format!(" ({label}, code={})", self.error_code), + None => String::new(), + }; + Some(format!("stopped: {}{}", self.status.label(), suffix)) + } + + pub fn to_formatted_output(&self) -> Vec { + let mut output = Vec::with_capacity(self.frames.len() + 2); + output.push(self.header_text()); + output.extend( + self.frames + .iter() + .map(BacktraceDisplayFrame::to_formatted_output), + ); + if let Some(stopped) = self.stopped_text() { + output.push(stopped); + } + output + } +} + +#[derive(Debug, Clone)] +pub struct BacktraceDisplayFrame { + pub index: usize, + pub inline: bool, + pub function: Option, + pub parameters: Vec, + pub address: Option, + pub location: Option, + pub module: String, + pub raw_ip: Option, + pub cookie: Option, + pub flags: Option, +} + +impl BacktraceDisplayFrame { + pub fn to_formatted_output(&self) -> String { + let mut line = String::from(" #"); + line.push_str(&self.index.to_string()); + if self.inline { + line.push_str(".inline"); + } + line.push(' '); + + if let Some(function) = &self.function { + line.push_str(function); + if !self.parameters.is_empty() { + line.push('('); + line.push_str(&self.parameters.join(", ")); + line.push(')'); + } + } else if let Some(address) = &self.address { + line.push_str(address); + } else { + line.push_str(""); + } + + if let Some(location) = &self.location { + line.push_str(" at "); + line.push_str(location); + } else if self.function.is_some() { + line.push_str(" at ??"); + } + line.push_str(" ["); + line.push_str(&self.module); + line.push(']'); + + if let Some(raw_ip) = self.raw_ip { + line.push_str(&format!(" raw=0x{raw_ip:x}")); + } + if let Some(cookie) = self.cookie { + line.push_str(&format!(" cookie=0x{cookie:016x}")); + } + if let Some(flags) = self.flags { + line.push_str(&format!(" flags=0x{flags:x}")); + } + + line + } +} + /// Trace status enumeration for shared use between UI and runtime #[derive(Debug, Clone, PartialEq, Eq)] pub enum TraceStatus { @@ -58,7 +271,7 @@ pub struct EventRegistry { pub command_sender: mpsc::UnboundedSender, // Runtime -> TUI communication - pub trace_receiver: mpsc::Receiver, + pub trace_receiver: mpsc::Receiver, pub status_receiver: mpsc::UnboundedReceiver, } @@ -1018,7 +1231,7 @@ impl EventRegistry { pub fn new_with_trace_capacity(trace_capacity: usize) -> (Self, RuntimeChannels) { let trace_capacity = trace_capacity.max(1); let (command_tx, command_rx) = mpsc::unbounded_channel(); - let (trace_tx, trace_rx) = mpsc::channel::(trace_capacity); + let (trace_tx, trace_rx) = mpsc::channel::(trace_capacity); let (status_tx, status_rx) = mpsc::unbounded_channel(); let registry = EventRegistry { @@ -1045,7 +1258,7 @@ pub const DEFAULT_TRACE_CHANNEL_CAPACITY: usize = 4096; #[derive(Debug)] pub struct RuntimeChannels { pub command_receiver: mpsc::UnboundedReceiver, - pub trace_sender: mpsc::Sender, + pub trace_sender: mpsc::Sender, pub status_sender: mpsc::UnboundedSender, pub trace_channel_capacity: usize, } @@ -1057,7 +1270,7 @@ impl RuntimeChannels { } /// Create a trace sender that can be shared with other tasks - pub fn create_trace_sender(&self) -> mpsc::Sender { + pub fn create_trace_sender(&self) -> mpsc::Sender { self.trace_sender.clone() } } @@ -1277,13 +1490,14 @@ impl RuntimeStatus { mod tests { use super::*; - fn sample_event(trace_id: u64) -> ParsedTraceEvent { - ParsedTraceEvent { + fn sample_event(trace_id: u64) -> UiTraceEvent { + UiTraceEvent { timestamp: 0, trace_id, pid: 42, tid: 42, - instructions: vec![], + items: vec![], + execution_status: None, } } diff --git a/ghostscope-ui/src/lib.rs b/ghostscope-ui/src/lib.rs index b45b4b40..24548a36 100644 --- a/ghostscope-ui/src/lib.rs +++ b/ghostscope-ui/src/lib.rs @@ -10,7 +10,10 @@ pub mod utils; // Public exports pub use action::{Action, PanelType}; pub use components::App; -pub use events::{EventRegistry, RuntimeChannels, RuntimeCommand, RuntimeStatus, TuiEvent}; +pub use events::{ + BacktraceDisplay, BacktraceDisplayFrame, EventRegistry, RuntimeChannels, RuntimeCommand, + RuntimeStatus, TraceDisplayItem, TuiEvent, UiTraceEvent, +}; pub use model::ui_state::{HistoryConfig, LayoutMode, UiConfig}; use anyhow::Result; diff --git a/ghostscope-ui/src/model/panel_state.rs b/ghostscope-ui/src/model/panel_state.rs index 3c09f537..e48cfb53 100644 --- a/ghostscope-ui/src/model/panel_state.rs +++ b/ghostscope-ui/src/model/panel_state.rs @@ -1,12 +1,12 @@ use crate::action::ResponseType; -use ghostscope_protocol::{ParsedInstruction, ParsedTraceEvent}; +use crate::events::UiTraceEvent; use std::collections::{HashMap, HashSet, VecDeque}; use std::time::Instant; /// Cached trace event with pre-formatted timestamp #[derive(Debug, Clone)] pub struct CachedTraceEvent { - pub event: ParsedTraceEvent, + pub event: UiTraceEvent, pub formatted_timestamp: String, } @@ -161,7 +161,7 @@ impl EbpfPanelState { } } - pub fn add_trace_event(&mut self, trace_event: ParsedTraceEvent) { + pub fn add_trace_event(&mut self, trace_event: UiTraceEvent) { // Format timestamp once when adding the event let formatted_timestamp = crate::utils::format_timestamp_ns(trace_event.timestamp); let cached_event = CachedTraceEvent { @@ -180,20 +180,8 @@ impl EbpfPanelState { } } - pub fn runtime_warning_event(message: String, timestamp: u64) -> ParsedTraceEvent { - ParsedTraceEvent { - trace_id: 0, - timestamp, - pid: 0, - tid: 0, - instructions: vec![ - ParsedInstruction::PrintString { content: message }, - ParsedInstruction::EndInstruction { - total_instructions: 1, - execution_status: 1, - }, - ], - } + pub fn runtime_warning_event(message: String, timestamp: u64) -> UiTraceEvent { + UiTraceEvent::text_event(0, timestamp, 0, 0, message, Some(1)) } pub fn add_runtime_warning_message(&mut self, message: String, timestamp: u64) { diff --git a/ghostscope-ui/tests/integration_test.rs b/ghostscope-ui/tests/integration_test.rs index 129cbf28..74c68a73 100644 --- a/ghostscope-ui/tests/integration_test.rs +++ b/ghostscope-ui/tests/integration_test.rs @@ -1,7 +1,6 @@ use anyhow::Result; use crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; -use ghostscope_protocol::ParsedTraceEvent; -use ghostscope_ui::events::{EventRegistry, RuntimeCommand, RuntimeStatus}; +use ghostscope_ui::events::{EventRegistry, RuntimeCommand, RuntimeStatus, UiTraceEvent}; use ratatui::{backend::TestBackend, Terminal}; use tokio::sync::mpsc; @@ -9,8 +8,8 @@ use tokio::sync::mpsc; pub struct TuiTestHarness { /// Mock channels for runtime communication pub command_receiver: mpsc::UnboundedReceiver, - pub trace_sender: mpsc::Sender, - pub trace_receiver: mpsc::Receiver, + pub trace_sender: mpsc::Sender, + pub trace_receiver: mpsc::Receiver, pub status_sender: mpsc::UnboundedSender, pub status_receiver: mpsc::UnboundedReceiver, @@ -99,7 +98,7 @@ impl TuiTestHarness { } /// Send a mock trace event - pub fn send_trace_event(&mut self, event: ParsedTraceEvent) -> Result<()> { + pub fn send_trace_event(&mut self, event: UiTraceEvent) -> Result<()> { self.trace_sender .try_send(event) .map_err(|e| anyhow::anyhow!("Failed to enqueue trace event: {e}"))?; @@ -118,7 +117,7 @@ impl TuiTestHarness { } /// Receive a trace event (for verification in tests) - pub async fn receive_trace_event(&mut self) -> Option { + pub async fn receive_trace_event(&mut self) -> Option { self.trace_receiver.recv().await } @@ -177,12 +176,13 @@ mod tests { let mut harness = TuiTestHarness::new(120, 40); // Create a mock trace event - let trace_event = ParsedTraceEvent { + let trace_event = UiTraceEvent { timestamp: 1234567890, trace_id: 1, pid: 12345, tid: 67890, - instructions: vec![], + items: vec![], + execution_status: None, }; // Clone for verification diff --git a/ghostscope/src/cli/color.rs b/ghostscope/src/cli/color.rs index f6fd4941..d49d0d7b 100644 --- a/ghostscope/src/cli/color.rs +++ b/ghostscope/src/cli/color.rs @@ -48,6 +48,10 @@ impl CliColors { self.wrap("33", value) } + pub fn magenta(&self, value: T) -> String { + self.wrap("35", value) + } + pub fn red(&self, value: T) -> String { self.wrap("31", value) } diff --git a/ghostscope/src/cli/script_output.rs b/ghostscope/src/cli/script_output.rs index 292c6031..87c0d0f1 100644 --- a/ghostscope/src/cli/script_output.rs +++ b/ghostscope/src/cli/script_output.rs @@ -50,7 +50,11 @@ impl ScriptOutputRenderer { let mut lines = Vec::with_capacity(formatted_output.len() + 1); lines.push(self.render_pretty_header(event)); - lines.extend(formatted_output.into_iter().map(|line| format!(" {line}"))); + lines.extend( + formatted_output + .iter() + .map(|line| self.render_pretty_payload_line(line)), + ); lines } } @@ -76,7 +80,9 @@ impl ScriptOutputRenderer { } writeln!(writer, "{}", self.render_pretty_header(event))?; - event.try_for_each_formatted_output(|line| writeln!(writer, " {line}"))?; + event.try_for_each_formatted_output(|line| { + writeln!(writer, "{}", self.render_pretty_payload_line(line)) + })?; Ok(true) } } @@ -103,6 +109,136 @@ impl ScriptOutputRenderer { None => metadata, } } + + fn render_pretty_payload_line(&self, line: &str) -> String { + format!(" {}", self.colorize_pretty_payload_line(line)) + } + + fn colorize_pretty_payload_line(&self, line: &str) -> String { + if !self.colors.enabled() { + return line.to_string(); + } + + if let Some(colored) = self.colorize_backtrace_header(line) { + return colored; + } + if let Some(colored) = self.colorize_backtrace_frame(line) { + return colored; + } + if line.starts_with("stopped: ") { + return self.colors.red(line); + } + + line.to_string() + } + + fn colorize_backtrace_header(&self, line: &str) -> Option { + let rest = line.strip_prefix("backtrace: ")?; + let (status, tail) = rest.split_once(',')?; + if !tail.contains(" frame") || !tail.contains(" (max ") { + return None; + } + + Some(format!( + "{}: {},{}", + self.colors.bold("backtrace"), + self.colorize_backtrace_status(status), + tail + )) + } + + fn colorize_backtrace_status(&self, status: &str) -> String { + match status { + "complete" => self.colors.green(status), + "truncated" => self.colors.yellow(status), + _ => self.colors.red(status), + } + } + + fn colorize_backtrace_frame(&self, line: &str) -> Option { + let leading_len = line.len() - line.trim_start().len(); + let leading = &line[..leading_len]; + let rest = &line[leading_len..]; + if !rest.starts_with('#') { + return None; + } + if !rest[1..] + .chars() + .next() + .is_some_and(|ch| ch.is_ascii_digit()) + { + return None; + } + + let (frame, body) = rest.split_once(' ')?; + let body = body.trim_start(); + let (body, module) = match body.rsplit_once(" [") { + Some((body, module)) if module.ends_with(']') => (body, Some(format!("[{module}"))), + _ => (body, None), + }; + let body = match body.split_once(" at ") { + Some((function, location)) => format!( + "{} at {}", + self.colorize_backtrace_function(function), + self.colors.dim(location) + ), + None => body.to_string(), + }; + + let module = module + .map(|module| format!(" {}", self.colors.yellow(module))) + .unwrap_or_default(); + + Some(format!( + "{}{} {}{}", + leading, + self.colors.blue(frame), + body, + module + )) + } + + fn colorize_backtrace_function(&self, function: &str) -> String { + match split_function_parameters(function) { + Some((name, parameters)) => { + format!( + "{}{}", + self.colors.bold(name), + self.colors.magenta(parameters) + ) + } + None => self.colors.bold(function), + } + } +} + +fn split_function_parameters(function: &str) -> Option<(&str, &str)> { + if !function.ends_with(')') { + return None; + } + + let mut depth = 0usize; + for (index, ch) in function.char_indices().rev() { + match ch { + ')' => depth = depth.saturating_add(1), + '(' => { + if depth == 0 { + return None; + } + depth -= 1; + if depth == 0 { + let name = &function[..index]; + if name.is_empty() { + return None; + } + return Some((name, &function[index..])); + } + } + _ => {} + } + } + + None } #[derive(Debug)] @@ -205,7 +341,10 @@ fn get_system_uptime_ns() -> Option { #[cfg(test)] mod tests { - use super::{LocalTimestampFormatter, ScriptOutputOptions, ScriptOutputRenderer}; + use super::{ + split_function_parameters, LocalTimestampFormatter, ScriptOutputOptions, + ScriptOutputRenderer, + }; use crate::config::{ScriptOutputMode, ScriptTimestampFormat}; use ghostscope_protocol::{ParsedInstruction, ParsedTraceEvent}; @@ -251,6 +390,27 @@ mod tests { } } + fn sample_backtrace_event() -> ParsedTraceEvent { + ParsedTraceEvent { + trace_id: 9, + timestamp: 3_000_000_000, + pid: 6001, + tid: 6002, + instructions: vec![ + ParsedInstruction::PrintString { + content: "backtrace: complete, 2 frames (max 128)".to_string(), + }, + ParsedInstruction::PrintString { + content: " #0 ngx_http_process_request(ngx_http_request_s* r) at /tmp/ngx_http_request.c:2054:1 [nginx+0x16e233]".to_string(), + }, + ParsedInstruction::PrintString { + content: "stopped: read error (return-address-read-failed, code=1)" + .to_string(), + }, + ], + } + } + fn render_with_renderer(event: &ParsedTraceEvent, options: ScriptOutputOptions) -> Vec { let mut renderer = ScriptOutputRenderer::new(options); renderer.render_event_lines(event) @@ -372,4 +532,59 @@ mod tests { assert!(lines[0].contains("\u{1b}[")); } + + #[test] + fn pretty_output_colorizes_backtrace_payload_only_when_enabled() { + let colored = render_with_renderer( + &sample_backtrace_event(), + ScriptOutputOptions { + mode: ScriptOutputMode::Pretty, + timestamp: ScriptTimestampFormat::None, + color_enabled: true, + }, + ); + assert!( + colored.iter().skip(1).any(|line| line.contains("\u{1b}[")), + "expected ANSI color in pretty backtrace payload: {colored:?}" + ); + assert!( + colored.iter().any(|line| line.contains( + "\u{1b}[1mngx_http_process_request\u{1b}[0m\u{1b}[35m(ngx_http_request_s* r)\u{1b}[0m" + )), + "expected function name and parameters to use distinct colors: {colored:?}" + ); + + let plain = render_with_renderer( + &sample_backtrace_event(), + ScriptOutputOptions { + mode: ScriptOutputMode::Plain, + timestamp: ScriptTimestampFormat::None, + color_enabled: true, + }, + ); + assert!( + plain.iter().all(|line| !line.contains("\u{1b}[")), + "plain script output must not colorize payload: {plain:?}" + ); + } + + #[test] + fn backtrace_signature_colorizer_splits_trailing_parameter_list() { + assert_eq!( + split_function_parameters("ngx_epoll_process_events(ngx_cycle_s* cycle, long unsigned int timer, long unsigned int flags)"), + Some(( + "ngx_epoll_process_events", + "(ngx_cycle_s* cycle, long unsigned int timer, long unsigned int flags)" + )) + ); + assert_eq!( + split_function_parameters("operator()(int value)"), + Some(("operator()", "(int value)")) + ); + assert_eq!( + split_function_parameters("call_with_callback(void (*cb)(int))"), + Some(("call_with_callback", "(void (*cb)(int))")) + ); + assert_eq!(split_function_parameters(""), None); + } } diff --git a/ghostscope/src/cli/script_runtime.rs b/ghostscope/src/cli/script_runtime.rs index ebc5d5dc..50d62a5b 100644 --- a/ghostscope/src/cli/script_runtime.rs +++ b/ghostscope/src/cli/script_runtime.rs @@ -7,7 +7,7 @@ use ghostscope_dwarf::ModuleLoadingEvent; use std::io::{self, IsTerminal, Write}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; -use tracing::{debug, error, info, warn}; +use tracing::{error, info, trace, warn}; const SCRIPT_OUTPUT_BACKPRESSURE_SLEEP: Duration = Duration::from_millis(10); @@ -387,6 +387,7 @@ async fn run_cli_with_session( ); let stdout = io::stdout(); let mut stdout = io::BufWriter::new(stdout.lock()); + let mut backtrace_renderer = crate::trace::backtrace::BacktraceRenderer::default(); let mut output_rate_limiter = ScriptOutputRateLimiter::new(config.script_output_events_per_sec); let mut ebpf_loss_report_ticker = tokio::time::interval(Duration::from_secs(1)); ebpf_loss_report_ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); @@ -408,7 +409,19 @@ async fn run_cli_with_session( ) { ScriptOutputRateDecision::Silent => {} ScriptOutputRateDecision::Render => { - match output_renderer.write_event(&event, &mut stdout) { + let rendered_event = { + let coordinator = session + .coordinator + .lock() + .expect("coordinator mutex poisoned"); + backtrace_renderer.render_event_backtraces( + &event, + session.process_analyzer.as_ref(), + &coordinator, + session.proc_pid(), + ) + }; + match output_renderer.write_event(&rendered_event, &mut stdout) { Ok(wrote) => wrote_output |= wrote, Err(e) => warn!("Failed to write event output: {e}"), } @@ -418,7 +431,7 @@ async fn run_cli_with_session( } } - debug!("Raw trace event: {:?}", event); + trace!("Raw trace event: {:?}", event); } output_rate_limiter.maybe_report(Instant::now()); // When stdout is piped (as in tests), Rust switches to block buffering. diff --git a/ghostscope/src/config/args.rs b/ghostscope/src/config/args.rs index ccb8d25c..25f9f2d6 100644 --- a/ghostscope/src/config/args.rs +++ b/ghostscope/src/config/args.rs @@ -258,6 +258,10 @@ pub struct Args { #[arg(long = "script-output-events-per-sec", value_name = "N")] pub script_output_events_per_sec: Option, + /// Max DWARF-unwound frames captured by each bt/backtrace instruction. + #[arg(long = "backtrace-depth", value_name = "N", value_parser = clap::value_parser!(u8).range(1..=128))] + pub backtrace_depth: Option, + /// Save LLVM IR files for each trace pattern (debug: true, release: false) #[arg(long, action = clap::ArgAction::SetTrue)] pub save_llvm_ir: bool, @@ -340,6 +344,7 @@ pub struct ParsedArgs { pub has_explicit_status_flag: bool, pub script_timestamp: Option, pub script_output_events_per_sec: Option, + pub backtrace_depth: Option, pub should_save_llvm_ir: bool, pub should_save_ebpf: bool, pub should_save_ast: bool, @@ -521,6 +526,7 @@ impl Args { has_explicit_status_flag, script_timestamp: parsed.script_timestamp, script_output_events_per_sec: parsed.script_output_events_per_sec, + backtrace_depth: parsed.backtrace_depth, should_save_llvm_ir, should_save_ebpf, should_save_ast, @@ -881,6 +887,43 @@ mod tests { } } + #[test] + fn parses_backtrace_depth_flag() { + let parsed = Args::parse_args_from(vec![ + "ghostscope".to_string(), + "--pid".to_string(), + "1234".to_string(), + "--script-file".to_string(), + "trace.gs".to_string(), + "--backtrace-depth".to_string(), + "64".to_string(), + ]); + + match parsed { + ParsedCommand::Trace(args) => { + assert_eq!(args.backtrace_depth, Some(64)); + } + other => panic!("unexpected parse result: {other:?}"), + } + } + + #[test] + fn rejects_backtrace_depth_out_of_range() { + let err = Args::try_parse_from(vec![ + "ghostscope", + "--pid", + "1234", + "--script-file", + "trace.gs", + "--backtrace-depth", + "129", + ]) + .unwrap_err() + .to_string(); + + assert!(err.contains("--backtrace-depth")); + } + #[test] fn parses_dry_run_details_flag() { let parsed = Args::parse_args_from(vec![ diff --git a/ghostscope/src/config/runtime.rs b/ghostscope/src/config/runtime.rs index 1ff3ab5b..896d3177 100644 --- a/ghostscope/src/config/runtime.rs +++ b/ghostscope/src/config/runtime.rs @@ -261,6 +261,7 @@ impl ResolvedConfig { mem_dump_cap: self.ebpf_config.mem_dump_cap, compare_cap: self.ebpf_config.compare_cap, max_trace_event_size: effective_max_event, + backtrace_depth: self.ebpf_config.backtrace_depth, selected_index: None, pid_filter_spec: self.runtime.pid_filter_spec, special_pid_ns: self.runtime.special_pid_ns, diff --git a/ghostscope/src/config/settings.rs b/ghostscope/src/config/settings.rs index a20c73b0..9108b104 100644 --- a/ghostscope/src/config/settings.rs +++ b/ghostscope/src/config/settings.rs @@ -250,6 +250,10 @@ pub struct EbpfConfig { /// Default: 32768 bytes (32KB). Increase for larger formatted prints. #[serde(default = "default_max_trace_event_size")] pub max_trace_event_size: u32, + /// Max DWARF-unwound frames captured by each bt/backtrace instruction. + /// Default: 128 frames. + #[serde(default = "default_backtrace_depth")] + pub backtrace_depth: u8, /// Start sysmon eBPF for standalone -t targets. /// Maintains ASLR offsets for late-start processes loading the target. /// WARNING: This enables system-wide sched tracepoints and may impact @@ -377,6 +381,10 @@ fn default_compare_cap() -> u32 { 64 } +fn default_backtrace_depth() -> u8 { + ghostscope_compiler::DEFAULT_BACKTRACE_DEPTH +} + fn default_enable_sysmon_for_target() -> bool { true } @@ -465,6 +473,7 @@ impl Default for EbpfConfig { mem_dump_cap: default_mem_dump_cap(), compare_cap: default_compare_cap(), max_trace_event_size: default_max_trace_event_size(), + backtrace_depth: default_backtrace_depth(), enable_sysmon_for_target: default_enable_sysmon_for_target(), } } @@ -543,6 +552,19 @@ impl EbpfConfig { )); } + if !(1..=ghostscope_compiler::MAX_BACKTRACE_DEPTH).contains(&self.backtrace_depth) { + return Err(anyhow::anyhow!( + "❌ Invalid eBPF configuration in '{}':\n\n\ + backtrace_depth {} is out of range\n\n\ + 💡 Valid range: 1 to {} frames\n\ + Recommended: {} frames", + file_path, + self.backtrace_depth, + ghostscope_compiler::MAX_BACKTRACE_DEPTH, + ghostscope_compiler::DEFAULT_BACKTRACE_DEPTH + )); + } + Ok(()) } } @@ -851,6 +873,30 @@ mod tests { assert!(Config::default().ebpf.enable_sysmon_for_target); } + #[test] + fn ebpf_defaults_backtrace_depth_to_maximum() { + assert_eq!( + Config::default().ebpf.backtrace_depth, + ghostscope_compiler::DEFAULT_BACKTRACE_DEPTH + ); + } + + #[test] + fn ebpf_rejects_backtrace_depth_zero() { + let error = toml::from_str::( + r#" + [ebpf] + backtrace_depth = 0 + "#, + ) + .expect("toml parses") + .ebpf + .validate("test.toml") + .expect_err("backtrace_depth=0 should be rejected"); + + assert!(error.to_string().contains("backtrace_depth")); + } + #[test] fn ebpf_rejects_legacy_shared_lib_sysmon_key() { let error = toml::from_str::( diff --git a/ghostscope/src/config/user.rs b/ghostscope/src/config/user.rs index 9785e635..bd15ab09 100644 --- a/ghostscope/src/config/user.rs +++ b/ghostscope/src/config/user.rs @@ -200,6 +200,9 @@ impl UserConfig { if args.enable_sysmon_for_target { ebpf_config.enable_sysmon_for_target = true; } + if let Some(backtrace_depth) = args.backtrace_depth { + ebpf_config.backtrace_depth = backtrace_depth; + } ebpf_config }, source: config.source, diff --git a/ghostscope/src/core/session.rs b/ghostscope/src/core/session.rs index 4833db6a..c065125e 100644 --- a/ghostscope/src/core/session.rs +++ b/ghostscope/src/core/session.rs @@ -458,6 +458,7 @@ mod tests { has_explicit_status_flag: false, script_timestamp: None, script_output_events_per_sec: None, + backtrace_depth: None, should_save_llvm_ir: false, should_save_ebpf: false, should_save_ast: false, diff --git a/ghostscope/src/script/compiler.rs b/ghostscope/src/script/compiler.rs index 2f7d74b0..3a712d28 100644 --- a/ghostscope/src/script/compiler.rs +++ b/ghostscope/src/script/compiler.rs @@ -178,6 +178,17 @@ async fn create_and_attach_loader( config.trace_context.variable_name_count() ); loader.set_trace_context(config.trace_context.clone()); + loader + .populate_backtrace_unwind_rows(&config.backtrace_unwind_rows) + .context("Failed to populate DWARF backtrace unwind rows")?; + loader + .register_backtrace_tail_call_program( + config + .backtrace_tail_call_program + .as_ref() + .map(|program| program.step_program_name.as_str()), + ) + .context("Failed to register bt tail-call program")?; // In -t mode (no attach PID), perform module prefill once per session and apply to this loader if attach_pid.is_none() { diff --git a/ghostscope/src/trace/backtrace.rs b/ghostscope/src/trace/backtrace.rs new file mode 100644 index 00000000..893b28e1 --- /dev/null +++ b/ghostscope/src/trace/backtrace.rs @@ -0,0 +1,848 @@ +use ghostscope_dwarf::{DwarfAnalyzer, FunctionParameter, ModuleAddress, PcContext}; +use ghostscope_process::{PidOffsetsEntry, ProcessManager}; +use ghostscope_protocol::trace_event::{ + backtrace_error_label, BacktraceStatus, BACKTRACE_FLAG_INLINE, BACKTRACE_FLAG_RAW, +}; +use ghostscope_protocol::{ParsedBacktraceFrame, ParsedInstruction, ParsedTraceEvent}; +use ghostscope_ui::{BacktraceDisplay, BacktraceDisplayFrame, TraceDisplayItem, UiTraceEvent}; +use std::collections::{BTreeSet, HashMap, VecDeque}; +use std::hash::Hash; +use std::path::{Path, PathBuf}; + +const FRAME_RENDER_CACHE_MAX_ENTRIES: usize = 16_384; +const STATUS_CACHE_MAX_ENTRIES: usize = 4_096; + +#[derive(Clone, Copy)] +struct ResolvedFrameModule<'a> { + entry: &'a PidOffsetsEntry, + pc: u64, +} + +#[derive(Debug)] +pub struct BacktraceRenderer { + frame_cache: SimpleCache>, + frame_display_cache: SimpleCache>, + status_cache: SimpleCache, +} + +impl Default for BacktraceRenderer { + fn default() -> Self { + Self { + frame_cache: SimpleCache::new(FRAME_RENDER_CACHE_MAX_ENTRIES), + frame_display_cache: SimpleCache::new(FRAME_RENDER_CACHE_MAX_ENTRIES), + status_cache: SimpleCache::new(STATUS_CACHE_MAX_ENTRIES), + } + } +} + +#[derive(Debug)] +struct SimpleCache { + entries: HashMap, + insertion_order: VecDeque, + max_entries: usize, +} + +impl SimpleCache +where + K: Clone + Eq + Hash, + V: Clone, +{ + fn new(max_entries: usize) -> Self { + Self { + entries: HashMap::new(), + insertion_order: VecDeque::new(), + max_entries, + } + } + + fn get(&self, key: &K) -> Option { + self.entries.get(key).cloned() + } + + fn insert(&mut self, key: K, value: V) { + if self.max_entries == 0 { + return; + } + if self.entries.insert(key.clone(), value).is_some() { + return; + } + + self.insertion_order.push_back(key); + while self.entries.len() > self.max_entries { + let Some(oldest) = self.insertion_order.pop_front() else { + break; + }; + self.entries.remove(&oldest); + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct PidCacheKey { + first: u32, + second: u32, + len: u8, +} + +impl PidCacheKey { + fn from_pids(pids: &[u32]) -> Self { + Self { + first: pids.first().copied().unwrap_or_default(), + second: pids.get(1).copied().unwrap_or_default(), + len: pids.len().min(2) as u8, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct FrameRenderCacheKey { + pids: PidCacheKey, + analyzer_present: bool, + index: u16, + flags: u8, + module_cookie: u64, + pc: u64, + raw_ip: u64, + frame_flags: u16, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct StatusCacheKey { + pids: PidCacheKey, + analyzer_present: bool, + module_cookie: u64, + pc: u64, + raw_ip: u64, + frame_flags: u16, + status: u8, + error_code: u16, +} + +impl BacktraceRenderer { + pub fn render_event_for_tui( + &mut self, + event: &ParsedTraceEvent, + analyzer: Option<&DwarfAnalyzer>, + coordinator: &ProcessManager, + proc_pid_hint: Option, + ) -> UiTraceEvent { + let mut items = Vec::new(); + let mut text_chunk = Vec::new(); + + for instruction in &event.instructions { + match instruction { + ParsedInstruction::Backtrace { .. } => { + flush_text_chunk(event, &mut text_chunk, &mut items); + let backtrace = self.format_backtrace_display( + instruction, + event.pid, + analyzer, + coordinator, + proc_pid_hint, + ); + items.push(TraceDisplayItem::Backtrace(backtrace)); + } + other => text_chunk.push(other.clone()), + } + } + flush_text_chunk(event, &mut text_chunk, &mut items); + + let execution_status = event.instructions.iter().rev().find_map(|instruction| { + if let ParsedInstruction::EndInstruction { + execution_status, .. + } = instruction + { + Some(*execution_status) + } else { + None + } + }); + + UiTraceEvent { + trace_id: event.trace_id, + timestamp: event.timestamp, + pid: event.pid, + tid: event.tid, + items, + execution_status, + } + } + + pub fn render_event_backtraces( + &mut self, + event: &ParsedTraceEvent, + analyzer: Option<&DwarfAnalyzer>, + coordinator: &ProcessManager, + proc_pid_hint: Option, + ) -> ParsedTraceEvent { + let mut changed = false; + let mut instructions = Vec::with_capacity(event.instructions.len()); + + for instruction in &event.instructions { + match instruction { + ParsedInstruction::Backtrace { .. } => { + changed = true; + for line in self.format_backtrace_instruction( + instruction, + event.pid, + analyzer, + coordinator, + proc_pid_hint, + ) { + instructions.push(ParsedInstruction::PrintString { content: line }); + } + } + other => instructions.push(other.clone()), + } + } + + if !changed { + return event.clone(); + } + + ParsedTraceEvent { + trace_id: event.trace_id, + timestamp: event.timestamp, + pid: event.pid, + tid: event.tid, + instructions, + } + } + + fn format_backtrace_instruction( + &mut self, + instruction: &ParsedInstruction, + event_pid: u32, + analyzer: Option<&DwarfAnalyzer>, + coordinator: &ProcessManager, + proc_pid_hint: Option, + ) -> Vec { + let ParsedInstruction::Backtrace { + requested_depth, + flags, + status, + error_code, + frames, + } = instruction + else { + return Vec::new(); + }; + + let pids = candidate_pids(event_pid, proc_pid_hint); + let display_status = self.display_backtrace_status( + *status, + *error_code, + frames, + analyzer, + coordinator, + &pids, + ); + let mut lines = vec![format_backtrace_header( + display_status, + frames.len(), + *requested_depth, + )]; + + for (index, frame) in frames.iter().enumerate() { + lines.extend(self.format_frame(index, frame, *flags, analyzer, coordinator, &pids)); + } + + if display_status != BacktraceStatus::Complete { + let suffix = match backtrace_error_label(*error_code) { + Some("unknown") => format!(" (code={error_code})"), + Some(label) => format!(" ({label}, code={error_code})"), + None => String::new(), + }; + lines.push(format!("stopped: {}{}", display_status.label(), suffix)); + } + + lines + } + + fn format_backtrace_display( + &mut self, + instruction: &ParsedInstruction, + event_pid: u32, + analyzer: Option<&DwarfAnalyzer>, + coordinator: &ProcessManager, + proc_pid_hint: Option, + ) -> BacktraceDisplay { + let ParsedInstruction::Backtrace { + requested_depth, + flags, + status, + error_code, + frames, + } = instruction + else { + return BacktraceDisplay { + requested_depth: 0, + physical_frame_count: 0, + status: BacktraceStatus::InternalError, + error_code: 0, + raw: false, + frames: Vec::new(), + }; + }; + + let pids = candidate_pids(event_pid, proc_pid_hint); + let display_status = self.display_backtrace_status( + *status, + *error_code, + frames, + analyzer, + coordinator, + &pids, + ); + + let mut display_frames = Vec::new(); + for (index, frame) in frames.iter().enumerate() { + display_frames.extend(self.display_frame( + index, + frame, + *flags, + analyzer, + coordinator, + &pids, + )); + } + + BacktraceDisplay { + requested_depth: *requested_depth, + physical_frame_count: frames.len(), + status: display_status, + error_code: *error_code, + raw: (*flags & BACKTRACE_FLAG_RAW) != 0, + frames: display_frames, + } + } + + fn display_backtrace_status( + &mut self, + status: BacktraceStatus, + error_code: u16, + frames: &[ParsedBacktraceFrame], + analyzer: Option<&DwarfAnalyzer>, + coordinator: &ProcessManager, + pids: &[u32], + ) -> BacktraceStatus { + if status != BacktraceStatus::UnsupportedCfi { + return status; + } + + let Some(analyzer) = analyzer else { + return status; + }; + let Some(last_frame) = frames.last() else { + return status; + }; + + let cache_key = StatusCacheKey { + pids: PidCacheKey::from_pids(pids), + analyzer_present: true, + module_cookie: last_frame.module_cookie, + pc: last_frame.pc, + raw_ip: last_frame.raw_ip, + frame_flags: last_frame.flags, + status: status as u8, + error_code, + }; + if let Some(cached) = self.status_cache.get(&cache_key) { + return cached; + } + + let Some(module) = resolve_frame_module(coordinator, pids, last_frame) else { + self.status_cache.insert(cache_key, status); + return status; + }; + + let display_status = + if is_process_entry_frame(analyzer, module.entry.module_path.as_ref(), module.pc) { + BacktraceStatus::Complete + } else { + status + }; + self.status_cache.insert(cache_key, display_status); + display_status + } + + fn format_frame( + &mut self, + index: usize, + frame: &ParsedBacktraceFrame, + flags: u8, + analyzer: Option<&DwarfAnalyzer>, + coordinator: &ProcessManager, + pids: &[u32], + ) -> Vec { + let cache_key = FrameRenderCacheKey { + pids: PidCacheKey::from_pids(pids), + analyzer_present: analyzer.is_some(), + index: index.min(u16::MAX as usize) as u16, + flags, + module_cookie: frame.module_cookie, + pc: frame.pc, + raw_ip: frame.raw_ip, + frame_flags: frame.flags, + }; + if let Some(cached) = self.frame_cache.get(&cache_key) { + return cached; + } + + let raw = (flags & BACKTRACE_FLAG_RAW) != 0; + let inline = (flags & BACKTRACE_FLAG_INLINE) != 0; + let module = resolve_frame_module(coordinator, pids, frame); + let frame_pc = module.map(|module| module.pc).unwrap_or(frame.pc); + + if raw { + let lines = vec![format_raw_frame(index, frame, module, true)]; + self.frame_cache.insert(cache_key, lines.clone()); + return lines; + } + + let resolved = analyzer + .and_then(|analyzer| module.map(|module| (analyzer, module))) + .and_then(|(analyzer, module)| { + let lookup_pc = if index == 0 { + module.pc + } else { + module.pc.saturating_sub(1) + }; + let address = + ModuleAddress::new(PathBuf::from(&module.entry.module_path), lookup_pc); + analyzer.resolve_pc(&address).ok() + }); + + let Some(ctx) = resolved else { + let lines = vec![format_raw_frame(index, frame, module, false)]; + self.frame_cache.insert(cache_key, lines.clone()); + return lines; + }; + + let mut lines = Vec::new(); + if inline { + for inline_frame in &ctx.inline_chain { + if let Some(name) = &inline_frame.function_name { + let location = inline_frame + .call_site + .as_ref() + .map(format_line_info) + .unwrap_or_else(|| "??".to_string()); + lines.push(format!(" #{index}.inline {name} at {location}")); + } + } + } + + let function = ctx.function_name.as_deref().unwrap_or(""); + let function = analyzer + .map(|analyzer| format_function_signature(function, analyzer, &ctx)) + .unwrap_or_else(|| function.to_string()); + let location = ctx + .line + .as_ref() + .map(format_line_info) + .unwrap_or_else(|| "??".to_string()); + let module_text = module + .map(|module| format_module_offset(&module.entry.module_path, frame_pc)) + .unwrap_or_else(|| format!("0x{:x}", frame.pc)); + lines.push(format!( + " #{index} {function} at {location} [{module_text}]" + )); + self.frame_cache.insert(cache_key, lines.clone()); + lines + } + + fn display_frame( + &mut self, + index: usize, + frame: &ParsedBacktraceFrame, + flags: u8, + analyzer: Option<&DwarfAnalyzer>, + coordinator: &ProcessManager, + pids: &[u32], + ) -> Vec { + let cache_key = FrameRenderCacheKey { + pids: PidCacheKey::from_pids(pids), + analyzer_present: analyzer.is_some(), + index: index.min(u16::MAX as usize) as u16, + flags, + module_cookie: frame.module_cookie, + pc: frame.pc, + raw_ip: frame.raw_ip, + frame_flags: frame.flags, + }; + if let Some(cached) = self.frame_display_cache.get(&cache_key) { + return cached; + } + + let raw = (flags & BACKTRACE_FLAG_RAW) != 0; + let inline = (flags & BACKTRACE_FLAG_INLINE) != 0; + let module = resolve_frame_module(coordinator, pids, frame); + let frame_pc = module.map(|module| module.pc).unwrap_or(frame.pc); + + if raw { + let frames = vec![raw_display_frame(index, frame, module, true)]; + self.frame_display_cache.insert(cache_key, frames.clone()); + return frames; + } + + let resolved = analyzer + .and_then(|analyzer| module.map(|module| (analyzer, module))) + .and_then(|(analyzer, module)| { + let lookup_pc = if index == 0 { + module.pc + } else { + module.pc.saturating_sub(1) + }; + let address = + ModuleAddress::new(PathBuf::from(&module.entry.module_path), lookup_pc); + analyzer.resolve_pc(&address).ok() + }); + + let Some(ctx) = resolved else { + let frames = vec![raw_display_frame(index, frame, module, false)]; + self.frame_display_cache.insert(cache_key, frames.clone()); + return frames; + }; + + let module_text = module + .map(|module| format_module_offset(&module.entry.module_path, frame_pc)) + .unwrap_or_else(|| format!("0x{:x}", frame.pc)); + + let mut display_frames = Vec::new(); + if inline { + for inline_frame in &ctx.inline_chain { + if let Some(name) = &inline_frame.function_name { + display_frames.push(BacktraceDisplayFrame { + index, + inline: true, + function: Some(name.clone()), + parameters: Vec::new(), + address: None, + location: inline_frame.call_site.as_ref().map(format_line_info), + module: module_text.clone(), + raw_ip: None, + cookie: None, + flags: None, + }); + } + } + } + + display_frames.push(BacktraceDisplayFrame { + index, + inline: false, + function: Some( + ctx.function_name + .clone() + .unwrap_or_else(|| "".to_string()), + ), + parameters: analyzer + .map(|analyzer| format_function_parameters(analyzer, &ctx)) + .unwrap_or_default(), + address: None, + location: ctx.line.as_ref().map(format_line_info), + module: module_text, + raw_ip: None, + cookie: None, + flags: None, + }); + + self.frame_display_cache + .insert(cache_key, display_frames.clone()); + display_frames + } +} + +fn flush_text_chunk( + event: &ParsedTraceEvent, + text_chunk: &mut Vec, + items: &mut Vec, +) { + if text_chunk.is_empty() { + return; + } + + let chunk_event = ParsedTraceEvent { + trace_id: event.trace_id, + timestamp: event.timestamp, + pid: event.pid, + tid: event.tid, + instructions: std::mem::take(text_chunk), + }; + items.extend( + chunk_event + .to_formatted_output() + .into_iter() + .map(|content| TraceDisplayItem::Text { content }), + ); +} + +fn is_process_entry_frame(analyzer: &DwarfAnalyzer, module_path: &str, pc: u64) -> bool { + let module_path = Path::new(module_path); + if let Some(entry) = analyzer.module_entry_address(module_path) { + if pc >= entry && pc < entry.saturating_add(0x100) { + return true; + } + } + + analyzer + .lookup_function_addresses("_start") + .into_iter() + .any(|start| { + start.module_path.as_path() == module_path + && pc >= start.address + && pc < start.address.saturating_add(0x100) + }) +} + +fn candidate_pids(event_pid: u32, proc_pid_hint: Option) -> Vec { + let mut seen = BTreeSet::new(); + if let Some(pid) = proc_pid_hint { + seen.insert(pid); + } + seen.insert(event_pid); + seen.into_iter().collect() +} + +fn format_backtrace_header(status: BacktraceStatus, frames: usize, requested_depth: u8) -> String { + let frame_word = if frames == 1 { "frame" } else { "frames" }; + format!( + "backtrace: {}, {} {} (max {})", + status.label(), + frames, + frame_word, + requested_depth + ) +} + +fn resolve_frame_module<'a>( + coordinator: &'a ProcessManager, + pids: &[u32], + frame: &ParsedBacktraceFrame, +) -> Option> { + for pid in pids { + if let Some(entries) = coordinator.cached_offsets_with_paths_for_pid(*pid) { + if let Some(entry) = entries.iter().find(|entry| { + frame.raw_ip >= entry.base && frame.raw_ip < entry.base.saturating_add(entry.size) + }) { + return Some(ResolvedFrameModule { + entry, + pc: frame.raw_ip.saturating_sub(entry.offsets.text), + }); + } + if let Some(entry) = entries + .iter() + .find(|entry| entry.cookie == frame.module_cookie && frame.pc < entry.size) + { + return Some(ResolvedFrameModule { + entry, + pc: frame.pc, + }); + } + } + } + None +} + +fn format_raw_frame( + index: usize, + frame: &ParsedBacktraceFrame, + module: Option>, + include_metadata: bool, +) -> String { + let pc = module.map(|module| module.pc).unwrap_or(frame.pc); + let module_text = module + .map(|module| format_module_offset(&module.entry.module_path, pc)) + .unwrap_or_else(|| format!("0x{:x}", frame.pc)); + let metadata_text = if include_metadata { + let cookie = module + .map(|module| module.entry.cookie) + .unwrap_or(frame.module_cookie); + format!(" raw=0x{:x} cookie=0x{:016x}", frame.raw_ip, cookie) + } else { + String::new() + }; + let flags_text = if include_metadata && frame.flags != 0 { + format!(" flags=0x{:x}", frame.flags) + } else { + String::new() + }; + format!(" #{index} 0x{pc:x} [{module_text}]{metadata_text}{flags_text}") +} + +fn raw_display_frame( + index: usize, + frame: &ParsedBacktraceFrame, + module: Option>, + include_metadata: bool, +) -> BacktraceDisplayFrame { + let pc = module.map(|module| module.pc).unwrap_or(frame.pc); + let module_text = module + .map(|module| format_module_offset(&module.entry.module_path, pc)) + .unwrap_or_else(|| format!("0x{:x}", frame.pc)); + let cookie = include_metadata.then(|| { + module + .map(|module| module.entry.cookie) + .unwrap_or(frame.module_cookie) + }); + + BacktraceDisplayFrame { + index, + inline: false, + function: None, + parameters: Vec::new(), + address: Some(format!("0x{pc:x}")), + location: None, + module: module_text, + raw_ip: include_metadata.then_some(frame.raw_ip), + cookie, + flags: (include_metadata && frame.flags != 0).then_some(frame.flags), + } +} + +fn format_line_info(line: &ghostscope_dwarf::PcLineInfo) -> String { + match line.column { + Some(column) => format!("{}:{}:{}", line.file_path, line.line_number, column), + None => format!("{}:{}", line.file_path, line.line_number), + } +} + +fn format_function_signature(function: &str, analyzer: &DwarfAnalyzer, ctx: &PcContext) -> String { + let parameters = format_function_parameters(analyzer, ctx); + if parameters.is_empty() { + return function.to_string(); + } + + format!("{function}({})", parameters.join(", ")) +} + +fn format_function_parameters(analyzer: &DwarfAnalyzer, ctx: &PcContext) -> Vec { + let Ok(mut parameters) = analyzer.function_parameters(ctx) else { + return Vec::new(); + }; + parameters.retain(|parameter| !parameter.is_artificial); + parameters.dedup_by(|a, b| a.name == b.name && a.type_name == b.type_name); + parameters.iter().map(format_parameter).collect::>() +} + +fn format_parameter(parameter: &FunctionParameter) -> String { + let type_name = normalize_signature_type(¶meter.type_name); + let name = parameter.name.trim(); + if type_name.is_empty() || type_name == "unknown" { + return name.to_string(); + } + if name.is_empty() { + return type_name; + } + format!("{type_name} {name}") +} + +fn normalize_signature_type(type_name: &str) -> String { + type_name.split_whitespace().collect::>().join(" ") +} + +fn format_module_offset(module_path: &str, pc: u64) -> String { + let name = Path::new(module_path) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(module_path); + format!("{name}+0x{pc:x}") +} + +#[cfg(test)] +mod tests { + use super::*; + use ghostscope_protocol::trace_event::BacktraceStatus; + + #[test] + fn renders_raw_backtrace_without_dwarf_context() { + let event = ParsedTraceEvent { + trace_id: 1, + timestamp: 0, + pid: 10, + tid: 11, + instructions: vec![ + ParsedInstruction::PrintString { + content: "before".to_string(), + }, + ParsedInstruction::Backtrace { + requested_depth: 3, + flags: BACKTRACE_FLAG_RAW, + status: BacktraceStatus::UnsupportedCfi, + error_code: 0, + frames: vec![ParsedBacktraceFrame { + module_cookie: 0x1234, + pc: 0x5678, + raw_ip: 0x7fff_0000_5678, + flags: 0, + }], + }, + ParsedInstruction::PrintString { + content: "after".to_string(), + }, + ], + }; + let coordinator = ProcessManager::new(); + let rendered = + BacktraceRenderer::default().render_event_backtraces(&event, None, &coordinator, None); + let output = rendered.to_formatted_output(); + + assert_eq!(output[0], "before"); + assert_eq!(output[1], "backtrace: unsupported CFI, 1 frame (max 3)"); + assert!(output.iter().any(|line| line.contains("#0 0x5678"))); + assert!(output + .iter() + .any(|line| line.contains("stopped: unsupported CFI"))); + assert_eq!(output.last().map(String::as_str), Some("after")); + } + + #[test] + fn renders_backtrace_as_structured_tui_item() { + let event = ParsedTraceEvent { + trace_id: 1, + timestamp: 0, + pid: 10, + tid: 11, + instructions: vec![ + ParsedInstruction::PrintString { + content: "before".to_string(), + }, + ParsedInstruction::Backtrace { + requested_depth: 3, + flags: BACKTRACE_FLAG_RAW, + status: BacktraceStatus::UnsupportedCfi, + error_code: 0, + frames: vec![ParsedBacktraceFrame { + module_cookie: 0x1234, + pc: 0x5678, + raw_ip: 0x7fff_0000_5678, + flags: 0, + }], + }, + ParsedInstruction::PrintString { + content: "after".to_string(), + }, + ], + }; + let coordinator = ProcessManager::new(); + let rendered = + BacktraceRenderer::default().render_event_for_tui(&event, None, &coordinator, None); + + assert_eq!(rendered.items.len(), 3); + assert!(matches!( + &rendered.items[0], + TraceDisplayItem::Text { content } if content == "before" + )); + let TraceDisplayItem::Backtrace(backtrace) = &rendered.items[1] else { + panic!("expected structured backtrace item"); + }; + assert_eq!(backtrace.status, BacktraceStatus::UnsupportedCfi); + assert_eq!(backtrace.frames.len(), 1); + assert_eq!(backtrace.frames[0].address.as_deref(), Some("0x5678")); + assert!(matches!( + &rendered.items[2], + TraceDisplayItem::Text { content } if content == "after" + )); + } +} diff --git a/ghostscope/src/trace/mod.rs b/ghostscope/src/trace/mod.rs index 9b023673..027ebd20 100644 --- a/ghostscope/src/trace/mod.rs +++ b/ghostscope/src/trace/mod.rs @@ -1,5 +1,6 @@ //! Trace module - manages trace instances and their lifecycle +pub mod backtrace; pub mod instance; pub mod manager; pub mod snapshot; diff --git a/ghostscope/src/tui/coordinator.rs b/ghostscope/src/tui/coordinator.rs index fe131919..02a33e09 100644 --- a/ghostscope/src/tui/coordinator.rs +++ b/ghostscope/src/tui/coordinator.rs @@ -2,8 +2,7 @@ use crate::config::ResolvedConfig; use crate::core::GhostSession; use crate::tui::{dwarf_loader, info_handlers, source_handlers, trace_handlers}; use anyhow::Result; -use ghostscope_protocol::ParsedTraceEvent; -use ghostscope_ui::{EventRegistry, RuntimeChannels, RuntimeCommand, RuntimeStatus}; +use ghostscope_ui::{EventRegistry, RuntimeChannels, RuntimeCommand, RuntimeStatus, UiTraceEvent}; use tokio::sync::mpsc::error::TrySendError; use tracing::{error, info, warn}; @@ -37,8 +36,8 @@ enum TraceForwardResult { } fn forward_trace_event( - trace_sender: &tokio::sync::mpsc::Sender, - event_data: ParsedTraceEvent, + trace_sender: &tokio::sync::mpsc::Sender, + event_data: UiTraceEvent, backpressure_state: &mut TraceBackpressureState, ) -> TraceForwardResult { match trace_sender.try_send(event_data) { @@ -144,6 +143,7 @@ async fn run_runtime_coordinator( // Create trace sender for event polling task let trace_sender = runtime_channels.create_trace_sender(); let trace_channel_capacity = runtime_channels.trace_channel_capacity; + let mut backtrace_renderer = crate::trace::backtrace::BacktraceRenderer::default(); let mut backpressure_state = TraceBackpressureState::default(); let mut backpressure_report_ticker = tokio::time::interval(tokio::time::Duration::from_secs(1)); backpressure_report_ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); @@ -162,11 +162,23 @@ async fn run_runtime_coordinator( }, if session.is_some() => { match result { Ok(events) => { - if let Some(ref _session) = session { + if let Some(ref session) = session { if !events.is_empty() { tracing::debug!("Forwarding {} trace events to UI", events.len()); } for event_data in events { + let event_data = { + let coordinator = session + .coordinator + .lock() + .expect("coordinator mutex poisoned"); + backtrace_renderer.render_event_for_tui( + &event_data, + session.process_analyzer.as_ref(), + &coordinator, + session.proc_pid(), + ) + }; match forward_trace_event( &trace_sender, event_data, @@ -710,13 +722,14 @@ async fn handle_load_traces( mod tests { use super::*; - fn sample_event(trace_id: u64) -> ParsedTraceEvent { - ParsedTraceEvent { + fn sample_event(trace_id: u64) -> UiTraceEvent { + UiTraceEvent { timestamp: 0, trace_id, pid: 1000, tid: 1000, - instructions: vec![], + items: vec![], + execution_status: None, } }