From 2006c63354e89a00ae4f723789720287a579d6cd Mon Sep 17 00:00:00 2001 From: swananan Date: Fri, 28 Aug 2026 21:18:48 +0800 Subject: [PATCH] feat: add opt-in sleepable uprobes --- config-zh.toml | 9 + config.toml | 10 + docs/configuration.md | 51 ++ docs/limitations.md | 2 +- docs/roadmap.md | 4 +- docs/zh/configuration.md | 45 ++ docs/zh/limitations.md | 2 +- docs/zh/roadmap.md | 4 +- e2e-tests/tests/script_execution.rs | 99 ++++ .../src/ebpf/codegen/backtrace/plan.rs | 100 +++- ghostscope-compiler/src/ebpf/context.rs | 136 ++++-- .../src/ebpf/helper_functions.rs | 97 ++-- ghostscope-dwarf/src/core/diagnostic.rs | 3 + .../src/semantics/variable_plan/tests.rs | 1 + ghostscope-loader/src/kernel_caps.rs | 438 +++++++++++++++++- ghostscope/src/cli/script_runtime.rs | 2 + ghostscope/src/config/args.rs | 25 + ghostscope/src/config/runtime.rs | 87 +++- ghostscope/src/config/settings.rs | 16 + ghostscope/src/config/user.rs | 3 + ghostscope/src/core/session.rs | 3 + 21 files changed, 1059 insertions(+), 78 deletions(-) diff --git a/config-zh.toml b/config-zh.toml index ec9e8e2e..54fe2aea 100644 --- a/config-zh.toml +++ b/config-zh.toml @@ -263,6 +263,15 @@ backtrace_depth = 128 # 默认值：false force_perf_event_array = false +# 使用可睡眠的 `uprobe.s` 程序替代普通 `uprobe` 程序。 +# 这是显式开启的实验性选项：需要 Linux 5.18+；开启后，固定长度的用户态 +# 内存读取可使用 bpf_copy_from_user_task()，从而允许处理用户页缺失。 +# 需要 NUL 终止语义的字符串读取仍使用 bpf_probe_read_user_str()。 +# sleepable uprobe 必须使用 RingBuf，不能与 force_perf_event_array 同时开启。 +# 较深的 DWARF bt 在内核允许时使用 tail call；否则限制为五帧 inline 回溯并打印警告。 +# 默认值：false +sleepable_uprobe = false + # 源代码路径配置 # 当 DWARF 调试信息中包含的编译时路径与运行时路径不同时， # 使用这些设置帮助 ghostscope 定位实际的源文件。 diff --git a/config.toml b/config.toml index 19ca7244..432e16b9 100644 --- a/config.toml +++ b/config.toml @@ -283,6 +283,16 @@ backtrace_depth = 128 # Default: false force_perf_event_array = false +# Use sleepable `uprobe.s` programs instead of regular `uprobe` programs. +# This is opt-in because it requires Linux 5.18+ and can let fixed-size user +# memory reads use bpf_copy_from_user_task(), which may fault in user pages. +# NUL-terminated string reads keep bpf_probe_read_user_str() semantics. +# Sleepable uprobes require RingBuf; do not combine with force_perf_event_array. +# Long DWARF bt uses tail calls when supported; otherwise it is limited to five +# inline frames and GhostScope prints a warning. +# Default: false +sleepable_uprobe = false + # Source code path configuration # When DWARF debug info contains compilation-time paths that differ from runtime paths, # use these settings to help ghostscope locate the actual source files. diff --git a/docs/configuration.md b/docs/configuration.md index 95acb24a..7e8ecd1b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -184,6 +184,10 @@ ghostscope --source-panel # Show source panel # WARNING: Testing purposes only. Forces PerfEventArray even on kernels >= 5.8 ghostscope --force-perf-event-array +# Opt in to sleepable uprobes for this invocation. This overrides +# sleepable_uprobe=false in the config file. +ghostscope --sleepable-uprobe + # Standalone -t starts target-mode sysmon by default. When -t is combined with # -p, GhostScope uses the -p watched-PID module-refresh path instead. # WARNING: Attaches system-wide lifecycle tracepoints (exec/fork/exit) and may @@ -194,6 +198,45 @@ ghostscope --force-perf-event-array ghostscope --enable-sysmon-for-target ``` +### Sleepable Uprobes + +Sleepable uprobes are an opt-in `[ebpf]` configuration because they require a +Linux 5.18+ kernel and can change probe latency by allowing a user-memory page +fault to be serviced during a probe hit. The default remains the regular, +non-sleepable `uprobe` path. + +```toml +[ebpf] +sleepable_uprobe = true +``` + +For a one-off run, use `--sleepable-uprobe`; it enables this setting even when +the config file has `sleepable_uprobe = false`. To keep regular uprobes, omit +the flag and leave the configuration at its default `false`. + +Sleepable uprobes require RingBuf event output. Do not combine +`sleepable_uprobe = true` (or `--sleepable-uprobe`) with +`force_perf_event_array = true` (or `--force-perf-event-array`): the kernel +does not permit sleepable BPF programs to use `BPF_MAP_TYPE_PERF_EVENT_ARRAY`, +and GhostScope reports this configuration conflict before attaching. + +When enabled, GhostScope emits `uprobe.s` programs. Fixed-size user-memory +reads use `bpf_copy_from_user_task()` so pages can be faulted in; NUL-terminated +string reads retain `bpf_probe_read_user_str()` because its early-NUL and +reported-length semantics differ. Startup fails clearly if the kernel does not +support the helper required by sleepable uprobe mode rather than silently +falling back. + +Long DWARF `bt` traces normally use a tail-call step program. GhostScope probes +that sleepable tail-call path at startup; on kernels that do not allow a +sleepable program to use `BPF_MAP_TYPE_PROG_ARRAY`, it compiles the trace with +the inline limit of five frames instead and prints a warning. This preserves a +working sleepable trace rather than failing eBPF verification. + +Enable this only when fault-capable reads are worth the added latency and kernel +requirement. Set `sleepable_uprobe = false` (the default) to use regular +uprobes again. + ### BPFFS Maintenance GhostScope uses bpffs because some runtime state must be shared across the userspace process layer, the loader, and the eBPF programs themselves. In practice, maps such as `proc_module_offsets` and `allowed_pids` are pinned into bpffs so later stages can reopen and reuse the same kernel maps by path instead of recreating them. GhostScope places these pins under a per-instance `pid-starttime` directory to avoid collisions between concurrent runs. @@ -276,6 +319,7 @@ unusable index is reported in CLI/TUI startup status before falling back. | `--source-panel` | | Show source panel | On | | `--config ` | | Custom config file | Auto-detect | | `--force-perf-event-array` | | Force PerfEventArray (testing) | Off | +| `--sleepable-uprobe` | | Enable sleepable uprobes for this run; overrides a `false` config value | Off | | `--enable-sysmon-for-target` | | Re-enable target-mode sysmon for standalone `-t` when config disables it. Standalone `-t` enables target-mode sysmon by default; `-t -p` uses the `-p` watched-PID module-refresh path instead. | Off | | `[BINARY] [ARGS...]` | | Launch target program with positional arguments | None | | `--args [ARGS...]` | | Separate GhostScope options from target program arguments | None | @@ -542,6 +586,13 @@ backtrace_unwind_rows_max_entries = 65536 # overhead compared to RingBuf and should only be used for compatibility testing. force_perf_event_array = false # Default (auto-detect based on kernel version) +# Opt-in sleepable uprobes. Requires Linux 5.18+. +# Fixed-size user-memory reads use bpf_copy_from_user_task(). +# NUL-terminated string reads retain bpf_probe_read_user_str() semantics. +# Long DWARF bt uses tail calls when the kernel allows them; otherwise it is +# limited to five inline frames and GhostScope prints a warning. +sleepable_uprobe = false # Default + # Start sysmon eBPF for standalone -t targets. # Maintains ASLR offsets for late-start processes and runtime module refresh for # processes that map the target or other backtrace modules later. diff --git a/docs/limitations.md b/docs/limitations.md index adf96203..6f2960c5 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -49,7 +49,7 @@ development time. JIT language support is an even more distant goal. ### 2. User-Memory Reads via `bpf_probe_read_user` In traditional non-sleepable probe paths, helpers such as `bpf_probe_read_user` cannot resolve user-space page faults, so reads from a target virtual address may still fail if the page is not resident or otherwise faults on access. -This is no longer an absolute eBPF limitation. Linux now supports sleepable uprobes (`uprobe.s` / `uretprobe.s`), and sleepable programs can use helpers such as `bpf_copy_from_user_task()` for fault-capable user-memory reads. GhostScope currently emits regular `uprobe` programs, so this remains a practical limitation today, but it is better described as a soft implementation limitation rather than a fundamental design limit of eBPF. +This is no longer an absolute eBPF limitation. GhostScope can emit sleepable `uprobe.s` programs when `[ebpf].sleepable_uprobe = true`; fixed-size user-memory reads then use `bpf_copy_from_user_task()` for fault-capable reads. The option is disabled by default and requires Linux 5.18+ because servicing a fault can increase the traced thread's latency. NUL-terminated string reads retain `bpf_probe_read_user_str()` semantics, so sleepable mode does not guarantee that every user-memory access can fault in a page. **References**: - https://lists.iovisor.org/g/iovisor-dev/topic/accessing_user_memory_and/21386221 diff --git a/docs/roadmap.md b/docs/roadmap.md index db300cca..581a4bbf 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -20,9 +20,9 @@ GhostScope is still evolving quickly. The milestones below are ordered from “s - See [Container support and limits](container.md) and [Limitations](limitations.md#10-container--wsl-limitations-for--p-pid-mode). ## Uprobe enhancements -- Add support for sleepable uprobes (`uprobe.s` / `uretprobe.s`) so GhostScope can use sleepable helpers where appropriate, especially for more reliable user-memory reads. +- Sleepable entry uprobes (`uprobe.s`) are available as an opt-in `[ebpf].sleepable_uprobe` setting. Regular `uprobe` remains the default compatibility path. - Add support for multi-attach uprobes (`uprobe.multi` / `uretprobe.multi`) to scale better when a script expands into many probe points. -- Keep compatibility fallbacks for kernels or libbpf/Aya paths that still require regular `uprobe` attachments. +- Continue improving compatibility diagnostics for kernels or Aya paths that only support regular `uprobe` attachments. ## Stack Unwinding - DWARF-only `bt` / `backtrace` is now supported for compact CFI rows that can diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index e62d72e6..6c513793 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -184,6 +184,10 @@ ghostscope --source-panel # 显示源码面板 # 警告：仅用于测试目的。即使在内核 >= 5.8 上也强制使用 PerfEventArray ghostscope --force-perf-event-array +# 为本次运行显式启用 sleepable uprobe；会覆盖配置文件中的 +# sleepable_uprobe=false。 +ghostscope --sleepable-uprobe + # 独立 -t 默认启动 target-mode sysmon。-t 与 -p 同时使用时， # GhostScope 会改用 -p 的 watched-PID 模块刷新路径。 # 警告：该选项会全局附加生命周期 tracepoint（exec/fork/exit），也可能 @@ -194,6 +198,40 @@ ghostscope --force-perf-event-array ghostscope --enable-sysmon-for-target ``` +### Sleepable Uprobe + +sleepable uprobe 是 `[ebpf]` 下显式开启的配置项：它要求 Linux 5.18+，并且 +探针命中时允许为用户态内存处理缺页，因此可能增加目标线程的探针延迟。默认仍是 +普通、不可睡眠的 `uprobe` 路径。 + +```toml +[ebpf] +sleepable_uprobe = true +``` + +如只需单次开启，可使用 `--sleepable-uprobe`；即使配置文件中设置了 +`sleepable_uprobe = false`，该参数也会启用它。若要继续使用普通 uprobe，请不传 +该参数，并保留默认配置 `false`。 + +sleepable uprobe 必须使用 RingBuf 输出。不要将 `sleepable_uprobe = true`（或 +`--sleepable-uprobe`）与 `force_perf_event_array = true`（或 +`--force-perf-event-array`）同时使用：内核不允许 sleepable BPF 程序使用 +`BPF_MAP_TYPE_PERF_EVENT_ARRAY`，GhostScope 会在 attach 前明确报告该配置冲突。 + +开启后，GhostScope 会生成 `uprobe.s` 程序。固定长度的用户态内存读取会使用 +`bpf_copy_from_user_task()`，使相关页面能够被 fault-in；需要 NUL 终止和返回实际 +长度语义的字符串读取仍使用 `bpf_probe_read_user_str()`。如果内核不支持 +sleepable uprobe 模式所需的 helper，启动会给出明确错误，而不会静默回退到普通 +uprobe。 + +较深的 DWARF `bt` 通常使用 tail-call step program。GhostScope 会在启动时探测 +sleepable tail-call 路径；如果内核不允许 sleepable 程序使用 +`BPF_MAP_TYPE_PROG_ARRAY`，则会将该回溯编译为最多五帧的 inline 路径并打印警告。 +这样 sleepable trace 仍可工作，而不会在 eBPF verifier 阶段失败。 + +只有在 fault-capable 读取的收益值得额外延迟和内核要求时才应开启；将 +`sleepable_uprobe = false`（默认值）即可恢复普通 uprobe。 + ### BPFFS 维护 GhostScope 使用 bpffs，是因为有一部分运行时状态需要在用户态的 process 层、loader 以及 eBPF 程序之间共享。实际里像 `proc_module_offsets` 和 `allowed_pids` 这样的 map，会先 pin 到 bpffs，这样后续阶段就能按路径重新打开并复用同一个内核 map，而不是重复创建。GhostScope 又把这些 pin 放在按实例隔离的 `pid-starttime` 目录下，用来避免多个实例并发运行时互相冲突。 @@ -275,6 +313,7 @@ GhostScope 会使用该原生索引选择 CU，并按需调用 fast parser。否 | `--source-panel` | | 显示源码面板 | 开 | | `--config ` | | 自定义配置文件 | 自动检测 | | `--force-perf-event-array` | | 强制 PerfEventArray（测试） | 关 | +| `--sleepable-uprobe` | | 为本次运行启用 sleepable uprobe；覆盖配置中的 `false` | 关 | | `--enable-sysmon-for-target` | | 当配置关闭 sysmon 时，重新为独立 `-t` 开启 target-mode sysmon。独立 `-t` 默认开启；`-t -p` 改用 `-p` 的 watched-PID 模块刷新路径。 | 关 | | `[BINARY] [ARGS...]` | | 启动目标程序并传递位置参数 | 无 | | `--args [ARGS...]` | | 分隔 GhostScope 选项和目标程序参数 | 无 | @@ -533,6 +572,12 @@ backtrace_unwind_rows_max_entries = 65536 # 有性能开销，仅应用于兼容性测试。 force_perf_event_array = false # 默认（根据内核版本自动检测） +# 显式开启 sleepable uprobe；要求 Linux 5.18+。 +# 固定长度用户态内存读取使用 bpf_copy_from_user_task()。 +# 需要 NUL 终止语义的字符串读取仍使用 bpf_probe_read_user_str()。 +# 较深的 DWARF bt 在内核允许时使用 tail call；否则限制为五帧 inline 回溯并打印警告。 +sleepable_uprobe = false # 默认关闭 + # 为独立 -t 目标启动 sysmon eBPF， # 用于维护后续启动进程里的 ASLR 偏移，以及后续映射目标模块或其他 # backtrace 模块时的运行时模块刷新。 diff --git a/docs/zh/limitations.md b/docs/zh/limitations.md index 5551ede6..0ada484b 100644 --- a/docs/zh/limitations.md +++ b/docs/zh/limitations.md @@ -40,7 +40,7 @@ Hash 表条目中的嵌套 adapter 会在配置的递归深度、集合宽度和 ### 2. 通过 `bpf_probe_read_user` 读取用户内存 在传统的非 sleepable probe 路径里，`bpf_probe_read_user` 这类 helper 仍然不能处理用户态缺页，因此当目标虚拟地址对应的页面尚未驻留，或访问时会触发 fault，读取就可能失败。 -但这已经不是 eBPF 的绝对硬限制。Linux 已支持 sleepable uprobe（`uprobe.s` / `uretprobe.s`），而 sleepable 程序可以使用 `bpf_copy_from_user_task()` 之类的 helper 执行可睡眠的用户态内存读取。GhostScope 当前仍生成普通 `uprobe` 程序，所以现阶段它在实践中仍然是个限制；但更准确地说，这属于实现层面的软限制，而不是 eBPF 的根本设计上限。 +但这已经不是 eBPF 的绝对硬限制。将 `[ebpf].sleepable_uprobe` 设为 `true` 后，GhostScope 会生成可睡眠的 `uprobe.s` 程序；固定长度的用户态内存读取会使用 `bpf_copy_from_user_task()`，从而支持可处理缺页的读取。该选项默认关闭，并要求 Linux 5.18+，因为处理缺页可能增加被跟踪线程的延迟。需要 NUL 终止语义的字符串读取仍使用 `bpf_probe_read_user_str()`，所以 sleepable 模式并不保证每一种用户态内存访问都能 fault-in 页面。 **参考**： - https://lists.iovisor.org/g/iovisor-dev/topic/accessing_user_memory_and/21386221 diff --git a/docs/zh/roadmap.md b/docs/zh/roadmap.md index 893d283a..d8aa0bb8 100644 --- a/docs/zh/roadmap.md +++ b/docs/zh/roadmap.md @@ -20,9 +20,9 @@ GhostScope 仍处在快速演进阶段，以下里程碑按照“优先修补基 - 详见[容器支持与限制](container.md)和[限制列表](limitations.md#10-容器-wsl-场景下--pid-pid-模式的软限制)。 ## Uprobe 增强 -- 支持 sleepable uprobe（`uprobe.s` / `uretprobe.s`），在合适场景下使用可睡眠 helper，尤其提升用户态内存读取的可靠性。 +- sleepable 入口 uprobe（`uprobe.s`）已通过 `[ebpf].sleepable_uprobe` 配置项提供；普通 `uprobe` 仍是默认的兼容路径。 - 支持 multi-attach uprobe（`uprobe.multi` / `uretprobe.multi`），让脚本展开出大量探针点时仍能保持更好的扩展性。 -- 对暂时只能走普通 `uprobe` 的内核或 libbpf/Aya 路径保留兼容性回退。 +- 继续改进只支持普通 `uprobe` 的内核或 Aya 路径的兼容性诊断。 ## 栈回溯（Stack Unwinding） - 已支持 DWARF-only `bt` / `backtrace`，用于可安全降到 eBPF 执行的 diff --git a/e2e-tests/tests/script_execution.rs b/e2e-tests/tests/script_execution.rs index 8cc31d24..c6d31d02 100644 --- a/e2e-tests/tests/script_execution.rs +++ b/e2e-tests/tests/script_execution.rs @@ -302,6 +302,105 @@ trace print_record { Ok(()) } +#[tokio::test] +async fn test_sleepable_uprobe_cli_opt_in_overrides_config_and_reads_user_memory( +) -> anyhow::Result<()> { + init(); + ensure_global_cleanup_registered(); + + let target = get_global_test_target_with_opt(OptimizationLevel::Debug).await?; + let script_content = r#" +trace process_record { + if (record.value > 0) { + print "SLEEPABLE_USER_READ"; + } +} +"#; + + let (exit_code, stdout, stderr) = common::runner::GhostscopeRunner::new() + .with_script(script_content) + .attach_to(&target) + .timeout_secs(5) + .enable_sysmon_for_target(false) + .with_config_content( + r#" +[ebpf] +sleepable_uprobe = false +"#, + ) + .with_cli_args(["--sleepable-uprobe"]) + .run() + .await?; + + assert_eq!(exit_code, 0, "stderr={stderr} stdout={stdout}"); + assert!( + stdout.contains("SLEEPABLE_USER_READ"), + "sleepable uprobe did not read the record field. stdout={stdout} stderr={stderr}" + ); + Ok(()) +} + +#[tokio::test] +async fn test_sleepable_uprobe_backtrace_uses_a_kernel_supported_depth() -> anyhow::Result<()> { + init(); + ensure_global_cleanup_registered(); + + let target = get_global_test_target_with_opt(OptimizationLevel::Debug).await?; + let script_content = r#" +trace test_function { + print "before-sleepable-bt"; + bt full; + print "after-sleepable-bt"; +} +"#; + + let (exit_code, stdout, stderr) = common::runner::GhostscopeRunner::new() + .with_script(script_content) + .attach_to(&target) + .timeout_secs(5) + .enable_sysmon_for_target(false) + .with_cli_args(["--sleepable-uprobe", "--backtrace-depth", "6"]) + .run() + .await?; + + assert_eq!(exit_code, 0, "stderr={stderr} stdout={stdout}"); + let backtrace = stdout + .find("backtrace:") + .ok_or_else(|| anyhow::anyhow!("missing backtrace header:\n{stdout}"))?; + assert!( + stdout[backtrace..].contains("(max 5)") || stdout[backtrace..].contains("(max 6)"), + "sleepable bt should preserve depth on kernels with sleepable tail calls or fall back to five inline frames:\n{stdout}" + ); + assert!( + stdout.contains("after-sleepable-bt"), + "sleepable backtrace did not finish:\n{stdout}" + ); + Ok(()) +} + +#[tokio::test] +async fn test_sleepable_uprobe_rejects_forced_perf_event_array() -> anyhow::Result<()> { + init(); + ensure_global_cleanup_registered(); + + let target = get_global_test_target_with_opt(OptimizationLevel::Debug).await?; + let (exit_code, _stdout, stderr) = common::runner::GhostscopeRunner::new() + .with_script("trace process_record { print record.value; }") + .attach_to(&target) + .timeout_secs(5) + .enable_sysmon_for_target(false) + .with_cli_args(["--sleepable-uprobe", "--force-perf-event-array"]) + .run() + .await?; + + assert_ne!(exit_code, 0, "conflicting options unexpectedly succeeded"); + assert!( + stderr.contains("sleepable BPF programs cannot use BPF_MAP_TYPE_PERF_EVENT_ARRAY"), + "missing conflict explanation: stderr={stderr}" + ); + Ok(()) +} + #[tokio::test] async fn test_backtrace_outputs_dwarf_frames_between_prints() -> anyhow::Result<()> { init(); diff --git a/ghostscope-compiler/src/ebpf/codegen/backtrace/plan.rs b/ghostscope-compiler/src/ebpf/codegen/backtrace/plan.rs index b1bc1034..38c6cb1f 100644 --- a/ghostscope-compiler/src/ebpf/codegen/backtrace/plan.rs +++ b/ghostscope-compiler/src/ebpf/codegen/backtrace/plan.rs @@ -100,6 +100,9 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { } fn required_backtrace_tail_call_slots(&self, statements: &[Statement]) -> u8 { + if !self.backtrace_tail_calls_supported() { + return 1; + } let depth = self .compile_options .backtrace_depth @@ -195,21 +198,32 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { } pub(super) fn plan_backtrace_instruction( - &self, + &mut self, stmt: &BacktraceStatement, ) -> BacktraceInstructionPlan { - let depth = self + let requested_depth = self .compile_options .backtrace_depth .clamp(1, crate::MAX_BACKTRACE_DEPTH); + let tail_call_required = self.backtrace_tail_call_required(requested_depth); + let depth = if tail_call_required && !self.backtrace_tail_calls_supported() { + if !self.warned_sleepable_tail_call_fallback { + tracing::warn!( + requested_depth, + inline_depth = BPF_INLINE_BACKTRACE_FRAME_LIMIT, + "Sleepable uprobe tail calls are unavailable; limiting bt depth to the inline limit" + ); + self.warned_sleepable_tail_call_fallback = true; + } + BPF_INLINE_BACKTRACE_FRAME_LIMIT + } else { + requested_depth + }; let flags = backtrace_flags(stmt); let payload_size = BACKTRACE_DATA_SIZE + depth as usize * std::mem::size_of::(); let instruction_size = INSTRUCTION_HEADER_SIZE + payload_size; - let mode = if depth > BPF_INLINE_BACKTRACE_FRAME_LIMIT - && !self.backtrace_unwind_rows.is_empty() - && self.current_compile_time_context.is_some() - { + let mode = if self.backtrace_tail_call_required(depth) { BacktraceEmitMode::TailCall } else { BacktraceEmitMode::Inline @@ -223,6 +237,33 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { instruction_size, } } + + pub(crate) fn should_generate_backtrace_tail_call_maps( + &self, + statements: &[Statement], + ) -> bool { + statements_have_backtrace(statements) + && self.backtrace_tail_calls_supported() + && self.backtrace_tail_call_required( + self.compile_options + .backtrace_depth + .clamp(1, crate::MAX_BACKTRACE_DEPTH), + ) + } + + fn backtrace_tail_call_required(&self, depth: u8) -> bool { + depth > BPF_INLINE_BACKTRACE_FRAME_LIMIT + && !self.backtrace_unwind_rows.is_empty() + && self.current_compile_time_context.is_some() + } + + fn backtrace_tail_calls_supported(&self) -> bool { + !self.compile_options.runtime_capabilities.sleepable_uprobe + || self + .compile_options + .runtime_capabilities + .sleepable_tail_calls + } } fn statements_have_backtrace(statements: &[Statement]) -> bool { @@ -328,4 +369,51 @@ mod tests { INSTRUCTION_HEADER_SIZE + plan.payload_size ); } + + #[test] + fn sleepable_backtrace_without_tail_calls_uses_the_inline_limit() { + let context = inkwell::context::Context::create(); + let mut options = crate::CompileOptions::default(); + options.runtime_capabilities.sleepable_uprobe = true; + options.runtime_capabilities.sleepable_tail_calls = false; + let mut ctx = EbpfContext::new(&context, "test", None, &options).expect("context"); + let stmt = BacktraceStatement::default(); + + ctx.backtrace_unwind_rows + .push(ghostscope_protocol::BacktraceUnwindRow::default()); + ctx.current_compile_time_context = Some(CompileTimeContext { + pc_address: 0x1234, + module_path: "/bin/test".to_string(), + }); + + let plan = ctx.plan_backtrace_instruction(&stmt); + assert_eq!(plan.mode, BacktraceEmitMode::Inline); + assert_eq!(plan.depth, BPF_INLINE_BACKTRACE_FRAME_LIMIT); + assert!( + !ctx.should_generate_backtrace_tail_call_maps(&[Statement::Backtrace(stmt.clone())]) + ); + } + + #[test] + fn sleepable_backtrace_with_tail_calls_keeps_the_tail_call_path() { + let context = inkwell::context::Context::create(); + let mut options = crate::CompileOptions::default(); + options.runtime_capabilities.sleepable_uprobe = true; + options.runtime_capabilities.sleepable_tail_calls = true; + let mut ctx = EbpfContext::new(&context, "test", None, &options).expect("context"); + let stmt = BacktraceStatement::default(); + + ctx.backtrace_unwind_rows + .push(ghostscope_protocol::BacktraceUnwindRow::default()); + ctx.current_compile_time_context = Some(CompileTimeContext { + pc_address: 0x1234, + module_path: "/bin/test".to_string(), + }); + + assert_eq!( + ctx.plan_backtrace_instruction(&stmt).mode, + BacktraceEmitMode::TailCall + ); + assert!(ctx.should_generate_backtrace_tail_call_maps(&[Statement::Backtrace(stmt)])); + } } diff --git a/ghostscope-compiler/src/ebpf/context.rs b/ghostscope-compiler/src/ebpf/context.rs index 6a57fe36..4f124a36 100644 --- a/ghostscope-compiler/src/ebpf/context.rs +++ b/ghostscope-compiler/src/ebpf/context.rs @@ -187,6 +187,7 @@ pub struct EbpfContext<'ctx, 'dw> { pub(crate) backtrace_module_row_ranges: Vec, pub(crate) backtrace_tail_call_slots: u8, pub(crate) next_backtrace_tail_call_slot: u8, + pub(crate) warned_sleepable_tail_call_fallback: bool, pub(crate) pending_backtrace_tail_call: Option, pub(crate) backtrace_tail_enabled_alloca: Option>, pub(crate) backtrace_tail_last_slot_alloca: Option>, @@ -197,6 +198,14 @@ pub struct EbpfContext<'ctx, 'dw> { } impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { + fn uprobe_section(&self) -> &'static str { + if self.compile_options.runtime_capabilities.sleepable_uprobe { + "uprobe.s" + } else { + "uprobe" + } + } + pub(crate) fn backtrace_unwind_row_map_entries(&self) -> u64 { (self.compile_options.backtrace_unwind_rows_max_entries as u64) .max(self.backtrace_unwind_rows.len() as u64) @@ -305,6 +314,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { backtrace_module_row_ranges: Vec::new(), backtrace_tail_call_slots: 1, next_backtrace_tail_call_slot: 0, + warned_sleepable_tail_call_fallback: false, pending_backtrace_tail_call: None, backtrace_tail_enabled_alloca: None, backtrace_tail_last_slot_alloca: None, @@ -418,10 +428,11 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let function = self.module.add_function(function_name, fn_type, None); - // Set section attribute for uprobe + // Set the section that tells Aya whether the uprobe is sleepable. function.add_attribute( inkwell::attributes::AttributeLoc::Function, - self.context.create_string_attribute("section", "uprobe"), + self.context + .create_string_attribute("section", self.uprobe_section()), ); // Create basic block @@ -571,6 +582,8 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { None }; self.prepare_backtrace_unwind_rows(trace_statements); + let create_backtrace_tail_call_maps = + self.should_generate_backtrace_tail_call_maps(trace_statements); // Create required maps - critical for eBPF loader // Create event output map based on compile options @@ -723,29 +736,31 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { )) })?; } - self.map_manager - .create_percpu_array_map( - &self.module, - &self.di_builder, - &self.compile_unit, - "bt_state", - self.backtrace_tail_call_slots.max(1) as u64, - crate::BACKTRACE_TAIL_STATE_SIZE as u64, - ) - .map_err(|e| { - CodeGenError::LLVMError(format!("Failed to create bt_state map: {e}")) - })?; - self.map_manager - .create_program_array_map( - &self.module, - &self.di_builder, - &self.compile_unit, - "bt_prog_array", - 1, - ) - .map_err(|e| { - CodeGenError::LLVMError(format!("Failed to create bt_prog_array map: {e}")) - })?; + if create_backtrace_tail_call_maps { + self.map_manager + .create_percpu_array_map( + &self.module, + &self.di_builder, + &self.compile_unit, + "bt_state", + self.backtrace_tail_call_slots.max(1) as u64, + crate::BACKTRACE_TAIL_STATE_SIZE as u64, + ) + .map_err(|e| { + CodeGenError::LLVMError(format!("Failed to create bt_state map: {e}")) + })?; + self.map_manager + .create_program_array_map( + &self.module, + &self.di_builder, + &self.compile_unit, + "bt_prog_array", + 1, + ) + .map_err(|e| { + CodeGenError::LLVMError(format!("Failed to create bt_prog_array map: {e}")) + })?; + } } // Variables are now queried on-demand when accessed in expressions @@ -804,8 +819,8 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let fn_type = i32_type.fn_type(&[ptr_type.into()], false); let function = self.module.add_function(function_name, fn_type, None); - // CRITICAL: Set section name for eBPF loader to find the function - function.set_section(Some("uprobe")); + // The section name selects regular or sleepable uprobe loading in Aya. + function.set_section(Some(self.uprobe_section())); // Create basic block and position builder let basic_block = self.context.append_basic_block(function, "entry"); @@ -844,7 +859,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let ptr_type = self.context.ptr_type(AddressSpace::default()); let fn_type = i32_type.fn_type(&[ptr_type.into()], false); let function = self.module.add_function(function_name, fn_type, None); - function.set_section(Some("uprobe")); + function.set_section(Some(self.uprobe_section())); let basic_block = self.context.append_basic_block(function, "entry"); self.builder.position_at_end(basic_block); @@ -1269,3 +1284,68 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn llvm_ir_with_basic_program(options: &crate::CompileOptions) -> String { + let llvm_context = Context::create(); + let mut codegen = + EbpfContext::new(&llvm_context, "section_test", Some(1), options).expect("context"); + codegen + .create_basic_ebpf_function("test_program") + .expect("basic program"); + codegen + .get_module() + .print_to_string() + .to_string_lossy() + .into_owned() + } + + #[test] + fn regular_uprobe_is_the_default_section() { + let llvm_ir = llvm_ir_with_basic_program(&crate::CompileOptions::default()); + assert!(llvm_ir.contains("uprobe")); + assert!(!llvm_ir.contains("uprobe.s")); + } + + #[test] + fn sleepable_option_emits_uprobe_sleepable_section() { + let mut options = crate::CompileOptions::default(); + options.runtime_capabilities.sleepable_uprobe = true; + + let llvm_ir = llvm_ir_with_basic_program(&options); + assert!(llvm_ir.contains("uprobe.s")); + } + + #[test] + fn sleepable_fixed_size_reads_use_copy_from_user_task() { + let llvm_context = Context::create(); + let mut options = crate::CompileOptions::default(); + options.runtime_capabilities.sleepable_uprobe = true; + options.runtime_capabilities.copy_from_user_task = true; + let mut codegen = + EbpfContext::new(&llvm_context, "read_test", Some(1), &options).expect("context"); + codegen + .create_basic_ebpf_function("test_program") + .expect("basic program"); + let address = RuntimeAddress::available( + llvm_context.i64_type().const_int(0x1000, false), + &llvm_context, + ); + codegen + .generate_memory_read(address, ghostscope_dwarf::MemoryAccessSize::U64, None) + .expect("memory read"); + + let llvm_ir = codegen + .get_module() + .print_to_string() + .to_string_lossy() + .into_owned(); + assert!(llvm_ir.contains("current_task_btf_for_user_read")); + assert!(llvm_ir.contains("i64 158")); + assert!(llvm_ir.contains("i64 191")); + assert!(!llvm_ir.contains("i64 112")); + } +} diff --git a/ghostscope-compiler/src/ebpf/helper_functions.rs b/ghostscope-compiler/src/ebpf/helper_functions.rs index 9d48fe52..96c58a8b 100644 --- a/ghostscope-compiler/src/ebpf/helper_functions.rs +++ b/ghostscope-compiler/src/ebpf/helper_functions.rs @@ -5,9 +5,10 @@ use super::context::{CodeGenError, EbpfContext, Result, RuntimeAddress}; use aya_ebpf_bindings::bindings::bpf_func_id::{ - BPF_FUNC_get_current_pid_tgid, BPF_FUNC_get_current_task, BPF_FUNC_ktime_get_ns, - BPF_FUNC_map_lookup_elem, BPF_FUNC_perf_event_output, BPF_FUNC_probe_read_kernel, - BPF_FUNC_probe_read_user, BPF_FUNC_probe_read_user_str, BPF_FUNC_ringbuf_output, + BPF_FUNC_copy_from_user_task, BPF_FUNC_get_current_pid_tgid, BPF_FUNC_get_current_task, + BPF_FUNC_get_current_task_btf, BPF_FUNC_ktime_get_ns, BPF_FUNC_map_lookup_elem, + BPF_FUNC_perf_event_output, BPF_FUNC_probe_read_kernel, BPF_FUNC_probe_read_user, + BPF_FUNC_probe_read_user_str, BPF_FUNC_ringbuf_output, }; use ghostscope_dwarf::MemoryAccessSize; use ghostscope_platform::register_mapping; @@ -953,13 +954,6 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .into_pointer_value(); let i32_type = self.context.i32_type(); - let helper_id = i64_type.const_int(BPF_FUNC_probe_read_user as u64, false); - let helper_fn_type = - i32_type.fn_type(&[ptr_type.into(), i32_type.into(), ptr_type.into()], false); - let helper_fn_ptr = self - .builder - .build_int_to_ptr(helper_id, ptr_type, "probe_read_user_fn") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let size_val = i32_type.const_int(result_size as u64, false); let zero_i32 = i32_type.const_zero(); let effective_size = self @@ -972,28 +966,20 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { ) .map_err(|e| CodeGenError::LLVMError(e.to_string()))? .into_int_value(); - let call_args: Vec = - vec![dst_ptr.into(), effective_size.into(), src_ptr.into()]; - - let call_site = self - .builder - .build_indirect_call( - helper_fn_type, - helper_fn_ptr, - &call_args, + let ret_i64 = self + .create_bpf_helper_call( + BPF_FUNC_probe_read_user as u64, + &[dst_ptr, effective_size.into(), src_ptr.into()], + i64_type.into(), "probe_read_result", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let ret_iv = call_site.try_as_basic_value().left().ok_or_else(|| { - CodeGenError::LLVMError("Expected integer return from helper".to_string()) - })?; - let ret_i32 = ret_iv.into_int_value(); + )? + .into_int_value(); let read_fail = self .builder .build_int_compare( inkwell::IntPredicate::NE, - ret_i32, - i32_type.const_zero(), + ret_i64, + i64_type.const_zero(), "read_fail", ) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; @@ -1036,7 +1022,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { Ok(ProbeReadResult { loaded_i64, - helper_result: ret_i32, + helper_result: ret_i64, combined_fail, not_found, }) @@ -1281,6 +1267,61 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { args: &[BasicValueEnum<'ctx>], return_type: BasicTypeEnum<'ctx>, call_name: &str, + ) -> Result> { + if helper_id == BPF_FUNC_probe_read_user as u64 + && self.compile_options.runtime_capabilities.sleepable_uprobe + && self + .compile_options + .runtime_capabilities + .copy_from_user_task + { + return self.create_copy_from_user_task_call(args, return_type, call_name); + } + + self.create_raw_bpf_helper_call(helper_id, args, return_type, call_name) + } + + fn create_copy_from_user_task_call( + &mut self, + args: &[BasicValueEnum<'ctx>], + return_type: BasicTypeEnum<'ctx>, + call_name: &str, + ) -> Result> { + if args.len() != 3 { + return Err(CodeGenError::LLVMError(format!( + "bpf_copy_from_user_task replacement expected 3 arguments, got {}", + args.len() + ))); + } + + let ptr_type = self.context.ptr_type(AddressSpace::default()); + let task = self + .create_raw_bpf_helper_call( + BPF_FUNC_get_current_task_btf as u64, + &[], + ptr_type.into(), + "current_task_btf_for_user_read", + )? + .into_pointer_value(); + + let mut task_args = args.to_vec(); + task_args.push(task.into()); + task_args.push(self.context.i64_type().const_zero().into()); + self.create_raw_bpf_helper_call( + BPF_FUNC_copy_from_user_task as u64, + &task_args, + return_type, + call_name, + ) + } + + /// Create one raw eBPF helper call without applying user-memory helper policy. + fn create_raw_bpf_helper_call( + &mut self, + helper_id: u64, + args: &[BasicValueEnum<'ctx>], + return_type: BasicTypeEnum<'ctx>, + call_name: &str, ) -> Result> { use inkwell::types::BasicMetadataTypeEnum; diff --git a/ghostscope-dwarf/src/core/diagnostic.rs b/ghostscope-dwarf/src/core/diagnostic.rs index 68688dad..da600c16 100644 --- a/ghostscope-dwarf/src/core/diagnostic.rs +++ b/ghostscope-dwarf/src/core/diagnostic.rs @@ -101,6 +101,8 @@ impl DebugInfoSource { pub struct RuntimeCapabilities { pub regular_uprobe: bool, pub sleepable_uprobe: bool, + /// Whether a sleepable uprobe may use `bpf_tail_call` with a ProgramArray. + pub sleepable_tail_calls: bool, pub uprobe_multi: bool, pub copy_from_user_task: bool, pub max_bpf_stack_bytes: usize, @@ -132,6 +134,7 @@ impl Default for RuntimeCapabilities { Self { regular_uprobe: true, sleepable_uprobe: false, + sleepable_tail_calls: false, uprobe_multi: false, copy_from_user_task: false, max_bpf_stack_bytes: 512, diff --git a/ghostscope-dwarf/src/semantics/variable_plan/tests.rs b/ghostscope-dwarf/src/semantics/variable_plan/tests.rs index 5877b514..c22216e5 100644 --- a/ghostscope-dwarf/src/semantics/variable_plan/tests.rs +++ b/ghostscope-dwarf/src/semantics/variable_plan/tests.rs @@ -6,6 +6,7 @@ fn capabilities(regular_uprobe: bool) -> RuntimeCapabilities { RuntimeCapabilities { regular_uprobe, sleepable_uprobe: false, + sleepable_tail_calls: false, uprobe_multi: false, copy_from_user_task: false, max_bpf_stack_bytes: 512, diff --git a/ghostscope-loader/src/kernel_caps.rs b/ghostscope-loader/src/kernel_caps.rs index 0bfa1914..3e3b3cf6 100644 --- a/ghostscope-loader/src/kernel_caps.rs +++ b/ghostscope-loader/src/kernel_caps.rs @@ -2,8 +2,13 @@ use aya::{ maps::MapType, programs::ProgramType, sys::{is_helper_supported, is_map_supported, BpfHelper}, + util::KernelVersion, }; -use std::{fmt, sync::OnceLock}; +use aya_obj::generated::{ + bpf_attr, bpf_cmd, bpf_insn, bpf_map_type, bpf_prog_type, BPF_ALU64, BPF_CALL, BPF_DW, + BPF_EXIT, BPF_F_SLEEPABLE, BPF_IMM, BPF_JMP, BPF_K, BPF_LD, BPF_MOV, BPF_PSEUDO_MAP_FD, +}; +use std::{fmt, io, mem, sync::OnceLock}; use tracing::{error, info, warn}; /// Global cache for complete, hardware-backed kernel capability probes. @@ -73,6 +78,13 @@ pub struct KernelCapabilities { pub supports_perf_event_array: bool, /// Whether bpf_get_ns_current_pid_tgid helper is supported for kprobe/uprobe class programs. pub supports_ns_current_pid_tgid_helper: bool, + /// Whether the kernel supports the helpers required by GhostScope's sleepable + /// uprobe mode (`bpf_get_current_task_btf` and `bpf_copy_from_user_task`, + /// introduced in Linux 5.18). + pub supports_sleepable_uprobe: bool, + /// Whether sleepable KProbe-class programs can use a ProgramArray and + /// `bpf_tail_call`. This is required for long DWARF backtraces. + pub supports_sleepable_tail_calls: bool, } impl KernelCapabilities { @@ -142,10 +154,12 @@ where }; info!( - "Kernel eBPF startup summary: ringbuf_supported={} perf_event_array_supported={} helper_ns_current_pid_tgid={}", + "Kernel eBPF startup summary: ringbuf_supported={} perf_event_array_supported={} helper_ns_current_pid_tgid={} sleepable_uprobe={} sleepable_tail_calls={}", capabilities.supports_ringbuf, capabilities.supports_perf_event_array, - capabilities.supports_ns_current_pid_tgid_helper + capabilities.supports_ns_current_pid_tgid_helper, + capabilities.supports_sleepable_uprobe, + capabilities.supports_sleepable_tail_calls, ); Ok(capabilities) @@ -218,15 +232,39 @@ fn detect_full_capabilities() -> Result Result CapabilityProbe { } } +fn detect_sleepable_uprobe_support() -> CapabilityProbe { + info!( + "Probing helpers required for sleepable uprobes; bpf_copy_from_user_task is probed with BPF_F_SLEEPABLE..." + ); + + let task_btf = is_helper_supported( + ProgramType::KProbe, + BpfHelper::BPF_FUNC_get_current_task_btf, + ); + let copy_from_user_task = + probe_sleepable_kprobe_helper_support(BpfHelper::BPF_FUNC_copy_from_user_task); + + match (task_btf, copy_from_user_task) { + (Ok(true), Ok(true)) => CapabilityProbe::cacheable(true), + (Ok(false), _) => { + info!("bpf_get_current_task_btf helper support probe reported unsupported"); + CapabilityProbe::cacheable(false) + } + (_, Ok(false)) => { + info!("bpf_copy_from_user_task helper support probe reported unsupported"); + CapabilityProbe::cacheable(false) + } + (Err(task_err), _) => { + warn!("bpf_get_current_task_btf helper support probe failed unexpectedly: {task_err}"); + CapabilityProbe::uncacheable_unsupported() + } + (_, Err(copy_err)) => { + warn!("bpf_copy_from_user_task helper support probe failed unexpectedly: {copy_err}"); + CapabilityProbe::uncacheable_unsupported() + } + } +} + +fn detect_sleepable_tail_call_support() -> CapabilityProbe { + info!( + "Probing sleepable tail-call support with a sleepable KProbe caller, callee, and ProgramArray..." + ); + + match probe_sleepable_tail_call_support() { + Ok(true) => CapabilityProbe::cacheable(true), + Ok(false) => { + info!("sleepable tail-call capability probe reported unsupported"); + CapabilityProbe::cacheable(false) + } + Err(err) => { + warn!("sleepable tail-call capability probe failed unexpectedly: {err}"); + CapabilityProbe::uncacheable_unsupported() + } + } +} + +/// Probe one KProbe-class helper under the same sleepable program flag that +/// Aya uses for an `uprobe.s` section. +/// +/// `bpf_copy_from_user_task` is reported as an invalid helper by a regular +/// KProbe on kernels where it is intentionally restricted to sleepable +/// programs. The public Aya helper probe cannot set BPF program load flags. +// TODO(aya): Replace these raw probes after Aya exposes semantic sleepable +// uprobe capability probes, including support for sleepable tail calls. +fn probe_sleepable_kprobe_helper_support(helper: BpfHelper) -> Result { + let call = sleepable_helper_probe_instruction((BPF_JMP | BPF_CALL) as u8, helper as i32); + let exit = sleepable_helper_probe_instruction((BPF_JMP | BPF_EXIT) as u8, 0); + let instructions = [call, exit]; + match load_sleepable_kprobe_program(&instructions) { + Ok(fd) => { + close_bpf_fd(fd); + Ok(true) + } + Err(failure) => match classify_sleepable_helper_probe_failure( + failure.error.raw_os_error(), + failure.verifier_log.as_ref(), + ) { + Some(supported) => Ok(supported), + None => Err(failure.error), + }, + } +} + +/// Probe the complete kernel feature sequence used by a long sleepable `bt`: +/// load a sleepable callee, load a sleepable caller referencing a ProgramArray +/// and `bpf_tail_call`, then put the callee in that array. +fn probe_sleepable_tail_call_support() -> Result { + let map_fd = match create_program_array() { + Ok(fd) => fd, + Err(error) => return classify_sleepable_tail_call_syscall_failure(error), + }; + + let outcome = (|| { + let callee = [ + bpf_instruction((BPF_ALU64 | BPF_MOV | BPF_K) as u8, 0, 0, 0), + bpf_instruction((BPF_JMP | BPF_EXIT) as u8, 0, 0, 0), + ]; + let callee_fd = match load_sleepable_kprobe_program(&callee) { + Ok(fd) => fd, + Err(failure) => return classify_sleepable_tail_call_load_failure(failure), + }; + + let outcome = (|| { + let caller = sleepable_tail_call_probe_caller(map_fd); + let caller_fd = match load_sleepable_kprobe_program(&caller) { + Ok(fd) => fd, + Err(failure) => return classify_sleepable_tail_call_load_failure(failure), + }; + close_bpf_fd(caller_fd); + + match update_program_array(map_fd, callee_fd) { + Ok(()) => Ok(true), + Err(error) => classify_sleepable_tail_call_syscall_failure(error), + } + })(); + + close_bpf_fd(callee_fd); + outcome + })(); + + close_bpf_fd(map_fd); + outcome +} + +fn create_program_array() -> Result { + // SAFETY: bpf_attr is a C ABI union and zero initialization is valid for + // the fields not used by a BPF_MAP_CREATE request. + let mut attr = unsafe { mem::zeroed::() }; + // SAFETY: __bindgen_anon_1 is the BPF_MAP_CREATE member of bpf_attr. + let create = unsafe { &mut attr.__bindgen_anon_1 }; + create.map_type = bpf_map_type::BPF_MAP_TYPE_PROG_ARRAY as u32; + create.key_size = std::mem::size_of::() as u32; + create.value_size = std::mem::size_of::() as u32; + create.max_entries = 1; + bpf_syscall(bpf_cmd::BPF_MAP_CREATE, &mut attr) +} + +fn update_program_array(map_fd: libc::c_int, program_fd: libc::c_int) -> Result<(), io::Error> { + let key = 0u32; + let value = program_fd as u32; + // SAFETY: bpf_attr is a C ABI union and zero initialization is valid for + // the fields not used by a BPF_MAP_UPDATE_ELEM request. + let mut attr = unsafe { mem::zeroed::() }; + // SAFETY: __bindgen_anon_2 is the BPF_MAP_UPDATE_ELEM member of bpf_attr. + let update = unsafe { &mut attr.__bindgen_anon_2 }; + update.map_fd = map_fd as u32; + update.key = std::ptr::addr_of!(key) as u64; + update.__bindgen_anon_1.value = std::ptr::addr_of!(value) as u64; + + bpf_syscall(bpf_cmd::BPF_MAP_UPDATE_ELEM, &mut attr).map(|_| ()) +} + +fn sleepable_tail_call_probe_caller(map_fd: libc::c_int) -> [bpf_insn; 6] { + [ + bpf_instruction( + (BPF_LD | BPF_DW | BPF_IMM) as u8, + 2, + BPF_PSEUDO_MAP_FD as u8, + map_fd, + ), + bpf_instruction(0, 0, 0, 0), + bpf_instruction((BPF_ALU64 | BPF_MOV | BPF_K) as u8, 3, 0, 0), + bpf_instruction( + (BPF_JMP | BPF_CALL) as u8, + 0, + 0, + BpfHelper::BPF_FUNC_tail_call as i32, + ), + bpf_instruction((BPF_ALU64 | BPF_MOV | BPF_K) as u8, 0, 0, 0), + bpf_instruction((BPF_JMP | BPF_EXIT) as u8, 0, 0, 0), + ] +} + +struct SleepableProgramLoadFailure { + error: io::Error, + verifier_log: Box<[u8; 4096]>, +} + +fn load_sleepable_kprobe_program( + instructions: &[bpf_insn], +) -> Result { + let mut verifier_log = [0u8; 4096]; + + // SAFETY: bpf_attr and bpf_insn are C ABI structs. Zero initialization is + // valid for the fields left unset in a BPF_PROG_LOAD request. + let mut attr = unsafe { mem::zeroed::() }; + // SAFETY: __bindgen_anon_3 is the BPF_PROG_LOAD member of bpf_attr. + let load = unsafe { &mut attr.__bindgen_anon_3 }; + load.prog_type = bpf_prog_type::BPF_PROG_TYPE_KPROBE as u32; + load.insn_cnt = instructions.len() as u32; + load.insns = instructions.as_ptr() as u64; + load.license = c"GPL".as_ptr() as u64; + load.log_level = 1; + load.log_size = verifier_log.len() as u32; + load.log_buf = verifier_log.as_mut_ptr() as u64; + load.kern_version = KernelVersion::current().map_or(0, KernelVersion::code); + load.prog_flags = BPF_F_SLEEPABLE; + + bpf_syscall(bpf_cmd::BPF_PROG_LOAD, &mut attr).map_err(|error| SleepableProgramLoadFailure { + error, + verifier_log: Box::new(verifier_log), + }) +} + +fn bpf_syscall(command: bpf_cmd, attr: &mut bpf_attr) -> Result { + // SAFETY: the command-specific bpf_attr member was initialized by the + // caller and the supplied size is the full ABI union size expected by the + // BPF syscall. + let result = unsafe { + libc::syscall( + libc::SYS_bpf, + command as libc::c_long, + attr as *mut bpf_attr, + mem::size_of::(), + ) + }; + if result >= 0 { + Ok(result as libc::c_int) + } else { + Err(io::Error::last_os_error()) + } +} + +fn close_bpf_fd(fd: libc::c_int) { + // SAFETY: successful BPF syscalls return an owned file descriptor. + unsafe { libc::close(fd) }; +} + +fn bpf_instruction(code: u8, dst: u8, src: u8, imm: i32) -> bpf_insn { + let mut instruction = sleepable_helper_probe_instruction(code, imm); + instruction.set_dst_reg(dst); + instruction.set_src_reg(src); + instruction +} + +fn classify_sleepable_tail_call_syscall_failure(error: io::Error) -> Result { + if is_sleepable_tail_call_unsupported_errno(error.raw_os_error()) { + Ok(false) + } else { + Err(error) + } +} + +fn classify_sleepable_tail_call_load_failure( + failure: SleepableProgramLoadFailure, +) -> Result { + if is_sleepable_tail_call_unsupported_errno(failure.error.raw_os_error()) + || verifier_log_mentions_sleepable_tail_call_rejection(failure.verifier_log.as_ref()) + { + Ok(false) + } else { + Err(failure.error) + } +} + +fn is_sleepable_tail_call_unsupported_errno(error_code: Option) -> bool { + matches!(error_code, Some(code) if code == libc::EINVAL || code == libc::EOPNOTSUPP) +} + +fn verifier_log_mentions_sleepable_tail_call_rejection(verifier_log: &[u8]) -> bool { + let verifier_log = verifier_log + .split(|byte| *byte == 0) + .next() + .unwrap_or(verifier_log); + [ + b"sleepable".as_slice(), + b"tail call", + b"prog_array", + b"program array", + ] + .iter() + .any(|diagnostic| { + verifier_log + .windows(diagnostic.len()) + .any(|window| window.eq_ignore_ascii_case(diagnostic)) + }) +} + +fn sleepable_helper_probe_instruction(code: u8, imm: i32) -> bpf_insn { + // SAFETY: bpf_insn is a C ABI instruction structure; its zero value is a + // valid baseline before the opcode and immediate are assigned. + let mut instruction = unsafe { mem::zeroed::() }; + instruction.code = code; + instruction.imm = imm; + instruction +} + +fn classify_sleepable_helper_probe_failure( + error_code: Option, + verifier_log: &[u8], +) -> Option { + const UNSUPPORTED_HELPER_DIAGNOSTICS: &[&[u8]] = &[ + b"invalid func ", + b"unknown func ", + b"program of this type cannot use helper ", + ]; + + let verifier_log = verifier_log + .split(|byte| *byte == 0) + .next() + .unwrap_or(verifier_log); + if verifier_log.is_empty() { + return match error_code { + Some(libc::EINVAL | libc::E2BIG) => Some(false), + _ => None, + }; + } + + if UNSUPPORTED_HELPER_DIAGNOSTICS.iter().any(|diagnostic| { + verifier_log + .windows(diagnostic.len()) + .any(|window| window == *diagnostic) + }) { + Some(false) + } else { + // The helper was recognized. Invalid arguments are expected because + // this deliberately minimal probe supplies no helper arguments. + Some(true) + } +} + #[cfg(test)] mod tests { use super::*; @@ -342,11 +718,15 @@ mod tests { supports_ringbuf: bool, supports_perf_event_array: bool, supports_ns_current_pid_tgid_helper: bool, + supports_sleepable_uprobe: bool, + supports_sleepable_tail_calls: bool, ) -> KernelCapabilities { KernelCapabilities { supports_ringbuf, supports_perf_event_array, supports_ns_current_pid_tgid_helper, + supports_sleepable_uprobe, + supports_sleepable_tail_calls, } } @@ -360,8 +740,8 @@ mod tests { #[test] fn forced_perf_startup_does_not_populate_full_capabilities_cache() { let cache = KernelCapabilityCache::new(); - let perf_only_caps = caps(false, true, true); - let full_caps = caps(true, true, true); + let perf_only_caps = caps(false, true, true, true, true); + let full_caps = caps(true, true, true, true, true); let forced = detect_for_startup_with_detectors( true, @@ -397,8 +777,8 @@ mod tests { #[test] fn uncacheable_full_probe_result_is_not_cached() { let cache = KernelCapabilityCache::new(); - let uncacheable_caps = caps(false, true, false); - let cacheable_caps = caps(true, true, true); + let uncacheable_caps = caps(false, true, false, false, false); + let cacheable_caps = caps(true, true, true, true, true); let first = cache .get_or_detect(|| Ok(detection(uncacheable_caps, false))) @@ -419,4 +799,46 @@ mod tests { cacheable_caps ); } + + #[test] + fn sleepable_helper_probe_classifies_verifier_results() { + assert_eq!( + classify_sleepable_helper_probe_failure(Some(libc::EPERM), b"invalid func 191\0"), + Some(false) + ); + assert_eq!( + classify_sleepable_helper_probe_failure( + Some(libc::EPERM), + b"R4 type=scalar expected=ptr_\0" + ), + Some(true) + ); + assert_eq!( + classify_sleepable_helper_probe_failure(Some(libc::EINVAL), b"\0"), + Some(false) + ); + assert_eq!( + classify_sleepable_helper_probe_failure(Some(libc::EPERM), b"\0"), + None + ); + } + + #[test] + fn sleepable_tail_call_probe_classifies_kernel_rejections() { + assert!(is_sleepable_tail_call_unsupported_errno(Some(libc::EINVAL))); + assert!(is_sleepable_tail_call_unsupported_errno(Some( + libc::EOPNOTSUPP + ))); + assert!(!is_sleepable_tail_call_unsupported_errno(Some(libc::EPERM))); + + assert!(verifier_log_mentions_sleepable_tail_call_rejection( + b"sleepable programs cannot use prog_array\0" + )); + assert!(verifier_log_mentions_sleepable_tail_call_rejection( + b"Tail call is not allowed here\0" + )); + assert!(!verifier_log_mentions_sleepable_tail_call_rejection( + b"R1 type=scalar expected=ctx\0" + )); + } } diff --git a/ghostscope/src/cli/script_runtime.rs b/ghostscope/src/cli/script_runtime.rs index dab144f3..47811c73 100644 --- a/ghostscope/src/cli/script_runtime.rs +++ b/ghostscope/src/cli/script_runtime.rs @@ -732,6 +732,8 @@ mod tests { supports_ringbuf: true, supports_perf_event_array: true, supports_ns_current_pid_tgid_helper: false, + supports_sleepable_uprobe: false, + supports_sleepable_tail_calls: false, }, } } diff --git a/ghostscope/src/config/args.rs b/ghostscope/src/config/args.rs index 0d05ec8a..05dacfc7 100644 --- a/ghostscope/src/config/args.rs +++ b/ghostscope/src/config/args.rs @@ -305,6 +305,12 @@ pub struct Args { #[arg(long, action = clap::ArgAction::SetTrue)] pub force_perf_event_array: bool, + /// Enable sleepable uprobes for this run. Requires Linux 5.18+ and may + /// increase probe-hit latency when user-memory pages must be faulted in. + /// Cannot be combined with --force-perf-event-array. + #[arg(long, action = clap::ArgAction::SetTrue)] + pub sleepable_uprobe: bool, + /// Re-enable sysmon eBPF for standalone -t if config disabled it. /// Standalone -t starts sysmon by default; -t with -p does not use sysmon. #[arg( @@ -357,6 +363,7 @@ pub struct ParsedArgs { pub should_save_ast: bool, pub layout_mode: LayoutMode, pub force_perf_event_array: bool, + pub sleepable_uprobe: bool, pub enable_sysmon_for_target: bool, pub allow_loose_debug_match: bool, pub debuginfod: Option, @@ -539,6 +546,7 @@ impl Args { should_save_ast, layout_mode: parsed.layout, force_perf_event_array: parsed.force_perf_event_array, + sleepable_uprobe: parsed.sleepable_uprobe, enable_sysmon_for_target: parsed.enable_sysmon_for_target, allow_loose_debug_match: parsed.allow_loose_debug_match, debuginfod: parsed.debuginfod, @@ -916,6 +924,23 @@ mod tests { } } + #[test] + fn parses_sleepable_uprobe_flag() { + let parsed = Args::parse_args_from(vec![ + "ghostscope".to_string(), + "--pid".to_string(), + "1234".to_string(), + "--script-file".to_string(), + "trace.gs".to_string(), + "--sleepable-uprobe".to_string(), + ]); + + match parsed { + ParsedCommand::Trace(args) => assert!(args.sleepable_uprobe), + other => panic!("unexpected parse result: {other:?}"), + } + } + #[test] fn rejects_backtrace_depth_out_of_range() { let err = Args::try_parse_from(vec![ diff --git a/ghostscope/src/config/runtime.rs b/ghostscope/src/config/runtime.rs index 32b3fcd2..e0eb7e5d 100644 --- a/ghostscope/src/config/runtime.rs +++ b/ghostscope/src/config/runtime.rs @@ -8,7 +8,7 @@ use ghostscope_process::{ }; use tracing::{info, warn}; -use crate::config::{LayoutMode, UserConfig}; +use crate::config::{settings::EbpfConfig, LayoutMode, UserConfig}; #[derive(Debug, Clone, Default)] pub struct RuntimeContext { @@ -176,6 +176,8 @@ pub struct ResolvedConfig { impl ResolvedConfig { pub fn resolve(user: UserConfig, kernel_caps: &KernelCapabilities) -> Result { + validate_sleepable_uprobe_config(&user.ebpf_config, kernel_caps)?; + let runtime = RuntimeContext::resolve(&user, kernel_caps)?; Ok(Self { user, @@ -276,16 +278,44 @@ impl ResolvedConfig { special_pid_ns: self.runtime.special_pid_ns, proc_offsets_pid_ns: self.runtime.proc_offsets_pid_ns, input_pid: self.input_pid, - runtime_capabilities: dwarf_runtime_capabilities_from_kernel(&self.kernel_capabilities), + runtime_capabilities: dwarf_runtime_capabilities_from_kernel( + &self.kernel_capabilities, + self.ebpf_config.sleepable_uprobe, + ), } } } +fn validate_sleepable_uprobe_config( + ebpf_config: &EbpfConfig, + kernel_caps: &KernelCapabilities, +) -> Result<()> { + if ebpf_config.sleepable_uprobe && ebpf_config.force_perf_event_array { + return Err(anyhow::anyhow!( + "[ebpf].sleepable_uprobe=true conflicts with [ebpf].force_perf_event_array=true: sleepable BPF programs cannot use BPF_MAP_TYPE_PERF_EVENT_ARRAY. Sleepable uprobes require RingBuf event output. Disable force_perf_event_array or disable sleepable_uprobe." + )); + } + + if ebpf_config.sleepable_uprobe && !kernel_caps.supports_sleepable_uprobe { + return Err(anyhow::anyhow!( + "[ebpf].sleepable_uprobe=true requires Linux 5.18+ with sleepable uprobe and bpf_copy_from_user_task support. Disable the setting or use a compatible kernel." + )); + } + + Ok(()) +} + fn dwarf_runtime_capabilities_from_kernel( kernel_caps: &KernelCapabilities, + sleepable_uprobe_enabled: bool, ) -> ghostscope_compiler::RuntimeCapabilities { ghostscope_compiler::RuntimeCapabilities { regular_uprobe: kernel_caps.supports_ringbuf || kernel_caps.supports_perf_event_array, + sleepable_uprobe: sleepable_uprobe_enabled && kernel_caps.supports_sleepable_uprobe, + sleepable_tail_calls: sleepable_uprobe_enabled + && kernel_caps.supports_sleepable_uprobe + && kernel_caps.supports_sleepable_tail_calls, + copy_from_user_task: sleepable_uprobe_enabled && kernel_caps.supports_sleepable_uprobe, ..Default::default() } } @@ -311,3 +341,56 @@ fn map_pid_session_error(err: ResolvePidSessionError) -> anyhow::Error { } } } + +#[cfg(test)] +mod tests { + use super::*; + + fn kernel_caps( + supports_sleepable_uprobe: bool, + supports_sleepable_tail_calls: bool, + ) -> KernelCapabilities { + KernelCapabilities { + supports_ringbuf: true, + supports_perf_event_array: true, + supports_ns_current_pid_tgid_helper: true, + supports_sleepable_uprobe, + supports_sleepable_tail_calls, + } + } + + #[test] + fn sleepable_runtime_capability_requires_opt_in_and_kernel_support() { + let disabled = dwarf_runtime_capabilities_from_kernel(&kernel_caps(true, true), false); + assert!(!disabled.sleepable_uprobe); + assert!(!disabled.sleepable_tail_calls); + assert!(!disabled.copy_from_user_task); + + let unavailable = dwarf_runtime_capabilities_from_kernel(&kernel_caps(false, false), true); + assert!(!unavailable.sleepable_uprobe); + assert!(!unavailable.sleepable_tail_calls); + assert!(!unavailable.copy_from_user_task); + + let enabled = dwarf_runtime_capabilities_from_kernel(&kernel_caps(true, false), true); + assert!(enabled.sleepable_uprobe); + assert!(!enabled.sleepable_tail_calls); + assert!(enabled.copy_from_user_task); + + let tail_calls = dwarf_runtime_capabilities_from_kernel(&kernel_caps(true, true), true); + assert!(tail_calls.sleepable_tail_calls); + } + + #[test] + fn sleepable_uprobe_rejects_forced_perf_event_array() { + let mut config = crate::config::Config::default().ebpf; + config.sleepable_uprobe = true; + config.force_perf_event_array = true; + + let error = + validate_sleepable_uprobe_config(&config, &kernel_caps(true, true)).unwrap_err(); + + assert!(error + .to_string() + .contains("sleepable BPF programs cannot use BPF_MAP_TYPE_PERF_EVENT_ARRAY")); + } +} diff --git a/ghostscope/src/config/settings.rs b/ghostscope/src/config/settings.rs index b9e6db81..fc29e0af 100644 --- a/ghostscope/src/config/settings.rs +++ b/ghostscope/src/config/settings.rs @@ -255,6 +255,12 @@ pub struct EbpfConfig { /// even on kernels that support RingBuf. #[serde(default = "default_force_perf_event_array")] pub force_perf_event_array: bool, + /// Emit sleepable (`uprobe.s`) programs and use fault-capable user-memory reads + /// where the generated program uses fixed-size reads. Requires Linux 5.18+ + /// and RingBuf event output, so it cannot be combined with + /// force_perf_event_array. + #[serde(default = "default_sleepable_uprobe")] + pub sleepable_uprobe: bool, /// Per-argument memory dump cap for extended format specifiers ({:x}/{:s}) /// Default: 256 bytes; larger requests are truncated to this cap. #[serde(default = "default_mem_dump_cap")] @@ -390,6 +396,10 @@ fn default_force_perf_event_array() -> bool { false } +fn default_sleepable_uprobe() -> bool { + false +} + fn default_mem_dump_cap() -> u32 { 256 } @@ -537,6 +547,7 @@ impl Default for EbpfConfig { proc_module_offsets_max_entries: default_proc_module_offsets_max_entries(), backtrace_unwind_rows_max_entries: default_backtrace_unwind_rows_max_entries(), force_perf_event_array: default_force_perf_event_array(), + sleepable_uprobe: default_sleepable_uprobe(), mem_dump_cap: default_mem_dump_cap(), compare_cap: default_compare_cap(), max_trace_event_size: default_max_trace_event_size(), @@ -1043,6 +1054,11 @@ mod tests { assert!(Config::default().ebpf.enable_sysmon_for_target); } + #[test] + fn ebpf_defaults_disable_sleepable_uprobes() { + assert!(!Config::default().ebpf.sleepable_uprobe); + } + #[test] fn ebpf_defaults_backtrace_depth_to_maximum() { assert_eq!( diff --git a/ghostscope/src/config/user.rs b/ghostscope/src/config/user.rs index dff01c2f..ac59a083 100644 --- a/ghostscope/src/config/user.rs +++ b/ghostscope/src/config/user.rs @@ -201,6 +201,9 @@ impl UserConfig { if args.force_perf_event_array { ebpf_config.force_perf_event_array = true; } + if args.sleepable_uprobe { + ebpf_config.sleepable_uprobe = true; + } if args.enable_sysmon_for_target { ebpf_config.enable_sysmon_for_target = true; } diff --git a/ghostscope/src/core/session.rs b/ghostscope/src/core/session.rs index b67ab238..f77795e5 100644 --- a/ghostscope/src/core/session.rs +++ b/ghostscope/src/core/session.rs @@ -786,6 +786,7 @@ mod tests { should_save_ast: false, layout_mode: crate::config::LayoutMode::Horizontal, force_perf_event_array: false, + sleepable_uprobe: false, enable_sysmon_for_target: false, allow_loose_debug_match: false, debuginfod: None, @@ -822,6 +823,8 @@ mod tests { supports_ringbuf: true, supports_perf_event_array: true, supports_ns_current_pid_tgid_helper: false, + supports_sleepable_uprobe: false, + supports_sleepable_tail_calls: false, }, };