From eb08a41a7404ba546b58d6ff896d5a842fdb3df9 Mon Sep 17 00:00:00 2001 From: swananan Date: Sat, 29 Aug 2026 16:51:39 +0800 Subject: [PATCH] fix: gate sleepable probes behind opt-in Skip sleepable helper and tail-call probes unless sleepable uprobes are enabled. Preserve the one-argument startup detection API and expose sleepable probing through an explicitly named opt-in entry point. Keep separate base and full capability caches so an unprobed startup result cannot mask later opt-in detection. Add an e2e fixture that evicts an anonymous page and verifies only a sleepable uprobe can fault it back in and read it. --- .../fixtures/sample_program/sample_lib.c | 57 +++++ .../fixtures/sample_program/sample_lib.h | 3 + .../fixtures/sample_program/sample_program.c | 1 + e2e-tests/tests/script_execution.rs | 60 +++++ ghostscope-loader/src/kernel_caps.rs | 235 +++++++++++++----- ghostscope/src/main.rs | 12 +- 6 files changed, 302 insertions(+), 66 deletions(-) diff --git a/e2e-tests/tests/fixtures/sample_program/sample_lib.c b/e2e-tests/tests/fixtures/sample_program/sample_lib.c index 3a75035a..5738d38e 100644 --- a/e2e-tests/tests/fixtures/sample_program/sample_lib.c +++ b/e2e-tests/tests/fixtures/sample_program/sample_lib.c @@ -94,3 +94,60 @@ void sink_void(const void* p) { call_counter++; (void)p; } + +/* + * Keep these fixture-only declarations below the existing code so hard-coded + * sample_lib.c line probes remain stable. + */ +#include +#include + +typedef struct { + uint64_t value; +} EvictedPage; + +static EvictedPage* evicted_page; +static size_t evicted_page_size; + +// The body deliberately does not dereference page. Sleepable-uprobe tests +// attach here after the caller has confirmed that the page is nonresident. +__attribute__((noinline)) void process_evicted_page(const EvictedPage* page) { + __asm__ volatile("" : : "r"(page) : "memory"); +} + +void trigger_evicted_page_probe(void) { + if (evicted_page == NULL) { + long page_size = sysconf(_SC_PAGESIZE); + if (page_size <= 0) { + perror("sysconf(_SC_PAGESIZE)"); + return; + } + + evicted_page_size = (size_t)page_size; + evicted_page = mmap(NULL, evicted_page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (evicted_page == MAP_FAILED) { + evicted_page = NULL; + perror("mmap"); + return; + } + } + + // Materialize the page, discard it, and verify that no backing page is + // resident before entering the probe target. MADV_DONTNEED makes the next + // successful read fault in a zero-filled anonymous page. + evicted_page->value = UINT64_C(0x1122334455667788); + if (madvise(evicted_page, evicted_page_size, MADV_DONTNEED) != 0) { + perror("madvise(MADV_DONTNEED)"); + return; + } + + unsigned char residency = 1; + if (mincore(evicted_page, evicted_page_size, &residency) != 0) { + perror("mincore"); + return; + } + if ((residency & 1U) == 0) { + process_evicted_page(evicted_page); + } +} diff --git a/e2e-tests/tests/fixtures/sample_program/sample_lib.h b/e2e-tests/tests/fixtures/sample_program/sample_lib.h index 529fad1d..3d369438 100644 --- a/e2e-tests/tests/fixtures/sample_program/sample_lib.h +++ b/e2e-tests/tests/fixtures/sample_program/sample_lib.h @@ -35,4 +35,7 @@ void cleanup_test_lib(); // Void pointer sink for pointer-arithmetic fallback tests void sink_void(const void* p); +// Trigger a probe target with a confirmed nonresident anonymous page. +void trigger_evicted_page_probe(void); + #endif // TEST_LIB_H diff --git a/e2e-tests/tests/fixtures/sample_program/sample_program.c b/e2e-tests/tests/fixtures/sample_program/sample_program.c index 930c7c8b..a77bfc7b 100644 --- a/e2e-tests/tests/fixtures/sample_program/sample_program.c +++ b/e2e-tests/tests/fixtures/sample_program/sample_program.c @@ -71,6 +71,7 @@ int main() { // Void pointer sink call for pointer-arithmetic fallback tests sink_void(numbers); + trigger_evicted_page_probe(); // printf("Sleeping for 2 seconds...\n"); sleep(2); // Sleep for 2 seconds diff --git a/e2e-tests/tests/script_execution.rs b/e2e-tests/tests/script_execution.rs index c6d31d02..aec9b24b 100644 --- a/e2e-tests/tests/script_execution.rs +++ b/e2e-tests/tests/script_execution.rs @@ -340,6 +340,66 @@ sleepable_uprobe = false Ok(()) } +#[tokio::test] +async fn test_sleepable_uprobe_faults_in_an_evicted_user_page() -> anyhow::Result<()> { + init(); + ensure_global_cleanup_registered(); + + let target = get_global_test_target_with_opt(OptimizationLevel::Debug).await?; + let script_content = r#" +trace process_evicted_page { + if (page.value == 0) { + print "SLEEPABLE_PAGE_FAULT_OK"; + } +} +"#; + + let (default_exit_code, default_stdout, default_stderr) = + common::runner::GhostscopeRunner::new() + .with_script(script_content) + .attach_to(&target) + .timeout_secs(5) + .enable_sysmon_for_target(false) + .with_log_level("info") + .run() + .await?; + + assert_eq!( + default_exit_code, 0, + "stderr={default_stderr} stdout={default_stdout}" + ); + assert!( + !default_stdout.contains("SLEEPABLE_PAGE_FAULT_OK") + && default_stdout.contains("ExprError"), + "ordinary uprobe unexpectedly read the evicted page. stdout={default_stdout} stderr={default_stderr}" + ); + assert!( + !default_stderr.contains("Probing helpers required for sleepable uprobes") + && !default_stderr.contains("Probing sleepable tail-call support"), + "default startup unexpectedly ran sleepable capability probes. stderr={default_stderr}" + ); + + let (exit_code, stdout, stderr) = common::runner::GhostscopeRunner::new() + .with_script(script_content) + .attach_to(&target) + .timeout_secs(5) + .enable_sysmon_for_target(false) + .with_cli_args(["--sleepable-uprobe"]) + .run() + .await?; + + assert_eq!(exit_code, 0, "stderr={stderr} stdout={stdout}"); + assert!( + stdout.contains("SLEEPABLE_PAGE_FAULT_OK"), + "sleepable uprobe did not fault in and read the evicted page. stdout={stdout} stderr={stderr}" + ); + assert!( + !stdout.contains("ExprError"), + "fault-capable user-memory read unexpectedly failed. stdout={stdout} stderr={stderr}" + ); + Ok(()) +} + #[tokio::test] async fn test_sleepable_uprobe_backtrace_uses_a_kernel_supported_depth() -> anyhow::Result<()> { init(); diff --git a/ghostscope-loader/src/kernel_caps.rs b/ghostscope-loader/src/kernel_caps.rs index 3e3b3cf6..51bd48e7 100644 --- a/ghostscope-loader/src/kernel_caps.rs +++ b/ghostscope-loader/src/kernel_caps.rs @@ -11,33 +11,45 @@ use aya_obj::generated::{ use std::{fmt, io, mem, sync::OnceLock}; use tracing::{error, info, warn}; -/// Global cache for complete, hardware-backed kernel capability probes. +/// Global caches for hardware-backed kernel capability probe sets. static KERNEL_CAPS: KernelCapabilityCache = KernelCapabilityCache::new(); #[derive(Debug)] struct KernelCapabilityCache { + base: OnceLock, full: OnceLock, } impl KernelCapabilityCache { const fn new() -> Self { Self { + base: OnceLock::new(), full: OnceLock::new(), } } - fn get_or_detect(&self, detect: F) -> Result + fn get_or_detect( + &self, + probe_sleepable: bool, + detect: F, + ) -> Result where F: FnOnce() -> Result, { - if let Some(capabilities) = self.full.get() { + let cache = if probe_sleepable { + &self.full + } else { + &self.base + }; + + if let Some(capabilities) = cache.get() { return Ok(*capabilities); } let detection = detect()?; if detection.cacheable { - let _ = self.full.set(detection.capabilities); - if let Some(capabilities) = self.full.get() { + let _ = cache.set(detection.capabilities); + if let Some(capabilities) = cache.get() { return Ok(*capabilities); } } else { @@ -88,23 +100,43 @@ pub struct KernelCapabilities { } impl KernelCapabilities { - /// Detect kernel capabilities for process startup, including startup-oriented logs and - /// user-facing error context. + /// Detect kernel capabilities for process startup without probing opt-in + /// sleepable-uprobe support. pub fn detect_for_startup(force_perf_event_array: bool) -> Result { - detect_for_startup_with_detectors(force_perf_event_array, Self::get, Self::get_perf_only) + Self::detect_for_startup_with_options(force_perf_event_array, false) + } + + /// Detect kernel capabilities for process startup, including opt-in + /// sleepable-uprobe support. + pub fn detect_for_startup_with_sleepable_uprobe( + force_perf_event_array: bool, + ) -> Result { + Self::detect_for_startup_with_options(force_perf_event_array, true) + } + + fn detect_for_startup_with_options( + force_perf_event_array: bool, + sleepable_uprobe: bool, + ) -> Result { + detect_for_startup_with_detectors( + force_perf_event_array, + sleepable_uprobe, + || Self::get_for_startup(sleepable_uprobe), + || detect_perf_only_capabilities(sleepable_uprobe), + ) } /// Get global kernel capabilities (detected once on first cacheable call) /// Returns an error if neither RingBuf nor PerfEventArray support can be verified. pub fn get() -> Result { - KERNEL_CAPS.get_or_detect(detect_full_capabilities) + KERNEL_CAPS.get_or_detect(true, || detect_full_capabilities(true)) } /// Detect kernel capabilities with PerfEventArray-only startup semantics. /// This intentionally bypasses the global cache because force-perf mode is a /// runtime policy override, not the kernel's complete hardware capability set. pub fn get_perf_only() -> Result { - detect_perf_only_capabilities() + detect_perf_only_capabilities(false) } /// Check if RingBuf is supported (convenience method) @@ -127,10 +159,17 @@ impl KernelCapabilities { .map(|caps| caps.supports_ns_current_pid_tgid_helper) .unwrap_or(false) } + + fn get_for_startup(probe_sleepable: bool) -> Result { + KERNEL_CAPS.get_or_detect(probe_sleepable, || { + detect_full_capabilities(probe_sleepable) + }) + } } fn detect_for_startup_with_detectors( force_perf_event_array: bool, + sleepable_uprobe: bool, detect_full: F, detect_perf_only: P, ) -> Result @@ -153,14 +192,23 @@ where })? }; - info!( - "Kernel eBPF startup summary: ringbuf_supported={} perf_event_array_supported={} helper_ns_current_pid_tgid={} sleepable_uprobe={} sleepable_tail_calls={}", - capabilities.supports_ringbuf, - capabilities.supports_perf_event_array, - capabilities.supports_ns_current_pid_tgid_helper, - capabilities.supports_sleepable_uprobe, - capabilities.supports_sleepable_tail_calls, - ); + if sleepable_uprobe { + info!( + "Kernel eBPF startup summary: ringbuf_supported={} perf_event_array_supported={} helper_ns_current_pid_tgid={} sleepable_uprobe={} sleepable_tail_calls={}", + capabilities.supports_ringbuf, + capabilities.supports_perf_event_array, + capabilities.supports_ns_current_pid_tgid_helper, + capabilities.supports_sleepable_uprobe, + capabilities.supports_sleepable_tail_calls, + ); + } else { + info!( + "Kernel eBPF startup summary: ringbuf_supported={} perf_event_array_supported={} helper_ns_current_pid_tgid={}", + capabilities.supports_ringbuf, + capabilities.supports_perf_event_array, + capabilities.supports_ns_current_pid_tgid_helper, + ); + } Ok(capabilities) } @@ -193,7 +241,9 @@ impl CapabilityProbe { } } -fn detect_full_capabilities() -> Result { +fn detect_full_capabilities( + probe_sleepable: bool, +) -> Result { let supports_ringbuf = detect_ringbuf_support(); let supports_perf_event_array = if !supports_ringbuf.supported { detect_perf_event_array_support() @@ -232,25 +282,8 @@ fn detect_full_capabilities() -> Result Result Result { +fn detect_perf_only_capabilities( + probe_sleepable: bool, +) -> Result { info!("Testing mode: Only detecting PerfEventArray support"); let supports_perf_event_array = detect_perf_event_array_support(); @@ -298,25 +333,8 @@ fn detect_perf_only_capabilities() -> Result CapabilityProbe { } } +fn detect_sleepable_capabilities(probe_sleepable: bool) -> (CapabilityProbe, CapabilityProbe) { + detect_sleepable_capabilities_with_detectors( + probe_sleepable, + detect_sleepable_uprobe_support, + detect_sleepable_tail_call_support, + ) +} + +fn detect_sleepable_capabilities_with_detectors( + probe_sleepable: bool, + detect_uprobe: U, + detect_tail_calls: T, +) -> (CapabilityProbe, CapabilityProbe) +where + U: FnOnce() -> CapabilityProbe, + T: FnOnce() -> CapabilityProbe, +{ + if !probe_sleepable { + return ( + CapabilityProbe::cacheable(false), + CapabilityProbe::cacheable(false), + ); + } + + let supports_sleepable_uprobe = detect_uprobe(); + if supports_sleepable_uprobe.supported { + info!( + "✓ Kernel supports GhostScope sleepable uprobe mode (bpf_get_current_task_btf and bpf_copy_from_user_task)" + ); + } else { + warn!( + "⚠️ Kernel does not support GhostScope sleepable uprobe mode (requires Linux 5.18+)" + ); + } + + let supports_sleepable_tail_calls = if supports_sleepable_uprobe.supported { + detect_tail_calls() + } else { + CapabilityProbe::cacheable(false) + }; + if supports_sleepable_tail_calls.supported { + info!("✓ Kernel supports sleepable uprobe tail calls"); + } + + (supports_sleepable_uprobe, supports_sleepable_tail_calls) +} + fn detect_sleepable_uprobe_support() -> CapabilityProbe { info!( "Probing helpers required for sleepable uprobes; bpf_copy_from_user_task is probed with BPF_F_SLEEPABLE..." @@ -745,6 +810,7 @@ mod tests { let forced = detect_for_startup_with_detectors( true, + false, || -> Result { panic!("full detector should not run for forced perf startup") }, @@ -756,7 +822,8 @@ mod tests { let normal = detect_for_startup_with_detectors( false, - || cache.get_or_detect(|| Ok(detection(full_caps, true))), + true, + || cache.get_or_detect(true, || Ok(detection(full_caps, true))), || -> Result { panic!("perf-only detector should not run for normal startup") }, @@ -766,7 +833,7 @@ mod tests { assert_eq!(normal, full_caps); assert_eq!( cache - .get_or_detect(|| { + .get_or_detect(true, || { panic!("full detector should not rerun after cacheable detection") }) .expect("cached full capabilities"), @@ -774,6 +841,14 @@ mod tests { ); } + #[test] + fn startup_api_preserves_one_argument_entry_point() { + let _: fn(bool) -> Result = + KernelCapabilities::detect_for_startup; + let _: fn(bool) -> Result = + KernelCapabilities::detect_for_startup_with_sleepable_uprobe; + } + #[test] fn uncacheable_full_probe_result_is_not_cached() { let cache = KernelCapabilityCache::new(); @@ -781,18 +856,18 @@ mod tests { let cacheable_caps = caps(true, true, true, true, true); let first = cache - .get_or_detect(|| Ok(detection(uncacheable_caps, false))) + .get_or_detect(true, || Ok(detection(uncacheable_caps, false))) .expect("uncacheable startup result"); assert_eq!(first, uncacheable_caps); let second = cache - .get_or_detect(|| Ok(detection(cacheable_caps, true))) + .get_or_detect(true, || Ok(detection(cacheable_caps, true))) .expect("cacheable startup result"); assert_eq!(second, cacheable_caps); assert_eq!( cache - .get_or_detect(|| { + .get_or_detect(true, || { panic!("full detector should not rerun after cacheable detection") }) .expect("cached full capabilities"), @@ -800,6 +875,40 @@ mod tests { ); } + #[test] + fn base_capability_cache_does_not_hide_later_sleepable_detection() { + let cache = KernelCapabilityCache::new(); + let base_caps = caps(true, true, true, false, false); + let sleepable_caps = caps(true, true, true, true, true); + + assert_eq!( + cache + .get_or_detect(false, || Ok(detection(base_caps, true))) + .expect("base capabilities"), + base_caps + ); + assert_eq!( + cache + .get_or_detect(true, || Ok(detection(sleepable_caps, true))) + .expect("sleepable capabilities"), + sleepable_caps + ); + } + + #[test] + fn disabled_sleepable_mode_skips_all_sleepable_probes() { + let (uprobe, tail_calls) = detect_sleepable_capabilities_with_detectors( + false, + || panic!("sleepable uprobe probe should not run when the mode is disabled"), + || panic!("sleepable tail-call probe should not run when the mode is disabled"), + ); + + assert!(!uprobe.supported); + assert!(!tail_calls.supported); + assert!(uprobe.cacheable); + assert!(tail_calls.cacheable); + } + #[test] fn sleepable_helper_probe_classifies_verifier_results() { assert_eq!( diff --git a/ghostscope/src/main.rs b/ghostscope/src/main.rs index 58a8a8fc..a76f066e 100644 --- a/ghostscope/src/main.rs +++ b/ghostscope/src/main.rs @@ -52,9 +52,15 @@ async fn main() -> Result<()> { // Dry-run does not attach uprobes, but it still validates the same eBPF // privileges and kernel capabilities as a real run. crate::util::ensure_privileges(); - let kernel_caps = ghostscope_loader::KernelCapabilities::detect_for_startup( - user_config.ebpf_config.force_perf_event_array, - )?; + let kernel_caps = if user_config.ebpf_config.sleepable_uprobe { + ghostscope_loader::KernelCapabilities::detect_for_startup_with_sleepable_uprobe( + user_config.ebpf_config.force_perf_event_array, + )? + } else { + ghostscope_loader::KernelCapabilities::detect_for_startup( + user_config.ebpf_config.force_perf_event_array, + )? + }; let resolved_config = config::ResolvedConfig::resolve(user_config, &kernel_caps)?; // Best-effort cleanup for this process's bpffs pins on graceful shutdown and panic unwind.