From 3a7c5afb16ec334bb48b3f7eb212711ce6f8d341 Mon Sep 17 00:00:00 2001 From: daniel Date: Fri, 4 Sep 2026 23:34:30 +0100 Subject: [PATCH 1/4] fix(session): tolerate disk-full transcript writes --- docs/user/tui-and-sessions.md | 10 ++ src/lib.rs | 1 + src/session.rs | 304 ++++++++++++++++++++++++++++++---- src/storage.rs | 210 +++++++++++++++++++++++ src/tui/mod.rs | 9 + 5 files changed, 502 insertions(+), 32 deletions(-) create mode 100644 src/storage.rs diff --git a/docs/user/tui-and-sessions.md b/docs/user/tui-and-sessions.md index 93670fb5..e0cd867b 100644 --- a/docs/user/tui-and-sessions.md +++ b/docs/user/tui-and-sessions.md @@ -23,6 +23,16 @@ The catalog requires an existing directory and is workspace-filtered and newest- A session ID must be 1–128 ASCII letters, digits, `-`, or `_`. `kit prompt` uses the same durable sessions: it prints `session_id: ` after its answer, and that ID can be continued by either `kit prompt --resume ` or `kit tui --resume `. +## When transcript storage is full + +If an open session's transcript append or sync fails because the disk is full or a quota is exceeded, Kit rolls back the failed append and warns on stderr. Provided rollback succeeds, the session continues with subsequent transcript changes held only in memory. Permission errors, lost locks, and rollback failures still stop the mutation rather than bypassing storage integrity checks. + +**Memory-only changes are not durable.** They are lost when the session closes or the process exits, and do not appear in disk-based session listings or resumed sessions. Freeing disk space does not automatically flush them: the affected writer stays memory-only to avoid gaps in the saved history. Save any needed content separately before closing or switching sessions. + +The fallback shares a 64 MiB allocation budget across session writers in one process. If that budget is exhausted or a fallback allocation fails, Kit makes a best-effort terminal restore, reports the loss of unsaved records on stderr, and exits with status 1 instead of panicking. This does not protect against unrelated out-of-memory failures elsewhere in the process. + +This fallback covers transcript records after initial session setup has completed. Session creation, migration, credential storage, configuration, and output artifacts can still report disk-space errors. + ## TUI keys, prompt editing, and navigation | Key or input | Action | diff --git a/src/lib.rs b/src/lib.rs index e776e216..0d9f9de3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,6 +13,7 @@ pub mod protocols; pub mod provider; pub mod runtime; pub mod session; +mod storage; pub mod telemetry; pub mod tools; pub mod transcript; diff --git a/src/session.rs b/src/session.rs index b6b13e01..df2f50b7 100644 --- a/src/session.rs +++ b/src/session.rs @@ -85,6 +85,11 @@ struct Writer { file: File, lock: SessionLock, created: Option, + // Once degraded, the on-disk prefix is never appended to again. + memory: Option, + allow_memory: bool, + #[cfg(test)] + append_failure: Option<(usize, io::ErrorKind)>, } struct SessionLock { @@ -380,6 +385,10 @@ fn open_with_initial_timestamps_in( file, lock, created, + memory: None, + allow_memory: false, + #[cfg(test)] + append_failure: None, }; if resume && stored_workspace.is_none() { writer.replace(&transcript)?; @@ -410,6 +419,9 @@ fn open_with_initial_timestamps_in( if initial_options.commit_creation { writer.commit_creation(); } + // Opening/repairing and cloning a session must still establish durable + // history. Degradation is limited to subsequent active-session mutations. + writer.allow_memory = true; Ok(OpenSession { transcript, observer: SessionObserver(Arc::new(Mutex::new(writer))), @@ -424,9 +436,10 @@ impl SessionObserver { .commit_creation(); } - /// Durably records a complete transcript replacement produced by a mutator. + /// Records a complete transcript replacement produced by a mutator. /// Existing append records remain intact, while readers treat this record as - /// a new canonical snapshot. + /// a new canonical snapshot. After a capacity failure, replacements share + /// the writer's memory-only tail rather than claiming disk durability. pub fn replace(&self, transcript: &[Item]) -> Result<(), String> { if transcript.is_empty() { return Err("cannot persist an empty transcript replacement".into()); @@ -443,8 +456,8 @@ impl TranscriptObserver for SessionObserver { let mut writer = self.0.lock().expect("session transcript writer poisoned"); if let Err(error) = writer.append(event.item) { // The loop invokes observers before committing the item in memory. - // Refusing that mutation is safer than continuing with history that - // was not durably recorded and cannot be resumed faithfully. + // Capacity failures use bounded volatile storage. Other failures + // still refuse the mutation rather than bypassing integrity checks. panic!("session persistence failed: {error}"); } } @@ -470,16 +483,7 @@ impl Writer { .generation .checked_add(1) .ok_or_else(|| "session generation overflowed".to_string())?; - let record = Record { - schema_version: SCHEMA_VERSION, - session_id: self.session_id.clone(), - generation, - workspace_root: Some(self.workspace_root.clone()), - item: Some(item.clone()), - replacement: None, - redirect: None, - }; - self.write_record(record, generation) + self.write_record(Some(item), None, generation) } fn replace(&mut self, transcript: &[Item]) -> Result<(), String> { @@ -488,33 +492,109 @@ impl Writer { .generation .checked_add(1) .ok_or_else(|| "session generation overflowed".to_string())?; - let record = Record { + self.write_record(None, Some(transcript), generation) + } + + fn write_record( + &mut self, + item: Option<&Item>, + replacement: Option<&[Item]>, + generation: u64, + ) -> Result<(), String> { + // Borrow replacements rather than cloning the entire transcript before + // the fallible writer has a chance to enforce its budget. + #[derive(Serialize)] + struct AppendRecord<'a> { + schema_version: u32, + session_id: &'a str, + generation: u64, + workspace_root: &'a Path, + #[serde(skip_serializing_if = "Option::is_none")] + item: Option<&'a Item>, + #[serde(skip_serializing_if = "Option::is_none")] + replacement: Option<&'a [Item]>, + } + let record = AppendRecord { schema_version: SCHEMA_VERSION, - session_id: self.session_id.clone(), + session_id: &self.session_id, generation, - workspace_root: Some(self.workspace_root.clone()), - item: None, - replacement: Some(transcript.to_vec()), - redirect: None, + workspace_root: &self.workspace_root, + item, + replacement, }; - self.write_record(record, generation) - } - - fn write_record(&mut self, record: Record, generation: u64) -> Result<(), String> { - // Encode before touching the append-only file, so serialization - // failures can never leave a partial JSON record behind. - let mut encoded = serde_json::to_vec(&record) + if self.memory.is_none() { + // Encode before touching disk; only append/sync capacity errors + // permit degradation, not serialization or metadata failures. + let mut encoded = serde_json::to_vec(&record) + .map_err(|error| format!("could not encode transcript record: {error}"))?; + encoded.push(b'\n'); + let offset = self + .file + .metadata() + .map_err(|error| format!("could not inspect transcript length: {error}"))? + .len(); + #[cfg(test)] + let result = if let Some((bytes, kind)) = self.append_failure.take() { + self.file + .write_all(&encoded[..bytes.min(encoded.len())]) + .and_then(|_| Err(io::Error::from(kind))) + } else { + self.file + .write_all(&encoded) + .and_then(|_| self.file.sync_data()) + }; + #[cfg(not(test))] + let result = self + .file + .write_all(&encoded) + .and_then(|_| self.file.sync_data()); + if let Err(error) = result { + // Sync failure can leave a complete record, not just a partial + // write. Remove either before accepting the memory-only tail. + self.file.set_len(offset).map_err(|rollback| { + format!("could not roll back transcript append after {error}: {rollback}") + })?; + if !self.allow_memory || !crate::storage::is_capacity_error(&error) { + return Err(format!("could not persist transcript record: {error}")); + } + // Sync itself may be failing: truncation restores the logical + // prefix, but its crash durability cannot be promised here. + self.memory = Some(crate::storage::MemoryBuffer::default()); + let _ = io::stderr().write_all( + b"kit: session storage is full; subsequent transcript changes are memory-only and will be lost when the session closes.\n", + ); + } else { + self.generation = generation; + return Ok(()); + } + } + // Validate without retaining bytes before mutating the memory log. A + // rejected timestamp/serialization must not leave a partial record if + // a caller handles the error and attempts another replacement. + serde_json::to_writer(io::sink(), &record) .map_err(|error| format!("could not encode transcript record: {error}"))?; - encoded.push(b'\n'); - self.file - .write_all(&encoded) - .and_then(|_| self.file.sync_data()) - .map_err(|error| format!("could not persist transcript record: {error}"))?; + let mut memory = self + .memory + .as_mut() + .expect("memory fallback initialized") + .writer_or_exit(); + // Handle allocation refusal inside the writer, before serde_json can + // allocate a boxed I/O error while the allocator may be exhausted. + serde_json::to_writer(&mut memory, &record) + .map_err(|error| format!("could not encode transcript record: {error}"))?; + if memory.write_all(b"\n").is_err() { + crate::storage::exit_exhausted(); + } self.generation = generation; Ok(()) } fn ensure_lock(&mut self) -> Result<(), String> { + if self.memory.is_some() { + // Never reconstruct disk history from a prefix missing the memory + // tail, nor fabricate authority if the degraded writer loses its lock. + return self.lock.check().map_err(|error| error.to_string()); + } match self.lock.check() { Ok(()) => { if self.path.try_exists().map_err(|error| { @@ -1900,6 +1980,166 @@ mod tests { use agentkit_core::{ItemKind, MetadataMap, Part, ReasoningPart}; use serde_json::json; + #[test] + fn capacity_append_fallback_rolls_back_and_preserves_generations() { + // A short write and a sync failure (all bytes written) must both leave + // exactly the original disk prefix, including for quota exhaustion. + for kind in [io::ErrorKind::StorageFull, io::ErrorKind::QuotaExceeded] { + for bytes in [7, usize::MAX] { + let root = tempfile::tempdir().unwrap(); + let opened = open( + root.path(), + "fallback", + false, + false, + vec![Item::text(ItemKind::System, "system")], + ) + .unwrap(); + let mut writer = opened.observer.0.lock().unwrap(); + let prefix = fs::read(&writer.path).unwrap(); + let item = Item::text(ItemKind::User, "memory").with_created_at(Timestamp(123)); + writer.append_failure = Some((bytes, kind)); + writer.append(&item).unwrap(); + assert_eq!(writer.generation, 2); + writer.replace(std::slice::from_ref(&item)).unwrap(); + writer.append(&item).unwrap(); + assert_eq!(writer.generation, 4); + assert_eq!(fs::read(&writer.path).unwrap(), prefix); + let records: Vec = writer + .memory + .as_ref() + .unwrap() + .as_slice() + .split(|byte| *byte == b'\n') + .filter(|line| !line.is_empty()) + .map(|line| serde_json::from_slice(line).unwrap()) + .collect(); + assert_eq!( + records + .iter() + .map(|record| record.generation) + .collect::>(), + vec![2, 3, 4] + ); + assert!(records[0].item.is_some()); + assert_eq!(records[1].replacement.as_ref().unwrap().len(), 1); + assert!(records[1].item.is_none()); + assert!(records[2].item.is_some()); + } + } + } + + #[test] + fn observer_accepts_capacity_failure_without_panicking() { + let root = tempfile::tempdir().unwrap(); + let opened = open( + root.path(), + "observer", + false, + false, + vec![Item::text(ItemKind::System, "system")], + ) + .unwrap(); + opened.observer.0.lock().unwrap().append_failure = Some((7, io::ErrorKind::StorageFull)); + opened.observer.on_transcript_event(TranscriptEvent { + session_id: &agentkit_core::SessionId::new("observer"), + item: &Item::text(ItemKind::User, "still running").with_created_at(Timestamp(123)), + }); + let writer = opened.observer.0.lock().unwrap(); + assert_eq!(writer.generation, 2); + assert!(writer.memory.is_some()); + assert_eq!(read_records(&writer.path, "observer").unwrap().1, 1); + } + + #[test] + fn non_capacity_append_errors_never_enable_memory_fallback() { + for kind in [io::ErrorKind::PermissionDenied, io::ErrorKind::Other] { + for bytes in [7, usize::MAX] { + let root = tempfile::tempdir().unwrap(); + let opened = open( + root.path(), + "failure", + false, + false, + vec![Item::text(ItemKind::System, "system")], + ) + .unwrap(); + let mut writer = opened.observer.0.lock().unwrap(); + let prefix = fs::read(&writer.path).unwrap(); + writer.append_failure = Some((bytes, kind)); + assert!( + writer + .append( + &Item::text(ItemKind::User, "refused").with_created_at(Timestamp(123)) + ) + .is_err() + ); + assert!(writer.memory.is_none()); + assert_eq!(writer.generation, 1); + assert_eq!(fs::read(&writer.path).unwrap(), prefix); + } + } + } + + #[test] + fn capacity_fallback_does_not_relax_startup_or_rollback_failures() { + let root = tempfile::tempdir().unwrap(); + let opened = open( + root.path(), + "startup", + false, + false, + vec![Item::text(ItemKind::System, "system")], + ) + .unwrap(); + let mut writer = opened.observer.0.lock().unwrap(); + let prefix = fs::read(&writer.path).unwrap(); + let item = Item::text(ItemKind::User, "refused").with_created_at(Timestamp(123)); + writer.allow_memory = false; + writer.append_failure = Some((7, io::ErrorKind::StorageFull)); + assert!(writer.append(&item).is_err()); + assert!(writer.memory.is_none()); + assert_eq!(fs::read(&writer.path).unwrap(), prefix); + writer.allow_memory = true; + // A read-only handle deterministically rejects truncation. Inject the + // capacity error before writing to exercise failed rollback itself. + writer.file = File::open(&writer.path).unwrap(); + writer.append_failure = Some((0, io::ErrorKind::StorageFull)); + assert!(writer.append(&item).unwrap_err().contains("roll back")); + assert!(writer.memory.is_none()); + assert_eq!(writer.generation, 1); + } + + #[test] + fn memory_fallback_preserves_validation_and_lock_checks() { + let root = tempfile::tempdir().unwrap(); + let opened = open( + root.path(), + "locks", + false, + false, + vec![Item::text(ItemKind::System, "system")], + ) + .unwrap(); + let mut writer = opened.observer.0.lock().unwrap(); + let item = Item::text(ItemKind::User, "memory").with_created_at(Timestamp(123)); + writer.append_failure = Some((7, io::ErrorKind::StorageFull)); + writer.append(&item).unwrap(); + let memory_len = writer.memory.as_ref().unwrap().as_slice().len(); + assert!( + writer + .append(&Item::text(ItemKind::User, "no timestamp")) + .is_err() + ); + // Alter the expected token instead of unlinking an OS-locked file, + // which Windows correctly forbids. + writer.lock.token.push_str("-no-longer-owner"); + assert!(writer.append(&item).is_err()); + assert!(writer.replace(std::slice::from_ref(&item)).is_err()); + assert_eq!(writer.generation, 2); + assert_eq!(writer.memory.as_ref().unwrap().as_slice().len(), memory_len); + } + fn session_directory(root: &Path) -> PathBuf { root.join("sessions") } diff --git a/src/storage.rs b/src/storage.rs new file mode 100644 index 00000000..97f8bdf2 --- /dev/null +++ b/src/storage.rs @@ -0,0 +1,210 @@ +//! Bounded volatile storage for internal persistence failures. +//! +//! Capacity errors are distinct from permission, locking and integrity errors: +//! only the former may relax durability. This is not a virtual filesystem and +//! must not be used to claim a user-requested file edit succeeded. + +use std::{ + io::{self, Write}, + sync::atomic::{AtomicUsize, Ordering}, +}; + +const MAX_FALLBACK_BYTES: usize = 64 * 1024 * 1024; +static RESERVED_BYTES: Budget = Budget(AtomicUsize::new(0)); + +pub(crate) fn is_capacity_error(error: &io::Error) -> bool { + matches!( + error.kind(), + io::ErrorKind::StorageFull | io::ErrorKind::QuotaExceeded + ) +} + +struct Budget(AtomicUsize); + +impl Budget { + fn reserve(&self, additional: usize, limit: usize) -> io::Result<()> { + self.0 + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |used| { + used.checked_add(additional).filter(|next| *next <= limit) + }) + .map(|_| ()) + .map_err(|_| io::Error::from(io::ErrorKind::OutOfMemory)) + } + + fn release(&self, bytes: usize) { + self.0.fetch_sub(bytes, Ordering::Relaxed); + } +} + +/// An append-only buffer sharing a process-wide allocation budget. +/// +/// Growth is fallible, including during serde serialization through `Write`. +/// Dropping a buffer releases its reservation. No disk retry occurs implicitly: +/// callers must never append beyond a missing durable record. +#[derive(Default)] +pub(crate) struct MemoryBuffer { + bytes: Vec, + reserved: usize, +} + +impl MemoryBuffer { + /// Exit before a serializer can heap-allocate an error wrapper on OOM. + pub(crate) fn writer_or_exit(&mut self) -> impl Write + '_ { + ExitOnFailure(self) + } + + #[cfg(test)] + pub(crate) fn as_slice(&self) -> &[u8] { + &self.bytes + } + + fn reserve(&mut self, needed: usize) -> io::Result<()> { + if needed > self.reserved { + // Geometric growth avoids reallocating for every serializer token. + let target = needed + .checked_next_power_of_two() + .unwrap_or(needed) + .min(MAX_FALLBACK_BYTES) + .max(needed); + let additional = target - self.reserved; + RESERVED_BYTES.reserve(additional, MAX_FALLBACK_BYTES)?; + if self + .bytes + .try_reserve_exact(target - self.bytes.len()) + .is_err() + { + RESERVED_BYTES.release(additional); + return Err(io::ErrorKind::OutOfMemory.into()); + } + self.reserved = target; + } + Ok(()) + } +} + +impl Write for MemoryBuffer { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let needed = self + .bytes + .len() + .checked_add(bytes.len()) + .ok_or(io::ErrorKind::OutOfMemory)?; + self.reserve(needed)?; + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +struct ExitOnFailure<'a>(&'a mut MemoryBuffer); + +impl Write for ExitOnFailure<'_> { + fn write(&mut self, bytes: &[u8]) -> io::Result { + match self.0.write(bytes) { + Ok(written) => Ok(written), + Err(_) => exit_exhausted(), + } + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl Drop for MemoryBuffer { + fn drop(&mut self) { + // Free the allocation before another thread can reuse its budget. + drop(std::mem::take(&mut self.bytes)); + RESERVED_BYTES.release(self.reserved); + } +} + +/// The observer API cannot return a persistence error to its caller. Stop +/// without unwinding or allocating an error report when volatile storage fills. +/// This cannot recover arbitrary allocator aborts elsewhere in the process. +pub(crate) fn exit_exhausted() -> ! { + crate::tui::restore_after_storage_failure(); + let _ = io::stderr().write_all( + b"kit: disk persistence failed and the memory fallback is exhausted; exiting. Unsaved session records will be lost.\n", + ); + std::process::exit(1) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_capacity_errors_allow_fallback() { + for kind in [io::ErrorKind::StorageFull, io::ErrorKind::QuotaExceeded] { + assert!(is_capacity_error(&kind.into())); + } + for kind in [ + io::ErrorKind::PermissionDenied, + io::ErrorKind::NotFound, + io::ErrorKind::WriteZero, + io::ErrorKind::Other, + ] { + assert!(!is_capacity_error(&kind.into())); + } + } + + #[test] + fn budget_rejects_overflow_and_releases_reservations() { + let budget = Budget(AtomicUsize::new(0)); + budget.reserve(8, 10).unwrap(); + assert!(budget.reserve(3, 10).is_err()); + assert!(budget.reserve(usize::MAX, usize::MAX).is_err()); + assert_eq!(budget.0.load(Ordering::Relaxed), 8); + budget.release(8); + budget.reserve(10, 10).unwrap(); + } + + #[test] + fn buffer_writes_and_failed_growth_preserves_content() { + let mut buffer = MemoryBuffer::default(); + buffer.write_all(b"hello").unwrap(); + buffer.write_all(b" world").unwrap(); + assert_eq!(buffer.as_slice(), b"hello world"); + // Exercise a budget refusal without allocating a huge input or relying + // on other concurrently running tests' reservations. + let reserved = buffer.reserved; + assert_eq!( + buffer.reserve(MAX_FALLBACK_BYTES + 1).unwrap_err().kind(), + io::ErrorKind::OutOfMemory + ); + assert_eq!(buffer.reserved, reserved); + assert_eq!(buffer.as_slice(), b"hello world"); + } + + #[test] + fn exhaustion_exits_without_a_panic() { + const CHILD_FLAG: &str = "KIT_TEST_STORAGE_EXHAUSTION_CHILD"; + if std::env::var_os(CHILD_FLAG).is_some() { + // Reserve the budget without actually filling memory. Exercise the + // serialization adapter's exit, not just the exit helper itself. + RESERVED_BYTES + .reserve(MAX_FALLBACK_BYTES, MAX_FALLBACK_BYTES) + .unwrap(); + let mut buffer = MemoryBuffer::default(); + let _ = serde_json::to_writer(buffer.writer_or_exit(), &"no budget left"); + unreachable!("the exhausted writer must exit"); + } + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "storage::tests::exhaustion_exits_without_a_panic", + "--nocapture", + ]) + .env(CHILD_FLAG, "1") + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("memory fallback is exhausted"), "{stderr}"); + assert!(!stderr.contains("panicked"), "{stderr}"); + } +} diff --git a/src/tui/mod.rs b/src/tui/mod.rs index 4acc3b2b..2318c222 100644 --- a/src/tui/mod.rs +++ b/src/tui/mod.rs @@ -2050,6 +2050,15 @@ async fn bounded_graceful_close( } } +/// Restore terminal state before the allocation-free storage failure exit. +pub(crate) fn restore_after_storage_failure() { + if crossterm::terminal::is_raw_mode_enabled().unwrap_or(false) { + restore_modes(); + let _ = execute!(std::io::stdout(), crossterm::cursor::Show); + let _ = ratatui::try_restore(); + } +} + fn leave(terminal: &mut DefaultTerminal) { restore_modes(); let _ = terminal.show_cursor(); From 9867ad7c5dfe2e77ec73e01a9f523115a3c0b3a2 Mon Sep 17 00:00:00 2001 From: daniel Date: Sat, 5 Sep 2026 00:52:39 +0100 Subject: [PATCH 2/4] fix(storage): share bounded fallback and recover internal writes --- docs/user/compose-and-local-tools.md | 13 + docs/user/tui-and-sessions.md | 12 +- src/artifacts.rs | 62 +- src/compose_output.rs | 43 +- src/credentials.rs | 376 +++-- src/fatal.rs | 39 +- src/file_search.rs | 7 +- src/lib.rs | 11 +- src/main.rs | 35 +- src/plugins.rs | 260 +-- src/protocols/acp.rs | 18 +- src/protocols/http.rs | 15 +- src/provider/openai_auth.rs | 156 +- src/resilient_fs/backend.rs | 1256 ++++++++++++++ src/resilient_fs/mod.rs | 2304 ++++++++++++++++++++++++++ src/resilient_fs/tests.rs | 1037 ++++++++++++ src/runtime.rs | 138 +- src/session.rs | 795 +++------ src/storage.rs | 210 --- src/storage_runtime.rs | 145 ++ src/tools/artifact.rs | 234 +++ src/tools/mcp.rs | 37 +- src/tools/mod.rs | 2 + src/tui/mod.rs | 84 +- tests/resilient_exit.rs | 64 + tests/resilient_session.rs | 176 ++ tests/resilient_shutdown.rs | 33 + tests/support/capacity.rs | 143 ++ 28 files changed, 6454 insertions(+), 1251 deletions(-) create mode 100644 src/resilient_fs/backend.rs create mode 100644 src/resilient_fs/mod.rs create mode 100644 src/resilient_fs/tests.rs delete mode 100644 src/storage.rs create mode 100644 src/storage_runtime.rs create mode 100644 src/tools/artifact.rs create mode 100644 tests/resilient_exit.rs create mode 100644 tests/resilient_session.rs create mode 100644 tests/resilient_shutdown.rs create mode 100644 tests/support/capacity.rs diff --git a/docs/user/compose-and-local-tools.md b/docs/user/compose-and-local-tools.md index 0ec91d69..c68b9a1e 100644 --- a/docs/user/compose-and-local-tools.md +++ b/docs/user/compose-and-local-tools.md @@ -84,6 +84,19 @@ Only the final compose return value crosses the model-context boundary. When its Kit's working directory is project context, not an operating-system security boundary. A shell command can use absolute paths, `..`, the network, and any credentials or files allowed to the Kit process. Quote untrusted values, inspect destructive commands before running them, and avoid putting secrets into command text or returned output. There is no automatic rollback for shell side effects. +## Read spilled output with `artifact` + +Large compose results include an `artifact` path. Read it through `artifact`, which sees both persisted files and output temporarily held in Kit's memory filesystem. Shell commands only see real disk files. + +```text +chunk = artifact({ path: output.artifact, offset: 0, limit: 1024 }) +return chunk +``` + +The result contains `content`, `next_offset`, `total_bytes`, and `eof`. Continue from `next_offset` to read another chunk. Reads preserve UTF-8 character boundaries and are limited to 1,024 bytes per call. Only artifacts in the calling session's namespace are accepted; traversal and symlinks are rejected. + +An artifact-storage error does not turn an already-completed tool into a failed tool call. Kit returns a bounded preview with `artifact_error` when output cannot be retained. Do not repeat a side-effecting tool merely to obtain its output again. + ## Make exact file changes with `edit` `edit` operates on one file path with `op: "add"`, `"edit"`, or `"delete"`. Relative paths are resolved from Kit's working directory. Absolute paths, `..`, and paths through symlinks are accepted, so `edit` can change files outside the root when the Kit process has permission. Paths must be non-empty. diff --git a/docs/user/tui-and-sessions.md b/docs/user/tui-and-sessions.md index e0cd867b..80ee162f 100644 --- a/docs/user/tui-and-sessions.md +++ b/docs/user/tui-and-sessions.md @@ -23,15 +23,17 @@ The catalog requires an existing directory and is workspace-filtered and newest- A session ID must be 1–128 ASCII letters, digits, `-`, or `_`. `kit prompt` uses the same durable sessions: it prints `session_id: ` after its answer, and that ID can be continued by either `kit prompt --resume ` or `kit tui --resume `. -## When transcript storage is full +## Recovering from full storage -If an open session's transcript append or sync fails because the disk is full or a quota is exceeded, Kit rolls back the failed append and warns on stderr. Provided rollback succeeds, the session continues with subsequent transcript changes held only in memory. Permission errors, lost locks, and rollback failures still stop the mutation rather than bypassing storage integrity checks. +Kit routes its internal persistence through a shared filesystem service. If a write fails because storage is full or a quota is exceeded, the service retains the pending change in a bounded memory overlay. Internal reads and session listings use the same view, so finishing a turn or closing a session handle does not discard accepted changes. An existing session can be reopened in the same running process while persistence is pending. -**Memory-only changes are not durable.** They are lost when the session closes or the process exits, and do not appear in disk-based session listings or resumed sessions. Freeing disk space does not automatically flush them: the affected writer stays memory-only to avoid gaps in the saved history. Save any needed content separately before closing or switching sessions. +Recovery is automatic: internal operations retry pending work, and a process-owned worker retries with bounded backoff while Kit is idle. Once storage recovers, pending changes are persisted in order before newer changes can bypass them. Freeing space through a normal tool call can therefore restore persistence without restarting the session. Kit reports degradation and recovery on stderr. -The fallback shares a 64 MiB allocation budget across session writers in one process. If that budget is exhausted or a fallback allocation fails, Kit makes a best-effort terminal restore, reports the loss of unsaved records on stderr, and exits with status 1 instead of panicking. This does not protect against unrelated out-of-memory failures elsewhere in the process. +**Pending memory state does not survive process termination.** A different process cannot read it. Keep Kit running until recovery completes if you need those changes to be durable. Before normal exit, Kit makes one final recovery pass; if accepted changes remain unpersisted, it warns and exits unsuccessfully. The default service bounds retained data to 64 MiB and pending operations to 4,096; exhaustion requests cancellation and orderly shutdown rather than silently evicting accepted data. If a fallible fallback allocation itself fails, Kit instead makes a best-effort terminal restoration and exits immediately without allocating an error message. -This fallback covers transcript records after initial session setup has completed. Session creation, migration, credential storage, configuration, and output artifacts can still report disk-space errors. +Explicit `edit` and shell operations still use the real filesystem and report real failures to the model. The fallback does not pretend those operations succeeded. Native consumers, such as plugin subprocesses, require their inputs to be materialized on disk, and interprocess locks still require real ownership. Unsafe paths, lost ownership, and non-capacity errors are not permission to overwrite another process's data. + +Use the `artifact` tool to read spilled output, including memory-only artifacts; a shell cannot see the memory overlay. ## TUI keys, prompt editing, and navigation diff --git a/src/artifacts.rs b/src/artifacts.rs index 7210b4e0..fe6250ac 100644 --- a/src/artifacts.rs +++ b/src/artifacts.rs @@ -1,27 +1,34 @@ -use std::path::{Path, PathBuf}; +use std::{ + io::{self, Write as _}, + path::{Path, PathBuf}, +}; -use tokio::fs::File; +use crate::resilient_fs as fs; -pub(crate) fn directory(root: &Path, session_id: &str, call_id: &str) -> PathBuf { - let root = std::env::var_os("HOME") +pub(crate) fn base(root: &Path) -> PathBuf { + std::env::var_os("HOME") .filter(|home| !home.is_empty()) .map(PathBuf::from) .map_or_else( || root.join(".kit/artifacts"), |home| home.join(".kit/artifacts"), - ); - root.join(safe_component(session_id)) - .join(safe_component(call_id)) + ) +} + +pub(crate) fn session_directory(base: &Path, session_id: &str) -> PathBuf { + base.join(safe_component(session_id)) } -pub(crate) async fn create(path: &Path) -> std::io::Result<(File, PathBuf)> { +pub(crate) fn directory(root: &Path, session_id: &str, call_id: &str) -> PathBuf { + session_directory(&base(root), session_id).join(safe_component(call_id)) +} + +/// Retain the complete artifact through the same filesystem as transcripts. +/// Call from a blocking task; a returned path is readable through ArtifactTool +/// even when the backing bytes have not yet reached disk. +pub(crate) fn write(path: &Path, bytes: &[u8]) -> io::Result { let parent = path.parent().unwrap_or_else(|| Path::new(".")); - tokio::fs::create_dir_all(parent).await?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt as _; - tokio::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700)).await?; - } + fs::create_private_dir_all(parent)?; let stem = path .file_stem() .and_then(|value| value.to_str()) @@ -36,19 +43,26 @@ pub(crate) async fn create(path: &Path) -> std::io::Result<(File, PathBuf)> { } else { path.with_file_name(format!("{stem}-{attempt}.{extension}")) }; - let mut options = tokio::fs::OpenOptions::new(); - options.write(true).create_new(true); - #[cfg(unix)] - options.mode(0o600); - match options.open(&candidate).await { - Ok(file) => return Ok((file, candidate)), - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + match fs::OpenOptions::new() + .write(true) + .create_new(true) + .private(true) + .open(&candidate) + { + Ok(mut file) => { + if let Err(error) = file.write_all(bytes).and_then(|()| file.sync_all()) { + let _ = fs::remove_file(&candidate); + return Err(error); + } + return Ok(candidate); + } + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, Err(error) => return Err(error), } } - Err(std::io::Error::new( - std::io::ErrorKind::AlreadyExists, - format!("no available artifact filename under {}", parent.display()), + Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "no available artifact filename", )) } diff --git a/src/compose_output.rs b/src/compose_output.rs index c3656c92..cad8833f 100644 --- a/src/compose_output.rs +++ b/src/compose_output.rs @@ -1,9 +1,8 @@ -use std::path::Path; +use std::{path::Path, sync::Arc}; use agentkit_core::ToolOutput; use agentkit_tools_core::ToolError; use serde_json::json; -use tokio::io::AsyncWriteExt as _; const MAX_MODEL_OUTPUT_BYTES: usize = 8 * 1024; @@ -24,22 +23,29 @@ pub(crate) async fn guard( return Ok(output); } - let (mut artifact, path) = - crate::artifacts::create(&artifact_directory.join("compose-output.json")) - .await - .map_err(|error| ToolError::Internal(error.to_string()))?; - artifact - .write_all(body.as_bytes()) - .await - .map_err(|error| ToolError::Internal(error.to_string()))?; - artifact - .flush() - .await - .map_err(|error| ToolError::Internal(error.to_string()))?; - - let marker = - format!("\n...[compose output spilled: {original_bytes} bytes; see artifact field]...\n"); - let artifact = path.display().to_string(); + let body = Arc::new(body); + let artifact_body = Arc::clone(&body); + let path = artifact_directory.join("compose-output.json"); + let stored = tokio::task::spawn_blocking(move || { + crate::artifacts::write(&path, artifact_body.as_bytes()) + }) + .await + .map_err(|error| ToolError::Internal(error.to_string()))?; + let (artifact, artifact_error) = match stored { + Ok(path) => (Some(path.display().to_string()), None), + Err(error) => (None, Some(prefix(&error.to_string(), 256).to_owned())), + }; + // Artifact storage must not turn an already-executed tool into a failed + // tool call: retrying that call could duplicate its side effects. + let marker = if artifact.is_some() { + format!( + "\n...[compose output spilled: {original_bytes} bytes; read with artifact(path)]...\n" + ) + } else { + format!( + "\n...[tool completed; output truncated: {original_bytes} bytes; artifact storage failed]...\n" + ) + }; let mut preview_budget = MAX_MODEL_OUTPUT_BYTES; loop { let preview = preview(&body, &marker, preview_budget); @@ -47,6 +53,7 @@ pub(crate) async fn guard( "preview": preview, "artifact": artifact, "original_bytes": original_bytes, + "artifact_error": artifact_error, }); let replacement_bytes = serde_json::to_vec(&replacement) .map_err(|error| ToolError::Internal(error.to_string()))? diff --git a/src/credentials.rs b/src/credentials.rs index 8dffa5fd..33efe168 100644 --- a/src/credentials.rs +++ b/src/credentials.rs @@ -1,12 +1,12 @@ use std::{ collections::HashMap, - fs::{self, OpenOptions}, - io::{ErrorKind, Write}, + io::ErrorKind, path::{Path, PathBuf}, - sync::{Arc, LazyLock, Mutex}, + sync::{Arc, LazyLock, Mutex, Weak}, time::{Duration, Instant}, }; +use crate::resilient_fs as fs; use keyring::{Entry, Error as KeyringError}; use zeroize::Zeroizing; @@ -19,6 +19,78 @@ static MEMORY: LazyLock>>>> = static MEMORY_REFRESH_LOCK: LazyLock>> = LazyLock::new(|| Arc::new(tokio::sync::Mutex::new(()))); +// Authority only: credential bytes, tombstones, retry, and lease retention live +// exclusively in the shared filesystem. Weak references do not extend a caller's +// refresh guard lifetime. The guarded Fs carried by each mutation retains its lease. +static FILESYSTEM_SCOPES: LazyLock>>> = + LazyLock::new(|| Mutex::new(Vec::new())); + +pub(crate) struct CredentialFilesystemScope { + path: PathBuf, + filesystem: fs::Fs, +} + +impl CredentialFilesystemScope { + pub(crate) fn register( + lease: &fs::Lease, + path: &Path, + ) -> Result, CredentialStoreError> { + let scope = Arc::new(Self { + // The final credential file may not exist yet (first login). + path: match (path.parent(), path.file_name()) { + (Some(parent), Some(name)) => { + fs::canonicalize(parent).map(|parent| parent.join(name)) + } + _ => fs::canonicalize(path), + } + .map_err(|value| context("could not resolve credential scope", value))?, + filesystem: fs::global() + .guarded(lease) + .map_err(|value| context("could not guard credential scope", value))?, + }); + let mut scopes = FILESYSTEM_SCOPES + .lock() + .map_err(|_| error("credential lease registry is poisoned"))?; + scopes.retain(|scope| scope.strong_count() != 0); + scopes.push(Arc::downgrade(&scope)); + Ok(scope) + } +} + +fn mutation_filesystem(path: &Path) -> Result { + let directory = path + .parent() + .ok_or_else(|| error("credential path has no directory"))?; + let normalized = fs::canonicalize(directory) + .map_err(|value| context("could not resolve credential directory", value))? + .join( + path.file_name() + .ok_or_else(|| error("credential path has no filename"))?, + ); + { + let scopes = FILESYSTEM_SCOPES + .lock() + .map_err(|_| error("credential lease registry is poisoned"))?; + if let Some(scope) = scopes + .iter() + .filter_map(Weak::upgrade) + .filter(|scope| normalized.starts_with(&scope.path)) + .max_by_key(|scope| scope.path.components().count()) + { + return Ok(scope.filesystem.clone()); + } + } + // Standalone login/delete callers also need real authority before a write can + // be queued. Never replace an unavailable cross-process lock with a mutex. + let guard = acquire_refresh_lock(&directory.join(".refresh.lock"))?; + Ok(guard + ._scope + .as_ref() + .expect("filesystem refresh scope") + .filesystem + .clone()) +} + #[derive(Clone, Debug, Default)] pub enum CredentialStorage { #[default] @@ -63,6 +135,7 @@ impl CredentialStorage { Self::Memory => { return Ok(CredentialRefreshLock { _file: None, + _scope: None, _memory: Some(Arc::clone(&MEMORY_REFRESH_LOCK).lock_owned().await), }); } @@ -126,6 +199,29 @@ impl CredentialEntry { } } + pub(crate) fn filesystem_path(&self) -> Option<&Path> { + match &self.backend { + EntryBackend::Filesystem(path) => Some(path), + EntryBackend::Memory | EntryBackend::Keychain { .. } => None, + } + } + + /// A user-facing persistence barrier, not a requirement for ongoing refresh. + pub(crate) fn require_disk(&self) -> Result<(), CredentialStoreError> { + match &self.backend { + EntryBackend::Filesystem(path) => fs::global().require_disk(path).map_err(|value| { + context( + "credential changes are retained only in this process; free disk space or quota and retry before exiting Kit", + value, + ) + }), + EntryBackend::Memory => Err(error( + "credentials are stored only in memory; select --credential-store file or keychain for durable authentication", + )), + EntryBackend::Keychain { .. } => Ok(()), + } + } + pub(crate) fn save(&self, bytes: &[u8]) -> Result<(), CredentialStoreError> { match &self.backend { EntryBackend::Memory => { @@ -159,13 +255,23 @@ impl CredentialEntry { } } -#[derive(Debug)] pub(crate) struct CredentialRefreshLock { - // Dropping either guard releases its process-wide or operating-system lock. - _file: Option, + // Pending facade mutations retain the real lease after this observer drops. + _file: Option, + _scope: Option>, _memory: Option>, } +impl std::fmt::Debug for CredentialRefreshLock { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("CredentialRefreshLock") + .field("real_lease", &self._file.is_some()) + .field("memory_backend", &self._memory.is_some()) + .finish_non_exhaustive() + } +} + #[derive(Debug)] pub(crate) struct CredentialStoreError(String); @@ -243,93 +349,38 @@ fn acquire_refresh_lock(path: &Path) -> Result break, - Err(fs::TryLockError::WouldBlock) if Instant::now() < deadline => { + // The facade owns no-follow/owner/identity validation and real OS locking. + // In particular, ENOSPC here must not grant in-process-only authority. + match fs::global().acquire_lease(path, directory, fs::LeaseMode::ExistingOrNew) { + Ok(lease) => { + return Ok(CredentialRefreshLock { + _scope: Some(CredentialFilesystemScope::register(&lease, directory)?), + _file: Some(lease), + _memory: None, + }); + } + Err(value) if value.kind() == ErrorKind::WouldBlock && Instant::now() < deadline => { std::thread::sleep(Duration::from_millis(20)); } - Err(fs::TryLockError::WouldBlock) => { + Err(value) if value.kind() == ErrorKind::WouldBlock => { return Err(error(format!( "timed out waiting for OAuth refresh lock {}", path.display() ))); } - Err(fs::TryLockError::Error(value)) => { + Err(value) => { return Err(context( - &format!("could not lock OAuth refresh {}", path.display()), + &format!( + "could not acquire real OAuth refresh lock {}", + path.display() + ), value, )); } } } - #[cfg(unix)] - { - use std::os::unix::fs::MetadataExt; - let path_metadata = fs::symlink_metadata(path).map_err(|value| { - context( - &format!("could not verify OAuth refresh lock {}", path.display()), - value, - ) - })?; - if path_metadata.file_type().is_symlink() - || path_metadata.dev() != metadata.dev() - || path_metadata.ino() != metadata.ino() - { - return Err(error(format!( - "OAuth refresh lock path changed while locking: {}", - path.display() - ))); - } - } - Ok(CredentialRefreshLock { - _file: Some(file), - _memory: None, - }) } fn namespaced_key(namespace: &str, identity: &str) -> String { @@ -432,7 +483,8 @@ fn filesystem_delete(path: &Path) -> Result { ))) } Ok(_) => { - fs::remove_file(path) + mutation_filesystem(path)? + .remove_file(path) .map_err(|value| context(&format!("could not remove {}", path.display()), value))?; Ok(true) } @@ -449,25 +501,25 @@ fn write_private_file(path: &Path, bytes: &[u8]) -> Result<(), CredentialStoreEr .parent() .ok_or_else(|| error("credential path has no directory"))?; prepare_directory(directory, true)?; - if let Ok(metadata) = fs::symlink_metadata(path) - && (metadata.file_type().is_symlink() || !metadata.is_file()) - { - return Err(error(format!( - "OAuth credential path must be a regular file: {}", - path.display() - ))); + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_file() => { + return Err(error(format!( + "OAuth credential path must be a regular file: {}", + path.display() + ))); + } + Ok(metadata) => check_private_file(path, &metadata)?, + Err(value) if value.kind() == ErrorKind::NotFound => {} + Err(value) => return Err(context("could not inspect credential file", value)), } - let mut options = OpenOptions::new(); - options.write(true).create(true).truncate(true); - set_private_mode(&mut options); - atomicwrites::AtomicFile::new(path, atomicwrites::AllowOverwrite) - .write_with_options(|file| file.write_all(bytes), options) + mutation_filesystem(path)? + .replace_private(path, bytes) .map_err(|value| context(&format!("could not replace {}", path.display()), value)) } fn prepare_directory(path: &Path, create: bool) -> Result { if create { - fs::create_dir_all(path) + fs::create_private_dir_all(path) .map_err(|value| context(&format!("could not create {}", path.display()), value))?; } let metadata = match fs::symlink_metadata(path) { @@ -489,39 +541,29 @@ fn prepare_directory(path: &Path, create: bool) -> Result Result<(), CredentialStoreError> { - use std::os::unix::fs::PermissionsExt; - fs::set_permissions(path, fs::Permissions::from_mode(0o700)) - .map_err(|value| context(&format!("could not protect {}", path.display()), value)) -} -#[cfg(not(unix))] -fn make_directory_private(_: &Path) -> Result<(), CredentialStoreError> { - Ok(()) -} #[cfg(unix)] fn check_private_file(path: &Path, metadata: &fs::Metadata) -> Result<(), CredentialStoreError> { - use std::os::unix::fs::PermissionsExt; + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + if let Some(disk) = metadata.disk_metadata() + && (disk.uid() != unsafe { libc::geteuid() } || disk.nlink() != 1) + { + return Err(error(format!( + "OAuth credential file must be owned by this user and have one link: {}", + path.display() + ))); + } if metadata.permissions().mode() & 0o077 != 0 { return Err(error(format!( "OAuth credential file is accessible by other users: {}", @@ -604,6 +646,77 @@ mod tests { assert!(acquire_refresh_lock(&path).is_err()); } + #[test] + fn memory_acceptance_does_not_claim_durable_credentials() { + let entry = CredentialStorage::Memory.entry("durability-test", "memory"); + entry.save(b"secret").unwrap(); + assert!( + entry + .require_disk() + .unwrap_err() + .to_string() + .contains("only in memory") + ); + assert_eq!(entry.load().unwrap().unwrap().as_slice(), b"secret"); + entry.delete().unwrap(); + } + + #[test] + fn standalone_filesystem_writes_and_deletions_have_durability_barriers() { + let directory = tempfile::tempdir().unwrap(); + let entry = CredentialStorage::Filesystem(directory.path().to_path_buf()) + .entry("durability-test", "file"); + entry.save(b"secret").unwrap(); + entry.require_disk().unwrap(); + assert_eq!( + std::fs::read(entry.filesystem_path().unwrap()).unwrap(), + b"secret" + ); + assert!(entry.delete().unwrap()); + entry.require_disk().unwrap(); + assert!(entry.load().unwrap().is_none()); + assert!(!entry.filesystem_path().unwrap().exists()); + } + + #[test] + fn refresh_scope_is_reused_for_guarded_mutations() { + let directory = tempfile::tempdir().unwrap(); + let entry = CredentialStorage::Filesystem(directory.path().to_path_buf()) + .entry("guard-test", "file"); + let guard = acquire_refresh_lock(&directory.path().join(".refresh.lock")).unwrap(); + // Reopening the OS lock here would deadlock against our own refresh guard. + entry.save(b"secret").unwrap(); + assert!(entry.delete().unwrap()); + entry.require_disk().unwrap(); + drop(guard); + assert!(entry.load().unwrap().is_none()); + } + + #[test] + fn native_scope_guards_a_credential_outside_the_lock_directory() { + let locks = tempfile::tempdir().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let entry = CredentialStorage::Filesystem(directory.path().to_path_buf()) + .entry("native-guard-test", "file"); + let path = entry.filesystem_path().unwrap(); + let lease = super::fs::global() + .acquire_lease( + locks.path().join("native.lock"), + path, + super::fs::LeaseMode::ExistingOrNew, + ) + .unwrap(); + let scope = super::CredentialFilesystemScope::register(&lease, path).unwrap(); + entry.save(b"secret").unwrap(); + entry.require_disk().unwrap(); + // A separate .refresh.lock would prove the native authority was not used. + assert!(!directory.path().join(".refresh.lock").exists()); + assert!(entry.delete().unwrap()); + entry.require_disk().unwrap(); + drop(scope); + drop(lease); + } + #[test] fn mcp_files_keep_the_legacy_identity_hash() { let directory = tempfile::tempdir().unwrap(); @@ -633,7 +746,17 @@ mod tests { second.save(b"two").unwrap(); assert_eq!(first.load().unwrap().unwrap().as_slice(), b"one"); assert_eq!(second.load().unwrap().unwrap().as_slice(), b"two"); - assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 2); + assert_eq!( + std::fs::read_dir(directory.path()) + .unwrap() + .filter_map(Result::ok) + .filter(|entry| entry + .path() + .extension() + .is_some_and(|value| value == "json")) + .count(), + 2 + ); } #[cfg(unix)] @@ -642,16 +765,10 @@ mod tests { use std::os::unix::fs::PermissionsExt; let directory = tempfile::tempdir().unwrap(); let storage = CredentialStorage::Filesystem(directory.path().to_path_buf()); - storage - .entry("test", "permissions") - .save(b"secret") - .unwrap(); - let file = std::fs::read_dir(directory.path()) - .unwrap() - .next() - .unwrap() - .unwrap() - .path(); + let entry = storage.entry("test", "permissions"); + entry.save(b"secret").unwrap(); + entry.require_disk().unwrap(); + let file = entry.filesystem_path().unwrap(); assert_eq!(file.metadata().unwrap().permissions().mode() & 0o777, 0o600); assert_eq!( directory.path().metadata().unwrap().permissions().mode() & 0o777, @@ -674,13 +791,8 @@ mod tests { let storage = CredentialStorage::Filesystem(real.clone()); let entry = storage.entry("test", "server"); entry.save(b"secret").unwrap(); - let path = std::fs::read_dir(real) - .unwrap() - .next() - .unwrap() - .unwrap() - .path(); - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + let path = entry.filesystem_path().unwrap(); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o644)).unwrap(); assert!(entry.load().is_err()); } } diff --git a/src/fatal.rs b/src/fatal.rs index f31acc1d..dd0e5e37 100644 --- a/src/fatal.rs +++ b/src/fatal.rs @@ -1,11 +1,10 @@ use std::{ - fs::{self, File, OpenOptions}, - io::Write as _, path::{Path, PathBuf}, sync::atomic::{AtomicU64, Ordering}, time::{Duration, SystemTime, UNIX_EPOCH}, }; +use crate::resilient_fs as fs; use agentkit_loop::LoopError; use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; use serde::{Deserialize, Serialize}; @@ -471,43 +470,15 @@ fn write_in_with_diagnostics( create_private_directory(base)?; create_private_directory(&directory)?; let path = directory.join(format!("{event_id}.json")); - let temporary = directory.join(format!(".{event_id}.tmp")); - let mut options = OpenOptions::new(); - options.write(true).create_new(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt as _; - options.mode(0o600); - } - let mut file = options - .open(&temporary) - .map_err(|error| format!("could not create fatal error log: {error}"))?; - if let Err(error) = file.write_all(&bytes).and_then(|()| file.sync_data()) { - let _ = fs::remove_file(&temporary); - return Err(format!("could not write fatal error log: {error}")); - } - fs::rename(&temporary, &path).map_err(|error| { - let _ = fs::remove_file(&temporary); - format!("could not commit fatal error log: {error}") - })?; - #[cfg(unix)] - File::open(&directory) - .and_then(|directory| directory.sync_all()) - .map_err(|error| format!("could not sync fatal error log directory: {error}"))?; + fs::replace_private(&path, &bytes) + .map_err(|error| format!("could not retain fatal error log: {error}"))?; prune(&directory); Ok(path) } fn create_private_directory(path: &Path) -> Result<(), String> { - fs::create_dir_all(path) - .map_err(|error| format!("could not create fatal error log directory: {error}"))?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt as _; - fs::set_permissions(path, fs::Permissions::from_mode(0o700)) - .map_err(|error| format!("could not secure fatal error log directory: {error}"))?; - } - Ok(()) + fs::create_private_dir_all(path) + .map_err(|error| format!("could not create fatal error log directory: {error}")) } fn prune(directory: &Path) { diff --git a/src/file_search.rs b/src/file_search.rs index 9f5e3fc8..34271ade 100644 --- a/src/file_search.rs +++ b/src/file_search.rs @@ -90,7 +90,7 @@ pub struct WorkspaceFileSearchState { impl WorkspaceFileSearch { pub fn start(root: PathBuf) -> Result { - if !root.is_dir() { + if !crate::resilient_fs::metadata(&root).is_ok_and(|metadata| metadata.is_dir()) { return Err(format!( "workspace root is not a directory: {}", root.display() @@ -106,6 +106,11 @@ impl WorkspaceFileSearch { return Err("workspace root is not valid UTF-8".to_string()); } + // The workspace picker reads native paths and never writes Kit state. + crate::resilient_fs::global() + .require_disk(&root) + .map_err(|error| format!("workspace files are not available on disk: {error}"))?; + let is_git_repo = root .ancestors() .any(|ancestor| ancestor.join(".git").exists()); diff --git a/src/lib.rs b/src/lib.rs index 0d9f9de3..116d8639 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,13 +7,22 @@ pub mod docs; pub mod events; mod fatal; mod file_search; +#[path = "resilient_fs/mod.rs"] +mod filesystem; pub mod plugins; pub(crate) mod process_tree; pub mod protocols; pub mod provider; pub mod runtime; +/// Shared internal filesystem and process-lifetime recovery controls. +pub mod resilient_fs { + pub use crate::filesystem::*; + pub use crate::storage_runtime::{ + finish_recovery, request_shutdown, shutdown_token, start_recovery_worker, + }; +} pub mod session; -mod storage; +mod storage_runtime; pub mod telemetry; pub mod tools; pub mod transcript; diff --git a/src/main.rs b/src/main.rs index 1d29453e..25f17a06 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,12 +1,13 @@ use std::{ collections::BTreeMap, - env, fs, + env, future::Future, io::{self, Write}, path::{Path, PathBuf}, }; use clap::{Args, Parser, Subcommand, ValueEnum}; +use kit::resilient_fs as fs; use kit::tools::CredentialStorage; use serde::Deserialize; @@ -863,7 +864,9 @@ async fn supervise_serve_with_trigger( }; tokio::pin!(stdio); tokio::pin!(termination); + let shutdown = fs::shutdown_token(); tokio::select! { + _ = shutdown.cancelled() => Exit::Signal(Ok(())), result = &mut stdio => Exit::Stdio(result), result = http.join() => Exit::Http(result), result = &mut termination => Exit::Signal(result), @@ -897,16 +900,34 @@ async fn supervise_serve_with_trigger( #[tokio::main] async fn main() -> Result<(), Box> { + kit::resilient_fs::start_recovery_worker(); + let result = run().await; + // Finish synchronously: a detached task can be terminated with the process, + // and timing out spawn_blocking would still make runtime teardown wait. + let recovery = kit::resilient_fs::finish_recovery(kit::resilient_fs::global()); + // Always attempt recovery, but preserve the original command error. The + // recovery helper separately warns if accepted data remains undurable. + result?; + recovery?; + Ok(()) +} + +async fn run() -> Result<(), Box> { let cli = Cli::parse(); if matches!(&cli.command, Command::Init) { - init_default_config()?; + tokio::task::spawn_blocking(init_default_config).await??; + if fs::global().status().pending_operations > 0 { + eprintln!( + "Config initialized in memory only; persistence is pending and will not survive process termination." + ); + } println!( "Kit {}\n\nlog in with your OpenAI, OpenRouter, or Speakeasy account, or set OPENROUTER_API_KEY to get started", env!("CARGO_PKG_VERSION") ); return Ok(()); } - let config = Config::load_default()?; + let config = tokio::task::spawn_blocking(Config::load_default).await??; if let Command::Sessions { action, root } = &cli.command { let root = config.root(root.clone()); match action { @@ -1210,6 +1231,14 @@ async fn main() -> Result<(), Box> { let credential_storage = mcp.credentials.storage(&config)?; let (_, explicit_mcp) = mcp.config_paths(&config)?; let _ = config.plugin_runtime(&root).await?; + let config_path = config.config_path.clone(); + tokio::task::spawn_blocking(move || { + if let Some(path) = config_path { + fs::global().require_disk(path)?; + } + Ok::<_, io::Error>(()) + }) + .await??; kit::tui::run_with_reasoning_effort_and_openrouter_key( &root, &model, diff --git a/src/plugins.rs b/src/plugins.rs index 2bdea96b..5d34e3ce 100644 --- a/src/plugins.rs +++ b/src/plugins.rs @@ -2,7 +2,6 @@ use std::{ collections::{BTreeMap, BTreeSet}, env, ffi::{OsStr, OsString}, - fs::{self, File, OpenOptions}, io::{self, Cursor, Read, Seek, Write}, path::{Component, Path, PathBuf}, process::{Command, Stdio}, @@ -19,9 +18,7 @@ use sha2::{Digest, Sha256}; use url::{Host, Url}; use crate::process_tree::{isolate_process_tree, terminate_process_tree_with_pid}; - -#[cfg(unix)] -use std::os::unix::fs::DirBuilderExt; +use crate::resilient_fs::{self as fs, File, OpenOptions}; const MAX_DOWNLOAD_BYTES: u64 = 64 * 1024 * 1024; const MAX_ARCHIVE_ENTRIES: usize = 10_000; @@ -346,7 +343,7 @@ impl PluginRuntime { } pub(crate) async fn stage(&self) -> Result { - let contents = match tokio::fs::read_to_string(&self.inner.config_path).await { + let contents = match fs::read_to_string(&self.inner.config_path) { Ok(contents) => contents, Err(error) if error.kind() == io::ErrorKind::NotFound => String::new(), Err(error) => { @@ -465,6 +462,7 @@ async fn resolve_with_skill_cache( .await .map_err(|error| format!("plugin resolver task failed: {error}"))??; let mut resolved = resolution.resolved; + require_skill_directories(&resolved.skill_directories)?; let registry = SkillRegistry::from_skill_dirs(resolved.skill_directories.clone()) .discover_skills() .await; @@ -538,6 +536,7 @@ async fn stage_with_skill_cache( mut resolved, source_fingerprint, } => { + require_skill_directories(&resolved.skill_directories)?; let registry = SkillRegistry::from_skill_dirs(resolved.skill_directories.clone()) .discover_skills() .await; @@ -585,7 +584,7 @@ fn resolve_blocking( None => resolve_git(url, rev.as_deref(), subdir.as_deref(), cache_root)?, }, }; - let plugin = AgentPlugin::load(&root).map_err(|error| { + let plugin = load_plugin(&root).map_err(|error| { format!( "could not load plugin {alias:?} from {}: {error}", root.display() @@ -606,15 +605,16 @@ fn resolve_blocking( data_dir.display() ) })?; - let data_dir = data_dir.canonicalize().map_err(|error| { + let data_dir = fs::canonicalize(&data_dir).map_err(|error| { format!("could not resolve data directory for plugin {alias:?}: {error}") })?; - if !data_dir.is_dir() { + if !fs::metadata(&data_dir).is_ok_and(|metadata| metadata.is_dir()) { return Err(format!( "plugin data path is not a directory: {}", data_dir.display() )); } + require_plugin_disk(&data_dir)?; resolved.mcp_plugins.push(ResolvedPluginMcp { alias: alias.clone(), manifest_name: plugin.manifest().name.clone(), @@ -639,7 +639,7 @@ fn resolve_blocking( skill_cache_root, )?; for (alias, root, expected) in loaded_generations { - let plugin = AgentPlugin::load(&root).map_err(|error| { + let plugin = load_plugin(&root).map_err(|error| { format!( "could not revalidate plugin {alias:?} from {}: {error}", root.display() @@ -836,7 +836,7 @@ fn make_tree_read_only(root: &Path) -> Result<(), String> { while index < paths.len() { let path = paths[index].clone(); index += 1; - if path.is_dir() { + if fs::metadata(&path).is_ok_and(|metadata| metadata.is_dir()) { for entry in fs::read_dir(&path).map_err(|error| { format!( "could not inspect plugin snapshot {}: {error}", @@ -936,7 +936,7 @@ fn collect_skill_inventory( inventory: &mut BTreeMap, captured_bytes: &mut u64, ) -> Result<(), String> { - let canonical = directory.canonicalize().map_err(|error| { + let canonical = fs::canonicalize(directory).map_err(|error| { format!( "could not resolve plugin skill {}: {error}", directory.display() @@ -1016,7 +1016,7 @@ fn collect_skill_inventory( directory.display() ) })?; - let canonical_after = directory.canonicalize().map_err(|error| { + let canonical_after = fs::canonicalize(directory).map_err(|error| { format!( "could not re-resolve plugin skill {}: {error}", directory.display() @@ -1042,7 +1042,7 @@ fn capture_snapshot_file( inventory: &mut BTreeMap, captured_bytes: &mut u64, ) -> Result<(), String> { - let canonical = source.canonicalize().map_err(|error| { + let canonical = fs::canonicalize(source).map_err(|error| { format!( "could not resolve plugin skill {}: {error}", source.display() @@ -1146,70 +1146,16 @@ fn capture_snapshot_file( Ok(()) } -#[cfg(unix)] fn open_snapshot_file(package_root: &Path, source: &Path) -> Result { - use std::{ - ffi::CString, - os::{ - fd::{AsRawFd, FromRawFd, OwnedFd}, - unix::ffi::OsStrExt, - }, - }; - let relative = source .strip_prefix(package_root) .map_err(|_| format!("plugin skill is outside its package: {}", source.display()))?; - let root = CString::new(package_root.as_os_str().as_bytes()) - .map_err(|_| "plugin package path contains a NUL byte".to_string())?; - // SAFETY: `root` is NUL terminated and the returned descriptor is owned. - let descriptor = unsafe { - libc::open( - root.as_ptr(), - libc::O_RDONLY | libc::O_DIRECTORY | libc::O_CLOEXEC | libc::O_NOFOLLOW, + fs::open_beneath(package_root, relative).map_err(|error| { + format!( + "could not open plugin skill {} without following links: {error}", + source.display() ) - }; - if descriptor < 0 { - return Err(format!( - "could not open plugin package {} without following links: {}", - package_root.display(), - io::Error::last_os_error() - )); - } - // SAFETY: `descriptor` was returned uniquely by `open` above. - let mut current = unsafe { OwnedFd::from_raw_fd(descriptor) }; - let components = relative.components().collect::>(); - for (index, component) in components.iter().enumerate() { - let Component::Normal(component) = component else { - return Err(format!("invalid plugin skill path: {}", source.display())); - }; - let component = CString::new(component.as_bytes()) - .map_err(|_| "plugin skill path contains a NUL byte".to_string())?; - let last = index + 1 == components.len(); - let flags = libc::O_RDONLY - | libc::O_CLOEXEC - | libc::O_NOFOLLOW - | if last { 0 } else { libc::O_DIRECTORY }; - // SAFETY: both the directory descriptor and component C string are valid. - let descriptor = unsafe { libc::openat(current.as_raw_fd(), component.as_ptr(), flags) }; - if descriptor < 0 { - return Err(format!( - "could not open plugin skill {} without following links: {}", - source.display(), - io::Error::last_os_error() - )); - } - // SAFETY: `descriptor` was returned uniquely by `openat` above. - current = unsafe { OwnedFd::from_raw_fd(descriptor) }; - } - Ok(fs::File::from(current)) -} - -#[cfg(not(unix))] -fn open_snapshot_file(_package_root: &Path, source: &Path) -> Result { - OpenOptions::new() - .read(true) - .open(source) - .map_err(|error| format!("could not open plugin skill {}: {error}", source.display())) + }) } fn same_file_state(left: &fs::Metadata, right: &fs::Metadata) -> bool { @@ -1218,28 +1164,19 @@ fn same_file_state(left: &fs::Metadata, right: &fs::Metadata) -> bool { && left.modified().ok() == right.modified().ok() } -#[cfg(unix)] -fn same_file(left: &fs::Metadata, right: &fs::Metadata) -> bool { - use std::os::unix::fs::MetadataExt; - left.dev() == right.dev() && left.ino() == right.ino() -} - -#[cfg(windows)] -fn same_file(left: &fs::Metadata, right: &fs::Metadata) -> bool { - use std::os::windows::fs::MetadataExt; - left.volume_serial_number() == right.volume_serial_number() - && left.file_index() == right.file_index() -} - -#[cfg(not(any(unix, windows)))] fn same_file(left: &fs::Metadata, right: &fs::Metadata) -> bool { - left.len() == right.len() && left.modified().ok() == right.modified().ok() + // Overlay objects have service identities, not invented physical inode numbers. + left.same_identity(right) } #[cfg(windows)] -fn symlink_kind(file_type: &fs::FileType) -> u8 { +fn symlink_kind(metadata: &fs::Metadata) -> u8 { use std::os::windows::fs::FileTypeExt; + let Some(metadata) = metadata.disk_metadata() else { + return 2; + }; + let file_type = metadata.file_type(); if file_type.is_symlink_file() { 0 } else if file_type.is_symlink_dir() { @@ -1250,7 +1187,7 @@ fn symlink_kind(file_type: &fs::FileType) -> u8 { } #[cfg(not(windows))] -fn symlink_kind(_file_type: &fs::FileType) -> u8 { +fn symlink_kind(_metadata: &fs::Metadata) -> u8 { 0 } @@ -1388,7 +1325,7 @@ fn validate_skill_snapshot( package_count: usize, expected_skills: &BTreeSet, ) -> Result<(Vec, Vec), String> { - let canonical_root = root.canonicalize().map_err(|error| { + let canonical_root = fs::canonicalize(root).map_err(|error| { format!( "could not resolve immutable plugin skill snapshot {}: {error}", root.display() @@ -1399,7 +1336,7 @@ fn validate_skill_snapshot( let mut actual_skills = BTreeSet::new(); for index in 0..package_count { let package = root.join(index.to_string()); - let plugin = AgentPlugin::load(&package).map_err(|error| { + let plugin = load_plugin(&package).map_err(|error| { format!( "could not validate immutable plugin skill snapshot {}: {error}", package.display() @@ -1466,6 +1403,28 @@ fn publish_cached_directory( } } +// These dependencies read disk directly. Do not hand them overlay-only paths. +fn require_plugin_disk(path: &Path) -> Result<(), String> { + fs::require_disk(path).map_err(|error| { + format!( + "plugin path {} is not available on disk: {error}", + path.display() + ) + }) +} + +fn load_plugin(root: &Path) -> Result { + require_plugin_disk(root)?; + AgentPlugin::load(root).map_err(|error| error.to_string()) +} + +fn require_skill_directories(directories: &[PathBuf]) -> Result<(), String> { + for directory in directories { + require_plugin_disk(directory)?; + } + Ok(()) +} + fn validate_plugin_diagnostics(alias: &str, plugin: &AgentPlugin) -> Result<(), String> { for diagnostic in plugin.diagnostics() { if matches!(diagnostic.kind, PluginDiagnosticKind::UnknownManifestField) { @@ -1508,7 +1467,7 @@ fn resolve_path(path: &Path, runtime_root: &Path) -> Result { } else { runtime_root.join(path) }; - path.canonicalize() + fs::canonicalize(&path) .map_err(|error| format!("could not resolve plugin path {}: {error}", path.display())) } @@ -1755,7 +1714,7 @@ fn hash_local_plugin_tree(root: &Path, fingerprint: &mut blake3::Hasher) -> Resu } } else if before.file_type().is_symlink() { // Hash the link itself rather than following it outside the package or into a cycle. - let before_kind = symlink_kind(&before.file_type()); + let before_kind = symlink_kind(&before); fingerprint.update(&[2, before_kind]); let target = fs::read_link(&path).map_err(|error| { format!( @@ -1782,7 +1741,7 @@ fn hash_local_plugin_tree(root: &Path, fingerprint: &mut blake3::Hasher) -> Resu ) })?; if !after.file_type().is_symlink() - || symlink_kind(&after.file_type()) != before_kind + || symlink_kind(&after) != before_kind || !same_file_state(&before, &after) || target != target_after { @@ -1872,6 +1831,7 @@ impl SystemGitRunner { request: GitRunRequest<'_>, stdout_mode: GitStdout, ) -> Result, GitFailure> { + fs::require_disk(request.cwd).map_err(|error| GitFailure::Unavailable(error.kind()))?; let mut command = Command::new("git"); command .args(request.args) @@ -2406,8 +2366,7 @@ fn resolve_git_local_revision( cache_root: &Path, runner: &dyn GitRunner, ) -> Result { - let repository = repository - .canonicalize() + let repository = fs::canonicalize(repository) .map_err(|error| format!("could not resolve test Git repository: {error}"))?; let subdir = subdir .map(|path| { @@ -2624,6 +2583,9 @@ impl GitCommandContext<'_> { stdout_limit: u64, object_store_limit: Option<&Path>, ) -> Result, String> { + require_plugin_disk(cwd)?; + require_plugin_disk(self.hooks)?; + require_plugin_disk(self.attributes)?; let args = hardened_git_args(self.protocol, self.hooks, self.attributes, args); let config = hardened_git_config(self.remote); self.runner @@ -2645,6 +2607,9 @@ impl GitCommandContext<'_> { args: &[&OsStr], destination: &Path, ) -> Result<(), String> { + require_plugin_disk(cwd)?; + require_plugin_disk(self.hooks)?; + require_plugin_disk(self.attributes)?; let args = hardened_git_args(self.protocol, self.hooks, self.attributes, args); let config = hardened_git_config(self.remote); self.runner @@ -3056,7 +3021,7 @@ fn resolve_git_source( fs::create_dir(&repository).map_err(|error| error.to_string())?; git.archive("archive", &git_dir, &archive_args, &repository)?; let candidate = select_git_package_root(&repository, subdir)?; - AgentPlugin::load(&candidate).map_err(|error| { + load_plugin(&candidate).map_err(|error| { format!( "invalid Git plugin package at {}: {error}", candidate.display() @@ -3305,20 +3270,19 @@ fn validate_git_cache_entry( return Err("Git plugin cache entry is incomplete".into()); } let root = select_git_package_root(&repository_path, subdir)?; - AgentPlugin::load(&root) - .map_err(|error| format!("invalid cached Git plugin package: {error}"))?; + load_plugin(&root).map_err(|error| format!("invalid cached Git plugin package: {error}"))?; Ok(root) } fn select_git_package_root(repository: &Path, subdir: Option<&Path>) -> Result { - let canonical_repository = repository - .canonicalize() + let canonical_repository = fs::canonicalize(repository) .map_err(|error| format!("could not resolve Git plugin cache: {error}"))?; let selected = subdir.map_or_else(|| repository.to_path_buf(), |path| repository.join(path)); - let selected = selected - .canonicalize() + let selected = fs::canonicalize(&selected) .map_err(|error| format!("could not resolve Git plugin subdir: {error}"))?; - if !selected.is_dir() || !selected.starts_with(&canonical_repository) { + if !fs::metadata(&selected).is_ok_and(|metadata| metadata.is_dir()) + || !selected.starts_with(&canonical_repository) + { return Err("plugin Git subdir escapes the repository".into()); } Ok(selected) @@ -3365,7 +3329,7 @@ fn resolve_archive( publish_cached_directory(&destination, "plugin archive", |staging| { extract_archive(&bytes, staging)?; let candidate = select_package_root(staging, subdir.as_deref())?; - AgentPlugin::load(&candidate).map_err(|error| { + load_plugin(&candidate).map_err(|error| { format!( "invalid plugin archive package at {}: {error}", candidate.display() @@ -3439,7 +3403,7 @@ fn publish_cached_file(destination: &Path, bytes: &[u8], context: &str) -> Resul .map_err(|error| format!("could not write {context} staging file: {error}"))?; let result = match fs::rename(&staging, destination) { Ok(()) => Ok(()), - Err(_) if destination.is_file() => Ok(()), + Err(_) if fs::metadata(destination).is_ok_and(|metadata| metadata.is_file()) => Ok(()), Err(error) => Err(format!("could not publish {context}: {error}")), }; let _ = fs::remove_file(staging); @@ -3740,31 +3704,34 @@ fn write_entry(reader: &mut impl Read, output: &Path, declared_size: u64) -> Res } fn select_package_root(extraction: &Path, subdir: Option<&Path>) -> Result { - let base = if extraction.join("plugin.json").is_file() { - extraction.to_path_buf() - } else { - let entries = fs::read_dir(extraction) - .map_err(|error| format!("could not inspect extracted plugin: {error}"))? - .collect::, _>>() - .map_err(|error| error.to_string())?; - if entries.len() != 1 || !entries[0].path().is_dir() { - return Err( - "plugin archive must contain plugin.json or one top-level directory".into(), - ); - } - entries[0].path() - }; + let base = + if fs::metadata(extraction.join("plugin.json")).is_ok_and(|metadata| metadata.is_file()) { + extraction.to_path_buf() + } else { + let entries = fs::read_dir(extraction) + .map_err(|error| format!("could not inspect extracted plugin: {error}"))? + .collect::, _>>() + .map_err(|error| error.to_string())?; + if entries.len() != 1 + || !fs::metadata(entries[0].path()).is_ok_and(|metadata| metadata.is_dir()) + { + return Err( + "plugin archive must contain plugin.json or one top-level directory".into(), + ); + } + entries[0].path() + }; let selected = subdir.map_or(base.clone(), |subdir| base.join(subdir)); - let canonical_base = extraction - .canonicalize() - .map_err(|error| error.to_string())?; - let selected = selected.canonicalize().map_err(|error| { + let canonical_base = fs::canonicalize(extraction).map_err(|error| error.to_string())?; + let selected = fs::canonicalize(&selected).map_err(|error| { format!( "could not resolve plugin archive package {}: {error}", selected.display() ) })?; - if !selected.is_dir() || !selected.starts_with(&canonical_base) { + if !fs::metadata(&selected).is_ok_and(|metadata| metadata.is_dir()) + || !selected.starts_with(&canonical_base) + { return Err("plugin archive subdir escapes the extracted package".into()); } Ok(selected) @@ -3776,6 +3743,7 @@ mod tests { use std::thread; use super::*; + use std::fs::{self, File}; const MANIFEST: &str = r#"{"$schema":"https://agent-plugins.org/schemas/1.0.0/plugin.schema.json","name":"test-plugin"}"#; @@ -3976,6 +3944,48 @@ mod tests { )); } + #[test] + fn snapshot_reader_preserves_file_identity_and_rejects_directories() { + let package = tempfile::tempdir().unwrap(); + let source = package.path().join("SKILL.md"); + fs::write(&source, "safe").unwrap(); + let before = super::fs::symlink_metadata(&source).unwrap(); + let mut file = open_snapshot_file(package.path(), &source).unwrap(); + assert!(same_file(&before, &file.metadata().unwrap())); + let mut contents = String::new(); + file.read_to_string(&mut contents).unwrap(); + assert_eq!(contents, "safe"); + + let replacement = package.path().join("replacement"); + fs::write(&replacement, "safe").unwrap(); + fs::remove_file(&source).unwrap(); + fs::rename(&replacement, &source).unwrap(); + assert!(!same_file( + &before, + &super::fs::symlink_metadata(&source).unwrap() + )); + let directory = package.path().join("directory"); + fs::create_dir(&directory).unwrap(); + assert!(open_snapshot_file(package.path(), &directory).is_err()); + } + + #[cfg(unix)] + #[test] + fn snapshot_reader_rejects_root_and_leaf_symlinks() { + use std::os::unix::fs::symlink; + + let directory = tempfile::tempdir().unwrap(); + let package = directory.path().join("package"); + fs::create_dir(&package).unwrap(); + fs::write(package.join("SKILL.md"), "safe").unwrap(); + let root_link = directory.path().join("root-link"); + symlink(&package, &root_link).unwrap(); + assert!(open_snapshot_file(&root_link, &root_link.join("SKILL.md")).is_err()); + let leaf_link = package.join("leaf-link"); + symlink(package.join("SKILL.md"), &leaf_link).unwrap(); + assert!(open_snapshot_file(&package, &leaf_link).is_err()); + } + #[cfg(unix)] #[test] fn intermediate_directory_symlink_mutation_is_rejected() { diff --git a/src/protocols/acp.rs b/src/protocols/acp.rs index f5258997..35a7c33c 100644 --- a/src/protocols/acp.rs +++ b/src/protocols/acp.rs @@ -1254,7 +1254,12 @@ impl Server { // the same id and any bind failure releases the selection or reservation. let session_id = agentkit_acp::SessionId::new(claim.id()); let agentkit_session_id = AgentkitSessionId::new(claim.id()); + if crate::resilient_fs::shutdown_token().is_cancelled() { + return Err(AcpRuntimeError::ClientClosed); + } let cancellation = CancellationController::new(); + let shutdown_bridge = + crate::runtime::StorageCancellationBridge::new(cancellation.clone(), None); let (client, messages) = AcpClientHandle::channel(); tokio::spawn(drain_client_messages(messages, connection.clone())); let (turn_states, turn_state_messages) = mpsc::unbounded_channel(); @@ -1347,6 +1352,7 @@ impl Server { completed, }; let actor_task = tokio::spawn(async move { + let _shutdown_bridge = shutdown_bridge; let _guard = guard; if activated.await.is_ok() { session_actor(actor).await; @@ -2156,10 +2162,14 @@ pub async fn serve_with_registry( runtime: Arc, registry: SessionRegistry, ) -> Result<(), AcpRuntimeError> { - component(runtime, registry)? - .connect_to(agent_client_protocol::Stdio::new()) - .await - .map_err(|error| AcpRuntimeError::Sdk(error.to_string())) + let component = component(runtime, registry.clone())?; + let shutdown = crate::resilient_fs::shutdown_token(); + let result = tokio::select! { + result = component.connect_to(agent_client_protocol::Stdio::new()) => result.map_err(|error| AcpRuntimeError::Sdk(error.to_string())), + _ = shutdown.cancelled() => Ok(()), + }; + registry.shutdown().await; + result } async fn serve_transport( diff --git a/src/protocols/http.rs b/src/protocols/http.rs index 14981dd9..ab05eb34 100644 --- a/src/protocols/http.rs +++ b/src/protocols/http.rs @@ -18,7 +18,7 @@ struct BearerToken(Vec); impl BearerToken { fn load(path: &Path) -> io::Result { - let mut token = std::fs::read(path).map_err(|error| { + let mut token = crate::resilient_fs::read(path).map_err(|error| { io::Error::new( error.kind(), format!( @@ -139,7 +139,14 @@ pub async fn start_with_registry( ) .into()); } - let credential = credential_file.map(BearerToken::load).transpose()?; + let credential_file = credential_file.map(Path::to_path_buf); + let credential = tokio::task::spawn_blocking(move || { + credential_file + .as_deref() + .map(BearerToken::load) + .transpose() + }) + .await??; // Bind once so an ephemeral port cannot be stolen between selection and serving. let listener = tokio::net::TcpListener::bind(&address).await?; let bound = listener.local_addr()?; @@ -151,9 +158,9 @@ pub async fn start_with_registry( crate::protocols::acp::http_router(runtime.clone(), sessions.clone()) .merge(crate::protocols::acp::v2::http_router(runtime, sessions)) }); - let stop_accepting = CancellationToken::new(); + let stop_accepting = crate::resilient_fs::shutdown_token().child_token(); let accepts_stopped = CancellationToken::new(); - let shutdown_connections = CancellationToken::new(); + let shutdown_connections = crate::resilient_fs::shutdown_token().child_token(); let task = tokio::spawn(serve_bound( listener, a2a, diff --git a/src/provider/openai_auth.rs b/src/provider/openai_auth.rs index 314345b0..bda6a713 100644 --- a/src/provider/openai_auth.rs +++ b/src/provider/openai_auth.rs @@ -1,6 +1,5 @@ use std::{ collections::{BTreeSet, HashMap}, - fs, io::{Read, Write}, net::{TcpListener, TcpStream}, path::PathBuf, @@ -9,7 +8,8 @@ use std::{ time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; -use crate::credentials::{CredentialEntry, CredentialStorage}; +use crate::credentials::{CredentialEntry, CredentialFilesystemScope, CredentialStorage}; +use crate::resilient_fs as fs; use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; use jsonwebtoken::{ Algorithm, DecodingKey, Validation, decode, decode_header, @@ -269,6 +269,12 @@ trait CredentialStore: Send + Sync { fn load(&self) -> Result, AuthError>; fn save(&self, record: &TokenRecord) -> Result<(), AuthError>; fn delete(&self) -> Result; + fn lock_scope(&self) -> Option<&std::path::Path> { + None + } + fn require_disk(&self) -> Result<(), AuthError> { + Ok(()) + } } struct BackendCredentialStore { @@ -284,6 +290,16 @@ impl BackendCredentialStore { } impl CredentialStore for BackendCredentialStore { + fn lock_scope(&self) -> Option<&std::path::Path> { + self.entry.filesystem_path() + } + + fn require_disk(&self) -> Result<(), AuthError> { + self.entry.require_disk().map_err(|value| { + AuthError::unavailable("credential_persistence_blocked", value.to_string()) + }) + } + fn load(&self) -> Result, AuthError> { let mut bytes = match self.entry.load() { Ok(Some(bytes)) => bytes, @@ -426,8 +442,9 @@ fn login( TOKEN_URL, ) .and_then(|record| { - let _lock = process_lock(deadline)?; + let _lock = process_lock_scoped(deadline, store.lock_scope())?; store.save(&record)?; + store.require_disk()?; Ok(record) }); code.zeroize(); @@ -497,13 +514,14 @@ fn logout_at( local_only: bool, revoke_url: &str, ) -> Result { - let _lock = process_lock(deadline)?; + let _lock = process_lock_scoped(deadline, store.lock_scope())?; if let Some(record) = store.load()? && !local_only { revoke_at(&record, deadline, revoke_url)?; } let removed = store.delete()?; + store.require_disk()?; if format == OutputFormat::Human { Ok(human(if removed && local_only { "WARNING: local OpenAI credentials removed without remote revocation.\n" @@ -575,7 +593,7 @@ pub(crate) fn access_token( })?; if !valid_generation(&record.generation) { let _thread = refresh_guard(deadline)?; - let _process = process_lock(deadline)?; + let _process = process_lock_scoped(deadline, store.lock_scope())?; record = store.load()?.ok_or_else(|| { AuthError::invalid( "openai_auth_required", @@ -618,7 +636,7 @@ fn refresh_locked( token_url: &str, ) -> Result { let _thread = refresh_guard(deadline)?; - let _process = process_lock(deadline)?; + let _process = process_lock_scoped(deadline, store.lock_scope())?; refresh_current(store, deadline, rejected_access_token, token_url) } @@ -1363,97 +1381,69 @@ fn revoke_at(record: &TokenRecord, deadline: Instant, revoke_url: &str) -> Resul } } -struct ProcessLock(fs::File); - -impl Drop for ProcessLock { - fn drop(&mut self) { - let _ = self.0.unlock(); - } +// No explicit unlock in Drop: queued credential mutations retain this real lease. +struct ProcessLock { + _lease: fs::Lease, + _scope: Option>, } +#[cfg(test)] fn process_lock(deadline: Instant) -> Result { + process_lock_scoped(deadline, None) +} + +fn process_lock_scoped( + deadline: Instant, + credential_scope: Option<&std::path::Path>, +) -> Result { let path = auth_lock_path()?; let parent = path.parent().expect("auth lock path has a parent"); - fs::create_dir_all(parent).map_err(|_| { - AuthError::unavailable("auth_lock_failed", "could not create the state directory") + fs::create_private_dir_all(parent).map_err(|_| { + AuthError::unavailable( + "auth_lock_failed", + "could not create the private state directory", + ) })?; - #[cfg(unix)] - { - use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; - let metadata = fs::symlink_metadata(parent).map_err(|_| { - AuthError::unavailable( - "auth_lock_failed", - "could not inspect the auth lock directory", - ) + // A filesystem store can live outside the OS lock directory. Its pending writes + // must retain this same cross-process refresh authority, not a memory mutex. + let scope = credential_scope.unwrap_or(parent); + if credential_scope.is_some() { + let directory = scope.parent().ok_or_else(|| { + AuthError::unavailable("auth_lock_failed", "credential path has no directory") })?; - if !metadata.is_dir() || metadata.uid() != unsafe { libc::geteuid() } { - return Err(AuthError::unavailable( - "auth_lock_failed", - "the auth lock directory is not owned by the current OS user", - )); - } - fs::set_permissions(parent, fs::Permissions::from_mode(0o700)).map_err(|_| { + fs::create_private_dir_all(directory).map_err(|_| { AuthError::unavailable( "auth_lock_failed", - "could not secure the auth lock directory", + "could not prepare the credential directory", ) })?; } - let mut options = fs::OpenOptions::new(); - options.read(true).write(true).create(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt as _; - options.mode(0o600).custom_flags(libc::O_NOFOLLOW); - } - #[cfg(windows)] - { - use std::os::windows::fs::OpenOptionsExt as _; - options.custom_flags(0x0020_0000); - } - let file = options.open(&path).map_err(|_| { - AuthError::unavailable("auth_lock_failed", "could not open the authentication lock") - })?; - #[cfg(unix)] - { - use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; - file.set_permissions(fs::Permissions::from_mode(0o600)) - .map_err(|_| { - AuthError::unavailable( - "auth_lock_failed", - "could not secure the authentication lock", - ) - })?; - let metadata = file.metadata().map_err(|_| { - AuthError::unavailable( - "auth_lock_failed", - "could not inspect the authentication lock", - ) - })?; - if !metadata.is_file() - || metadata.uid() != unsafe { libc::geteuid() } - || metadata.mode() & 0o777 != 0o600 - { - return Err(AuthError::unavailable( - "auth_lock_failed", - "the authentication lock is not a secure user-owned regular file", - )); - } - } - #[cfg(not(unix))] - if !file.metadata().is_ok_and(|metadata| metadata.is_file()) { - return Err(AuthError::unavailable( - "auth_lock_failed", - "the authentication lock is not a regular file", - )); - } loop { - match file.try_lock() { - Ok(()) => return Ok(ProcessLock(file)), - Err(_) if Instant::now() < deadline => std::thread::sleep(Duration::from_millis(20)), + match fs::global().acquire_lease(&path, scope, fs::LeaseMode::ExistingOrNew) { + Ok(lease) => { + let scope = credential_scope + .map(|path| CredentialFilesystemScope::register(&lease, path)) + .transpose() + .map_err(|value| { + AuthError::unavailable("auth_lock_failed", value.to_string()) + })?; + return Ok(ProcessLock { + _lease: lease, + _scope: scope, + }); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + if Instant::now() >= deadline { + return Err(AuthError::timeout( + "timed out waiting for the authentication lock", + )); + } + std::thread::sleep(Duration::from_millis(20)); + } Err(_) => { - return Err(AuthError::timeout( - "timed out waiting for the authentication lock", + return Err(AuthError::unavailable( + "auth_lock_failed", + "could not acquire a secure real authentication lock; check disk space, quota, ownership and permissions", )); } } diff --git a/src/resilient_fs/backend.rs b/src/resilient_fs/backend.rs new file mode 100644 index 00000000..4bcff97f --- /dev/null +++ b/src/resilient_fs/backend.rs @@ -0,0 +1,1256 @@ +//! Native filesystem boundary. Ownership never falls back to an in-process lock. +use std::ffi::OsString; +use std::fs::{self, File, OpenOptions, Permissions}; +use std::io::{self, Read, Seek, Write}; +use std::path::{Path, PathBuf}; + +/// Stable native file identity, scoped to a volume. Not a content fingerprint. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct FileIdentity { + pub volume: u64, + pub file: u64, +} +#[cfg(unix)] +fn metadata_identity(metadata: &fs::Metadata) -> FileIdentity { + use std::os::unix::fs::MetadataExt; + FileIdentity { + volume: metadata.dev(), + file: metadata.ino(), + } +} + +#[derive(Default, Clone, Debug)] +pub struct DiskOpenOptions { + pub read: bool, + pub write: bool, + pub append: bool, + pub truncate: bool, + pub create: bool, + pub create_new: bool, + pub private: bool, + /// Native open flags. Access, creation and truncation use the fields above. + pub custom_flags: i32, +} +#[derive(Clone, Debug)] +pub struct DiskEntry { + pub path: PathBuf, + pub file_name: OsString, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LeaseMode { + CreateNew, + ExistingOrNew, +} +#[derive(Clone, Debug)] +pub struct LeaseRequest { + pub path: PathBuf, + pub scope: PathBuf, + pub mode: LeaseMode, + pub remove_on_drop: bool, +} +pub trait Backend: Send + Sync { + fn open(&self, path: &Path, options: &DiskOpenOptions) -> io::Result>; + fn metadata(&self, path: &Path, follow: bool) -> io::Result; + fn identity(&self, path: &Path, follow: bool) -> io::Result> { + #[cfg(unix)] + { + self.metadata(path, follow) + .map(|m| Some(metadata_identity(&m))) + } + #[cfg(not(unix))] + { + let _ = (path, follow); + Ok(None) + } + } + fn read_dir(&self, path: &Path) -> io::Result>; + fn read_link(&self, path: &Path) -> io::Result; + fn canonicalize(&self, path: &Path) -> io::Result; + fn create_dir(&self, path: &Path, private: bool) -> io::Result<()>; + fn remove_file(&self, path: &Path) -> io::Result<()>; + fn remove_dir(&self, path: &Path) -> io::Result<()>; + fn rename(&self, from: &Path, to: &Path) -> io::Result<()>; + fn set_permissions(&self, path: &Path, p: Permissions) -> io::Result<()>; + fn sync_directory(&self, path: &Path) -> io::Result<()>; + fn acquire_lease(&self, request: &LeaseRequest) -> io::Result>; + fn open_beneath(&self, root: &Path, relative: &Path) -> io::Result>; +} +pub trait BackendFile: Read + Write + Seek + Send { + fn metadata(&self) -> io::Result; + fn identity(&self) -> io::Result> { + #[cfg(unix)] + { + self.metadata().map(|m| Some(metadata_identity(&m))) + } + #[cfg(not(unix))] + { + Ok(None) + } + } + fn set_len(&self, size: u64) -> io::Result<()>; + fn sync_data(&self) -> io::Result<()>; + fn sync_all(&self) -> io::Result<()>; + fn set_permissions(&self, p: Permissions) -> io::Result<()>; +} +pub trait BackendLease: Send + Sync { + fn check(&self) -> io::Result<()>; +} +impl BackendFile for File { + fn metadata(&self) -> io::Result { + File::metadata(self) + } + fn identity(&self) -> io::Result> { + native::file_identity(self).map(Some) + } + fn set_len(&self, size: u64) -> io::Result<()> { + File::set_len(self, size) + } + fn sync_data(&self) -> io::Result<()> { + File::sync_data(self) + } + fn sync_all(&self) -> io::Result<()> { + File::sync_all(self) + } + fn set_permissions(&self, p: Permissions) -> io::Result<()> { + File::set_permissions(self, p) + } +} +#[derive(Default)] +pub struct DiskBackend; +impl Backend for DiskBackend { + fn open(&self, path: &Path, o: &DiskOpenOptions) -> io::Result> { + Ok(Box::new(native::open(path, o)?)) + } + fn metadata(&self, path: &Path, follow: bool) -> io::Result { + if follow { + fs::metadata(path) + } else { + fs::symlink_metadata(path) + } + } + fn identity(&self, path: &Path, follow: bool) -> io::Result> { + native::path_identity(path, follow).map(Some) + } + fn read_dir(&self, path: &Path) -> io::Result> { + let mut entries = Vec::new(); + for entry in fs::read_dir(path)? { + let entry = entry?; + entries.try_reserve(1).map_err(|_| { + io::Error::new( + io::ErrorKind::OutOfMemory, + "directory inventory allocation failed", + ) + })?; + entries.push(DiskEntry { + path: entry.path(), + file_name: entry.file_name(), + }); + } + Ok(entries) + } + fn read_link(&self, path: &Path) -> io::Result { + fs::read_link(path) + } + fn canonicalize(&self, path: &Path) -> io::Result { + fs::canonicalize(path) + } + fn create_dir(&self, path: &Path, private: bool) -> io::Result<()> { + native::create_dir(path, private) + } + fn remove_file(&self, path: &Path) -> io::Result<()> { + native::remove(path, false) + } + fn remove_dir(&self, path: &Path) -> io::Result<()> { + native::remove(path, true) + } + fn rename(&self, from: &Path, to: &Path) -> io::Result<()> { + native::rename(from, to) + } + fn set_permissions(&self, path: &Path, p: Permissions) -> io::Result<()> { + native::set_permissions(path, p) + } + fn sync_directory(&self, path: &Path) -> io::Result<()> { + native::sync_directory(path) + } + fn acquire_lease(&self, request: &LeaseRequest) -> io::Result> { + acquire_lease(request) + } + fn open_beneath(&self, root: &Path, relative: &Path) -> io::Result> { + Ok(Box::new(native::open_beneath(root, relative)?)) + } +} +fn denied() -> io::Error { + io::Error::new( + io::ErrorKind::PermissionDenied, + "file identity, token, or private ownership changed", + ) +} +#[cfg(unix)] +mod native { + use super::*; + use std::ffi::CString; + use std::os::fd::{AsRawFd, FromRawFd}; + use std::os::unix::ffi::OsStrExt; + use std::os::unix::fs::{FileExt, MetadataExt, OpenOptionsExt}; + use std::path::Component; + fn denied() -> io::Error { + io::Error::new( + io::ErrorKind::PermissionDenied, + "lease identity, token, or private ownership changed", + ) + } + fn open_at(parent: &File, name: &std::ffi::OsStr, directory: bool) -> io::Result { + let name = CString::new(name.as_bytes()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "path contains NUL"))?; + let flags = libc::O_RDONLY + | libc::O_CLOEXEC + | libc::O_NOFOLLOW + | libc::O_NONBLOCK + | if directory { libc::O_DIRECTORY } else { 0 }; + // SAFETY: parent is live, name is NUL terminated, and no creation mode is needed. + let fd = unsafe { libc::openat(parent.as_raw_fd(), name.as_ptr(), flags) }; + if fd < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: openat returned a fresh descriptor owned by this function. + Ok(unsafe { File::from_raw_fd(fd) }) + } + pub(super) fn open_beneath(root: &Path, relative: &Path) -> io::Result { + let parts: Vec<_> = relative.components().collect(); + if parts.is_empty() || parts.iter().any(|p| !matches!(p, Component::Normal(_))) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "expected nonempty relative path without traversal", + )); + } + let mut dir = parent(&root.join(".kit-root-anchor"))?.dir; + for (index, part) in parts.iter().enumerate() { + if let Component::Normal(name) = part { + dir = open_at(&dir, name, index + 1 < parts.len())?; + } + } + if !dir.metadata()?.is_file() || dir.metadata()?.nlink() != 1 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "expected a regular file", + )); + } + Ok(dir) + } + pub(super) struct Parent { + dir: File, + name: CString, + path: PathBuf, + } + pub(super) fn parent(path: &Path) -> io::Result { + let absolute = if path.is_absolute() { + path.to_owned() + } else { + std::env::current_dir()?.join(path) + }; + let mut dir = OpenOptions::new() + .read(true) + .custom_flags(libc::O_DIRECTORY | libc::O_CLOEXEC) + .open("/")?; + let mut parts = absolute.components().peekable(); + while let Some(part) = parts.next() { + match part { + Component::RootDir | Component::CurDir => continue, + Component::Normal(name) if parts.peek().is_none() => { + return Ok(Parent { + path: absolute.clone(), + dir, + name: CString::new(name.as_bytes()).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "path contains NUL") + })?, + }); + } + Component::Normal(name) => dir = open_at(&dir, name, true)?, + _ => return Err(denied()), + } + } + Err(io::Error::new( + io::ErrorKind::InvalidInput, + "expected a file name", + )) + } + impl Parent { + pub(super) fn open(&self, o: &DiskOpenOptions) -> io::Result { + if !(o.read || o.write || o.append) + || ((o.truncate || o.create || o.create_new) && !(o.write || o.append)) + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid open options", + )); + } + let access = if o.read && (o.write || o.append) { + libc::O_RDWR + } else if o.write || o.append { + libc::O_WRONLY + } else { + libc::O_RDONLY + }; + // Never truncate before checking the opened inode. In particular a + // replay image must not change another file through a hard link. + let flags = access + | libc::O_CLOEXEC + | libc::O_NOFOLLOW + | libc::O_NONBLOCK + | (o.custom_flags + & !(libc::O_ACCMODE | libc::O_TRUNC | libc::O_CREAT | libc::O_EXCL)) + | if o.append { libc::O_APPEND } else { 0 } + | if o.create_new { + libc::O_CREAT | libc::O_EXCL + } else if o.create { + libc::O_CREAT + } else { + 0 + }; + // SAFETY: live directory descriptor and NUL-terminated name; mode supplied for creation. + let fd = unsafe { + libc::openat( + self.dir.as_raw_fd(), + self.name.as_ptr(), + flags, + if o.private { 0o600 } else { 0o666 }, + ) + }; + if fd < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: fd is newly owned. + let file = unsafe { File::from_raw_fd(fd) }; + let held = file.metadata()?; + if !held.is_file() || held.nlink() != 1 { + return Err(denied()); + } + self.check(&file)?; + if o.truncate { + file.set_len(0)?; + } + Ok(file) + } + pub(super) fn check(&self, file: &File) -> io::Result<()> { + let current = parent(&self.path)?; + let named = open_at( + ¤t.dir, + std::ffi::OsStr::from_bytes(current.name.as_bytes()), + false, + )?; + let held_dir = self.dir.metadata()?; + let named_dir = current.dir.metadata()?; + if held_dir.dev() != named_dir.dev() || held_dir.ino() != named_dir.ino() { + return Err(denied()); + } + let a = file.metadata()?; + let b = named.metadata()?; + if !a.is_file() || a.nlink() != 1 || a.dev() != b.dev() || a.ino() != b.ino() { + return Err(denied()); + } + Ok(()) + } + pub(super) fn remove(&self, directory: bool) -> io::Result<()> { + // SAFETY: descriptors and names remain live through the syscall. + let result = unsafe { + libc::unlinkat( + self.dir.as_raw_fd(), + self.name.as_ptr(), + if directory { libc::AT_REMOVEDIR } else { 0 }, + ) + }; + if result == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } + } + } + pub(super) fn open(path: &Path, o: &DiskOpenOptions) -> io::Result { + parent(path)?.open(o) + } + pub(super) fn remove(path: &Path, directory: bool) -> io::Result<()> { + parent(path)?.remove(directory) + } + pub(super) fn create_dir(path: &Path, private: bool) -> io::Result<()> { + let p = parent(path)?; + // SAFETY: live descriptor, terminated name and valid mode. + if unsafe { + libc::mkdirat( + p.dir.as_raw_fd(), + p.name.as_ptr(), + if private { 0o700 } else { 0o777 }, + ) + } == 0 + { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } + } + pub(super) fn rename(from: &Path, to: &Path) -> io::Result<()> { + let a = parent(from)?; + let b = parent(to)?; + // SAFETY: both parent descriptors and names remain live. + if unsafe { + libc::renameat( + a.dir.as_raw_fd(), + a.name.as_ptr(), + b.dir.as_raw_fd(), + b.name.as_ptr(), + ) + } == 0 + { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } + } + pub(super) fn set_permissions(path: &Path, permissions: Permissions) -> io::Result<()> { + let p = parent(path)?; + let file = open_at( + &p.dir, + std::ffi::OsStr::from_bytes(p.name.as_bytes()), + false, + )?; + let m = file.metadata()?; + if !m.is_dir() && (!m.is_file() || m.nlink() != 1) { + return Err(denied()); + } + file.set_permissions(permissions) + } + pub(super) fn sync_directory(path: &Path) -> io::Result<()> { + let p = parent(&path.join(".kit-sync-anchor"))?; + p.dir.sync_all() + } + pub(super) fn file_identity(file: &File) -> io::Result { + file.metadata().map(|m| metadata_identity(&m)) + } + pub(super) fn path_identity(path: &Path, follow: bool) -> io::Result { + if path.file_name().is_none() { + return file_identity(&parent(&path.join(".kit-identity-anchor"))?.dir); + } + let p = parent(path)?; + // fstatat reads identity relative to the pinned parent, including a + // symlink itself when follow=false. It never opens for mutation. + let mut info = std::mem::MaybeUninit::::uninit(); + // SAFETY: live parent descriptor, terminated name, writable stat storage. + if unsafe { + libc::fstatat( + p.dir.as_raw_fd(), + p.name.as_ptr(), + info.as_mut_ptr(), + if follow { 0 } else { libc::AT_SYMLINK_NOFOLLOW }, + ) + } != 0 + { + return Err(io::Error::last_os_error()); + } + // SAFETY: successful fstatat initialized the structure. + let info = unsafe { info.assume_init() }; + Ok(FileIdentity { + volume: info.st_dev as u64, + file: info.st_ino, + }) + } + pub(super) fn read_token(file: &File, token: &mut [u8]) -> io::Result<()> { + file.read_exact_at(token, 0) + } + pub(super) fn owned_regular(metadata: &fs::Metadata) -> bool { + // Lock tokens are not credential data. Historical session locks can + // have mode 0644, but must still be a single-link file owned by us. + // SAFETY: geteuid has no preconditions. + metadata.is_file() && metadata.uid() == unsafe { libc::geteuid() } && metadata.nlink() == 1 + } + pub(super) fn private_regular(metadata: &fs::Metadata) -> bool { + owned_regular(metadata) && metadata.mode() & 0o077 == 0 + } + pub(super) fn tighten_owned_lease(file: &File) -> io::Result<()> { + use std::os::unix::fs::PermissionsExt; + let metadata = file.metadata()?; + if !owned_regular(&metadata) { + return Err(denied()); + } + if metadata.mode() & 0o077 != 0 { + // Called only after real OS exclusion and named identity checks. + // Change the held lock inode, never a pathname or unrelated data. + file.set_permissions(Permissions::from_mode(0o600))?; + } + Ok(()) + } +} + +#[cfg(windows)] +mod native { + use super::*; + use std::os::windows::fs::{FileExt, MetadataExt, OpenOptionsExt}; + use std::os::windows::io::AsRawHandle; + use std::path::Component; + const REPARSE: u32 = 0x00200000; + const BACKUP: u32 = 0x02000000; + const REPARSE_ATTRIBUTE: u32 = 0x400; + #[repr(C)] + #[derive(Default)] + struct FileInfo { + attributes: u32, + creation: [u32; 2], + access: [u32; 2], + write: [u32; 2], + volume: u32, + size_high: u32, + size_low: u32, + links: u32, + index_high: u32, + index_low: u32, + } + #[link(name = "kernel32")] + unsafe extern "system" { + fn GetFileInformationByHandle(handle: *mut std::ffi::c_void, info: *mut FileInfo) -> i32; + } + fn file_info(file: &File) -> io::Result { + let mut info = FileInfo::default(); + // SAFETY: live file handle and correctly sized writable Win32 structure. + if unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut info) } == 0 { + return Err(io::Error::last_os_error()); + } + Ok(info) + } + pub(super) fn file_identity(file: &File) -> io::Result { + let info = file_info(file)?; + Ok(FileIdentity { + volume: u64::from(info.volume), + file: (u64::from(info.index_high) << 32) | u64::from(info.index_low), + }) + } + pub(super) fn path_identity(path: &Path, follow: bool) -> io::Result { + // Keep the same ancestor reparse protection as mutation opens, but + // permit directories and query without write access or truncation. + let p = if path.file_name().is_none() { + parent(&path.join(".kit-identity-anchor"))? + } else { + parent(path)? + }; + let named = if path.file_name().is_none() { + p.path.parent().ok_or_else(denied)? + } else { + &p.path + }; + let file = OpenOptions::new() + .read(true) + .custom_flags(BACKUP | if follow { 0 } else { REPARSE }) + .open(named)?; + file_identity(&file) + } + fn identity(file: &File) -> io::Result { + let info = file_info(file)?; + if info.links != 1 + || info.attributes & REPARSE_ATTRIBUTE != 0 + || !file.metadata()?.is_file() + { + return Err(denied()); + } + Ok(FileIdentity { + volume: u64::from(info.volume), + file: (u64::from(info.index_high) << 32) | u64::from(info.index_low), + }) + } + fn directory(path: &Path) -> io::Result { + // Denying write/delete sharing pins every ancestor against rename and + // reparse-point modification until the operation has completed. + let f = OpenOptions::new() + .read(true) + .share_mode(1) + .custom_flags(REPARSE | BACKUP) + .open(path)?; + let m = f.metadata()?; + if !m.is_dir() || m.file_attributes() & REPARSE_ATTRIBUTE != 0 { + return Err(denied()); + } + Ok(f) + } + pub(super) struct Parent { + path: PathBuf, + _ancestors: Vec, + } + pub(super) fn parent(path: &Path) -> io::Result { + let absolute = if path.is_absolute() { + path.to_owned() + } else { + std::env::current_dir()?.join(path) + }; + if absolute.file_name().is_none() { + return Err(denied()); + } + let mut prefix = PathBuf::new(); + let mut ancestors = Vec::new(); + for part in absolute.parent().ok_or_else(denied)?.components() { + match part { + Component::Prefix(_) => prefix.push(part.as_os_str()), + Component::RootDir | Component::Normal(_) => { + prefix.push(part.as_os_str()); + ancestors.push(directory(&prefix)?); + } + Component::CurDir => {} + _ => return Err(denied()), + } + } + Ok(Parent { + path: absolute, + _ancestors: ancestors, + }) + } + impl Parent { + pub(super) fn open(&self, o: &DiskOpenOptions) -> io::Result { + let f = OpenOptions::new() + .read(o.read) + .write(o.write) + .append(o.append) + .create(o.create) + .create_new(o.create_new) + .custom_flags(REPARSE | o.custom_flags as u32) + .open(&self.path)?; + identity(&f)?; + self.check(&f)?; + if o.truncate { + f.set_len(0)?; + } + // Windows private files/directories inherit the containing user's ACL, + // as the pre-resilient backend did; POSIX mode bits do not model ACLs. + Ok(f) + } + pub(super) fn check(&self, file: &File) -> io::Result<()> { + let named = OpenOptions::new() + .read(true) + .custom_flags(REPARSE) + .open(&self.path)?; + if identity(file)? != identity(&named)? { + return Err(denied()); + } + Ok(()) + } + pub(super) fn remove(&self, directory: bool) -> io::Result<()> { + if directory { + fs::remove_dir(&self.path) + } else { + fs::remove_file(&self.path) + } + } + } + pub(super) fn open(path: &Path, o: &DiskOpenOptions) -> io::Result { + parent(path)?.open(o) + } + pub(super) fn remove(path: &Path, directory: bool) -> io::Result<()> { + parent(path)?.remove(directory) + } + pub(super) fn create_dir(path: &Path, _private: bool) -> io::Result<()> { + let p = parent(path)?; + fs::create_dir(&p.path) + } + pub(super) fn rename(from: &Path, to: &Path) -> io::Result<()> { + let a = parent(from)?; + let b = parent(to)?; + fs::rename(&a.path, &b.path) + } + pub(super) fn set_permissions(path: &Path, permissions: Permissions) -> io::Result<()> { + let p = parent(path)?; + let file = OpenOptions::new() + .read(true) + .custom_flags(REPARSE | BACKUP) + .open(&p.path)?; + let m = file.metadata()?; + if m.file_attributes() & REPARSE_ATTRIBUTE != 0 { + return Err(denied()); + } + if !m.is_dir() { + identity(&file)?; + } + file.set_permissions(permissions) + } + pub(super) fn sync_directory(path: &Path) -> io::Result<()> { + let _p = parent(&path.join(".kit-sync-anchor"))?; + // Windows does not support FlushFileBuffers on directory handles. + Ok(()) + } + pub(super) fn open_beneath(root: &Path, relative: &Path) -> io::Result { + if relative.as_os_str().is_empty() + || relative + .components() + .any(|p| !matches!(p, Component::Normal(_))) + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "expected nonempty relative path without traversal", + )); + } + open( + &root.join(relative), + &DiskOpenOptions { + read: true, + ..Default::default() + }, + ) + } + pub(super) fn owned_regular(m: &fs::Metadata) -> bool { + private_regular(m) + } + pub(super) fn tighten_owned_lease(file: &File) -> io::Result<()> { + // Windows retains its inherited ACL; POSIX modes do not apply. + identity(file).map(|_| ()) + } + pub(super) fn private_regular(m: &fs::Metadata) -> bool { + m.is_file() && m.file_attributes() & REPARSE_ATTRIBUTE == 0 + } + pub(super) fn read_token(file: &File, mut token: &mut [u8]) -> io::Result<()> { + let mut offset = 0; + while !token.is_empty() { + let n = file.seek_read(token, offset)?; + if n == 0 { + return Err(io::ErrorKind::UnexpectedEof.into()); + } + offset += n as u64; + token = &mut token[n..]; + } + Ok(()) + } +} + +struct DiskLease { + file: File, + parent: native::Parent, + token: [u8; 64], + remove_on_drop: bool, +} +impl DiskLease { + fn check_identity(&self) -> io::Result<()> { + self.parent.check(&self.file)?; + if !native::private_regular(&self.file.metadata()?) { + return Err(denied()); + } + Ok(()) + } +} +impl BackendLease for DiskLease { + fn check(&self) -> io::Result<()> { + self.check_identity()?; + if self.file.metadata()?.len() != self.token.len() as u64 { + return Err(denied()); + } + let mut token = [0; 64]; + native::read_token(&self.file, &mut token)?; + if token != self.token { + return Err(denied()); + } + self.check_identity() + } +} +impl Drop for DiskLease { + fn drop(&mut self) { + // OS ownership remains held through cleanup. The final unlink assumes + // other actors with write access to this private directory cooperate. + if self.remove_on_drop && self.check().is_ok() { + let _ = self.parent.remove(false); + } + } +} +fn initialize_lease( + lease: &mut DiskLease, + created: bool, + initialize: impl FnOnce(&mut File, &[u8]) -> io::Result<()>, +) -> io::Result<()> { + let result = lease + .check_identity() + .and_then(|()| initialize(&mut lease.file, &lease.token)) + .and_then(|()| lease.check()); + if result.is_err() && created && lease.check_identity().is_ok() { + // Initialization may have failed before a complete token was written. + // Identity, not token equality, authorizes rollback of our new inode. + // The OS lock is still held, and a losing acquirer never reaches here. + lease.parent.remove(false)?; + } + result +} +fn acquire_lease(request: &LeaseRequest) -> io::Result> { + let mut random = zeroize::Zeroizing::new([0u8; 32]); + getrandom::fill(&mut *random).map_err(io::Error::other)?; + let mut token = [0; 64]; + for (i, byte) in random.iter().enumerate() { + token[i * 2] = b"0123456789abcdef"[(byte >> 4) as usize]; + token[i * 2 + 1] = b"0123456789abcdef"[(byte & 15) as usize]; + } + let parent = native::parent(&request.path)?; + let mut options = DiskOpenOptions { + read: true, + write: true, + create_new: true, + private: true, + ..Default::default() + }; + let (file, created) = match parent.open(&options) { + Ok(file) => (file, true), + Err(e) + if e.kind() == io::ErrorKind::AlreadyExists + && request.mode == LeaseMode::ExistingOrNew => + { + options.create_new = false; + (parent.open(&options)?, false) + } + Err(e) => return Err(e), + }; + if !native::owned_regular(&file.metadata()?) { + return Err(denied()); + } + match file.try_lock() { + Ok(()) => {} + Err(fs::TryLockError::WouldBlock) => { + return Err(io::Error::new( + io::ErrorKind::WouldBlock, + "lease is held by another owner", + )); + } + Err(fs::TryLockError::Error(e)) => return Err(e), + } + // Never tighten a live historical owner's permissions. Only the winner + // can validate and secure this inode before replacing its opaque token. + if !created { + parent.check(&file)?; + native::tighten_owned_lease(&file)?; + } + let mut lease = DiskLease { + file, + parent, + token, + remove_on_drop: false, + }; + initialize_lease(&mut lease, created, |file, token| { + file.set_len(0)?; + file.write_all(token)?; + file.sync_all() + })?; + lease.remove_on_drop = request.remove_on_drop; + Ok(Box::new(lease)) +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::os::unix::fs::{MetadataExt, PermissionsExt, symlink}; + use std::sync::atomic::{AtomicU64, Ordering}; + static NEXT: AtomicU64 = AtomicU64::new(0); + struct Temp(PathBuf); + impl Temp { + fn new() -> Self { + let path = std::env::temp_dir().join(format!( + "kit-native-fs-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&path).unwrap(); + Self(fs::canonicalize(path).unwrap()) + } + fn request(&self, remove_on_drop: bool) -> LeaseRequest { + LeaseRequest { + path: self.0.join("lock"), + scope: self.0.clone(), + mode: LeaseMode::ExistingOrNew, + remove_on_drop, + } + } + } + impl Drop for Temp { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + #[test] + fn stable_identity_matches_handles_directories_and_symlink_semantics() { + let temp = Temp::new(); + let path = temp.0.join("data"); + fs::write(&path, b"data").unwrap(); + let file = DiskBackend + .open( + &path, + &DiskOpenOptions { + read: true, + ..Default::default() + }, + ) + .unwrap(); + let id = file.identity().unwrap().unwrap(); + assert_eq!(Some(id), DiskBackend.identity(&path, false).unwrap()); + assert_eq!( + Some(id), + BackendFile::identity(&File::open(&path).unwrap()).unwrap() + ); + let directory = File::open(&temp.0).unwrap(); + assert_eq!( + BackendFile::identity(&directory).unwrap(), + DiskBackend.identity(&temp.0, false).unwrap() + ); + assert!( + DiskBackend + .identity(Path::new("/"), false) + .unwrap() + .is_some() + ); + let link = temp.0.join("link"); + symlink(&path, &link).unwrap(); + assert_eq!(Some(id), DiskBackend.identity(&link, true).unwrap()); + assert_ne!(Some(id), DiskBackend.identity(&link, false).unwrap()); + fs::rename(&path, temp.0.join("old")).unwrap(); + fs::write(&path, b"replacement").unwrap(); + assert_ne!(Some(id), DiskBackend.identity(&path, false).unwrap()); + assert_eq!(Some(id), file.identity().unwrap()); + } + #[test] + fn absent_lease_paths_report_not_found_but_replacements_remain_fenced() { + let temp = Temp::new(); + let dir = temp.0.join("dir"); + fs::create_dir(&dir).unwrap(); + let request = LeaseRequest { + path: dir.join("lock"), + scope: dir.clone(), + mode: LeaseMode::CreateNew, + remove_on_drop: true, + }; + let lease = DiskBackend.acquire_lease(&request).unwrap(); + fs::remove_file(&request.path).unwrap(); + assert_eq!(lease.check().unwrap_err().kind(), io::ErrorKind::NotFound); + drop(lease); + let lease = DiskBackend.acquire_lease(&request).unwrap(); + fs::rename(&dir, temp.0.join("old")).unwrap(); + assert_eq!(lease.check().unwrap_err().kind(), io::ErrorKind::NotFound); + fs::create_dir(&dir).unwrap(); + assert_eq!(lease.check().unwrap_err().kind(), io::ErrorKind::NotFound); + let replacement = DiskBackend.acquire_lease(&request).unwrap(); + assert_eq!( + lease.check().unwrap_err().kind(), + io::ErrorKind::PermissionDenied + ); + drop(lease); + replacement.check().unwrap(); + } + #[test] + fn historical_ascii_lock_takeover_stays_utf8() { + let temp = Temp::new(); + let request = temp.request(false); + fs::write(&request.path, "historical-owner-123\n").unwrap(); + fs::set_permissions(&request.path, Permissions::from_mode(0o644)).unwrap(); + let lease = DiskBackend.acquire_lease(&request).unwrap(); + let token = fs::read_to_string(&request.path).unwrap(); + assert_eq!(token.len(), 64); + assert!(token.bytes().all(|b| b.is_ascii_hexdigit())); + lease.check().unwrap(); + assert_eq!(fs::metadata(&request.path).unwrap().mode() & 0o777, 0o600); + } + #[test] + fn live_historical_ascii_lock_preserves_permissions_and_token() { + let temp = Temp::new(); + let request = temp.request(true); + let mut owner = OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(&request.path) + .unwrap(); + owner + .set_permissions(Permissions::from_mode(0o644)) + .unwrap(); + owner.try_lock().unwrap(); + owner.write_all(b"historical-live-owner\n").unwrap(); + owner.sync_all().unwrap(); + assert_eq!( + DiskBackend.acquire_lease(&request).err().unwrap().kind(), + io::ErrorKind::WouldBlock + ); + assert_eq!(fs::metadata(&request.path).unwrap().mode() & 0o777, 0o644); + assert_eq!( + fs::read_to_string(&request.path).unwrap(), + "historical-live-owner\n" + ); + assert_eq!( + fs::metadata(&request.path).unwrap().ino(), + owner.metadata().unwrap().ino() + ); + drop(owner); + let lease = DiskBackend.acquire_lease(&request).unwrap(); + lease.check().unwrap(); + assert_eq!(fs::metadata(&request.path).unwrap().mode() & 0o777, 0o600); + } + fn new_test_lease(path: &Path) -> DiskLease { + let parent = native::parent(path).unwrap(); + let file = parent + .open(&DiskOpenOptions { + read: true, + write: true, + create_new: true, + private: true, + ..Default::default() + }) + .unwrap(); + file.try_lock().unwrap(); + DiskLease { + file, + parent, + token: [b'a'; 64], + remove_on_drop: false, + } + } + #[test] + fn initialization_write_and_sync_errors_remove_own_new_inode() { + for after_write in [false, true] { + let temp = Temp::new(); + let path = temp.0.join("lock"); + let mut lease = new_test_lease(&path); + let result = initialize_lease(&mut lease, true, |file, token| { + if after_write { + file.write_all(token)?; + } + // Exercise the actual initialization rollback with a failing IO + // operation, without adding fault controls to production state. + Err(io::Error::from_raw_os_error(libc::ENOSPC)) + }); + assert!(result.is_err()); + assert!(!path.exists()); + assert_eq!(lease.file.metadata().unwrap().nlink(), 0); + } + } + #[test] + fn initialization_failure_never_removes_replacement_or_existing_inode() { + let temp = Temp::new(); + let path = temp.0.join("lock"); + let mut lease = new_test_lease(&path); + assert!( + initialize_lease(&mut lease, false, |_, _| Err( + io::ErrorKind::StorageFull.into() + )) + .is_err() + ); + assert!(path.exists()); + assert!( + initialize_lease(&mut lease, true, |_, _| { + fs::rename(&path, temp.0.join("old"))?; + fs::write(&path, b"replacement")?; + Err(io::ErrorKind::StorageFull.into()) + }) + .is_err() + ); + assert_eq!(fs::read(&path).unwrap(), b"replacement"); + } + #[test] + fn mutation_parents_and_hardlink_truncation_are_rejected() { + let temp = Temp::new(); + let real = temp.0.join("real"); + fs::create_dir(&real).unwrap(); + symlink(&real, temp.0.join("alias")).unwrap(); + assert!( + DiskBackend + .create_dir(&temp.0.join("alias/child"), true) + .is_err() + ); + let file = real.join("file"); + fs::write(&file, b"unchanged").unwrap(); + fs::hard_link(&file, real.join("link")).unwrap(); + assert!( + DiskBackend + .open( + &file, + &DiskOpenOptions { + write: true, + truncate: true, + ..Default::default() + } + ) + .is_err() + ); + assert_eq!(fs::read(&file).unwrap(), b"unchanged"); + assert!(DiskBackend.remove_file(&temp.0.join("alias/file")).is_err()); + assert!( + DiskBackend + .rename(&file, &temp.0.join("alias/new")) + .is_err() + ); + } + #[test] + fn lease_parent_replacement_fences_owner() { + let temp = Temp::new(); + let dir = temp.0.join("dir"); + fs::create_dir(&dir).unwrap(); + let request = LeaseRequest { + path: dir.join("lock"), + scope: dir.clone(), + mode: LeaseMode::CreateNew, + remove_on_drop: true, + }; + let lease = DiskBackend.acquire_lease(&request).unwrap(); + fs::rename(&dir, temp.0.join("old")).unwrap(); + fs::create_dir(&dir).unwrap(); + assert!(lease.check().is_err()); + drop(lease); + assert!(temp.0.join("old/lock").exists()); + } + #[test] + fn private_creation_and_io() { + let temp = Temp::new(); + let dir = temp.0.join("private"); + DiskBackend.create_dir(&dir, true).unwrap(); + assert_eq!( + fs::metadata(&dir).unwrap().permissions().mode() & 0o777, + 0o700 + ); + let path = dir.join("data"); + let mut file = DiskBackend + .open( + &path, + &DiskOpenOptions { + read: true, + write: true, + create_new: true, + private: true, + ..Default::default() + }, + ) + .unwrap(); + file.write_all(b"contents").unwrap(); + file.sync_all().unwrap(); + file.rewind().unwrap(); + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes).unwrap(); + assert_eq!(bytes, b"contents"); + assert_eq!(file.metadata().unwrap().permissions().mode() & 0o777, 0o600); + assert_eq!(DiskBackend.read_dir(&dir).unwrap()[0].path, path); + DiskBackend.sync_directory(&dir).unwrap(); + } + #[test] + fn secure_traversal_rejects_symlinks_and_escape() { + let temp = Temp::new(); + fs::create_dir(temp.0.join("dir")).unwrap(); + fs::write(temp.0.join("dir/file"), b"safe").unwrap(); + symlink("dir", temp.0.join("alias")).unwrap(); + symlink("dir/file", temp.0.join("link")).unwrap(); + assert!( + DiskBackend + .open_beneath(&temp.0, Path::new("dir/file")) + .is_ok() + ); + for name in ["alias/file", "link", "../outside", "/etc/passwd", "", "dir"] { + assert!( + DiskBackend.open_beneath(&temp.0, Path::new(name)).is_err(), + "{name}" + ); + } + } + #[test] + fn real_lock_exclusion_and_retention() { + let temp = Temp::new(); + let request = temp.request(false); + let lease = DiskBackend.acquire_lease(&request).unwrap(); + lease.check().unwrap(); + let error = DiskBackend.acquire_lease(&request).err().unwrap(); + assert_eq!(error.kind(), io::ErrorKind::WouldBlock); + lease.check().unwrap(); // Failed contender must not change the token. + let child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + &format!( + "{}::lease_child_probe", + module_path!() + .split_once("::") + .map_or(module_path!(), |(_, path)| path) + ), + "--quiet", + ]) + .env("KIT_NATIVE_LEASE_TEST_PATH", &request.path) + .output() + .unwrap(); + assert!( + child.status.success(), + "{}", + String::from_utf8_lossy(&child.stdout) + ); + assert!( + String::from_utf8_lossy(&child.stdout).contains("running 1 test"), + "child filter did not run the lease probe: {}", + String::from_utf8_lossy(&child.stdout) + ); + drop(lease); + assert!(request.path.exists()); + let next = DiskBackend.acquire_lease(&request).unwrap(); + next.check().unwrap(); + } + #[test] + fn lease_child_probe() { + let Some(path) = std::env::var_os("KIT_NATIVE_LEASE_TEST_PATH") else { + return; + }; + let request = LeaseRequest { + path: PathBuf::from(path), + scope: PathBuf::new(), + mode: LeaseMode::ExistingOrNew, + remove_on_drop: false, + }; + assert_eq!( + DiskBackend.acquire_lease(&request).err().unwrap().kind(), + io::ErrorKind::WouldBlock + ); + } + #[test] + fn cleanup_and_create_new() { + let temp = Temp::new(); + let mut request = temp.request(true); + request.mode = LeaseMode::CreateNew; + let lease = DiskBackend.acquire_lease(&request).unwrap(); + assert_eq!( + DiskBackend.acquire_lease(&request).err().unwrap().kind(), + io::ErrorKind::AlreadyExists + ); + drop(lease); + assert!(!request.path.exists()); + } + #[test] + fn token_and_replacement_fence_old_owner() { + let temp = Temp::new(); + let request = temp.request(true); + let lease = DiskBackend.acquire_lease(&request).unwrap(); + fs::write(&request.path, [0; 32]).unwrap(); + assert!(lease.check().is_err()); + drop(lease); + assert!(request.path.exists()); + // Another concurrently running test can fork while this descriptor is + // live. Its CLOEXEC duplicate releases at exec, not at our local drop. + let mut attempts = 0; + let lease = loop { + match DiskBackend.acquire_lease(&request) { + Ok(lease) => break lease, + Err(e) if e.kind() == io::ErrorKind::WouldBlock && attempts < 100 => { + attempts += 1; + std::thread::sleep(std::time::Duration::from_millis(1)); + } + Err(e) => panic!("lease remained held after drop: {e}"), + } + }; + fs::rename(&request.path, temp.0.join("old")).unwrap(); + let replacement = DiskBackend.acquire_lease(&request).unwrap(); + assert!(lease.check().is_err()); + drop(lease); + replacement.check().unwrap(); + drop(replacement); + assert!(!request.path.exists()); + } + #[test] + fn unsafe_lock_files_rejected_without_modification() { + let temp = Temp::new(); + let request = temp.request(false); + let target = temp.0.join("target"); + fs::write(&target, b"untouched").unwrap(); + symlink(&target, &request.path).unwrap(); + assert!(DiskBackend.acquire_lease(&request).is_err()); + assert_eq!(fs::read(&target).unwrap(), b"untouched"); + fs::remove_file(&request.path).unwrap(); + fs::write(&request.path, b"public").unwrap(); + fs::set_permissions(&request.path, Permissions::from_mode(0o644)).unwrap(); + fs::hard_link(&request.path, temp.0.join("hardlink")).unwrap(); + assert!(DiskBackend.acquire_lease(&request).is_err()); + assert_eq!(fs::read(&request.path).unwrap(), b"public"); + assert_eq!(fs::metadata(&request.path).unwrap().mode() & 0o777, 0o644); + } +} diff --git a/src/resilient_fs/mod.rs b/src/resilient_fs/mod.rs new file mode 100644 index 00000000..c9761cc5 --- /dev/null +++ b/src/resilient_fs/mod.rs @@ -0,0 +1,2304 @@ +//! Process-owned, bounded write-back filesystem. +//! +//! Successful writes and syncs mean *accepted*, not durable: ENOSPC/EDQUOT +//! obligations remain in memory until recovery succeeds. `require_disk` is the +//! explicit durability barrier. No memory state survives process termination. +//! Native locks and secure descriptor traversal never pretend disk success. +mod backend; +pub use backend::*; +pub use std::fs::Permissions; +use std::{ + collections::VecDeque, + ffi::OsString, + io::{self, Read, Seek, SeekFrom, Write}, + path::{Component, Path, PathBuf}, + sync::{Arc, Mutex, OnceLock}, + time::SystemTime, +}; +use zeroize::Zeroizing; + +fn error(kind: io::ErrorKind, msg: &str) -> io::Error { + io::Error::new(kind, msg) +} +fn capacity(e: &io::Error) -> bool { + if matches!( + e.kind(), + io::ErrorKind::StorageFull | io::ErrorKind::QuotaExceeded + ) { + return true; + } + #[cfg(unix)] + { + matches!(e.raw_os_error(), Some(libc::ENOSPC) | Some(libc::EDQUOT)) + } + #[cfg(not(unix))] + { + matches!(e.raw_os_error(), Some(112) | Some(39) | Some(1816)) + } +} +// Actual allocator failures signal process-wide pressure, unlike an individual +// service's configurable budget. The application decides cancellation policy. +static ALLOCATION_EXHAUSTED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); +static ALLOCATION_FAILURE_HANDLER: OnceLock !> = OnceLock::new(); + +/// Install a process-level emergency exit before callers can allocate an error +/// wrapper. Configured-budget exhaustion still uses ordinary cancellation. +pub fn set_allocation_failure_handler(handler: fn() -> !) -> Result<(), fn() -> !> { + ALLOCATION_FAILURE_HANDLER.set(handler) +} + +fn allocation_oom() -> io::Error { + ALLOCATION_EXHAUSTED.store(true, std::sync::atomic::Ordering::Release); + if let Some(handler) = ALLOCATION_FAILURE_HANDLER.get() { + handler(); + } + oom() +} +fn oom() -> io::Error { + io::ErrorKind::OutOfMemory.into() +} +fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, T> { + m.lock().unwrap_or_else(|e| e.into_inner()) +} +fn bytes(data: &[u8]) -> io::Result>> { + let mut v = Vec::new(); + v.try_reserve_exact(data.len()) + .map_err(|_| allocation_oom())?; + v.extend_from_slice(data); + Ok(Zeroizing::new(v)) +} +#[cfg(unix)] +fn permissions(private: bool, dir: bool) -> Permissions { + use std::os::unix::fs::PermissionsExt; + Permissions::from_mode(if private { + if dir { 0o700 } else { 0o600 } + } else if dir { + 0o755 + } else { + 0o644 + }) +} +#[derive(Clone, Copy, Debug)] +pub struct FileType { + file: bool, + dir: bool, + symlink: bool, +} +impl FileType { + pub fn is_file(&self) -> bool { + self.file + } + pub fn is_dir(&self) -> bool { + self.dir + } + pub fn is_symlink(&self) -> bool { + self.symlink + } +} +fn same_disk_identity(a: Option, b: Option) -> bool { + a.is_some() && a == b +} +fn next_identity() -> u64 { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); + NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed) +} +#[derive(Clone, Debug)] +pub struct Metadata { + identity: u64, + disk_identity: Option, + disk: Option, + kind: FileType, + len: u64, + permissions: Permissions, + modified: SystemTime, +} +impl Metadata { + fn disk(m: std::fs::Metadata, disk_identity: Option) -> Self { + Self { + identity: 0, + disk_identity, + kind: FileType { + file: m.is_file(), + dir: m.is_dir(), + symlink: m.file_type().is_symlink(), + }, + len: m.len(), + permissions: m.permissions(), + modified: m.modified().unwrap_or(SystemTime::UNIX_EPOCH), + disk: Some(m), + } + } + pub fn is_file(&self) -> bool { + self.kind.file + } + pub fn is_dir(&self) -> bool { + self.kind.dir + } + pub fn file_type(&self) -> FileType { + self.kind + } + pub fn len(&self) -> u64 { + self.len + } + pub fn is_empty(&self) -> bool { + self.len == 0 + } + pub fn permissions(&self) -> Permissions { + self.permissions.clone() + } + pub fn modified(&self) -> io::Result { + Ok(self.modified) + } + pub fn same_identity(&self, other: &Metadata) -> bool { + if self.identity != 0 && other.identity != 0 { + return self.identity == other.identity; + } + same_disk_identity(self.disk_identity, other.disk_identity) + } + pub fn disk_metadata(&self) -> Option<&std::fs::Metadata> { + self.disk.as_ref() + } +} +type Payload = Arc>>; +type Native = Arc>>; +#[derive(Clone)] +enum Source { + Memory(Payload), + Native(Native), +} +#[derive(Clone)] +struct Patch { + offset: u64, + data: Payload, + len: u64, +} +#[derive(Clone)] +struct Image { + source: Source, + base_len: u64, + len: u64, + patches: Arc>, +} +impl Image { + fn memory(data: Payload) -> Self { + let len = data.len() as u64; + Self { + source: Source::Memory(data), + base_len: len, + len, + patches: Arc::new(Vec::new()), + } + } + fn native(file: Box) -> io::Result { + let len = file.metadata()?.len(); + Ok(Self { + source: Source::Native(Arc::new(Mutex::new(file))), + base_len: len, + len, + patches: Arc::new(Vec::new()), + }) + } + fn payload_bytes(&self) -> usize { + let base = match &self.source { + Source::Memory(d) => d.len(), + Source::Native(_) => 0, + }; + self.patches + .iter() + .fold(base, |n, p| n.saturating_add(p.data.len())) + } + fn patched(&self, offset: u64, data: &[u8], len: u64) -> io::Result { + let mut patches = Vec::new(); + patches + .try_reserve_exact(self.patches.len() + 1) + .map_err(|_| allocation_oom())?; + patches.extend(self.patches.iter().cloned()); + patches.push(Patch { + offset, + data: Arc::new(bytes(data)?), + len, + }); + Ok(Self { + source: self.source.clone(), + base_len: self.base_len, + len, + patches: Arc::new(patches), + }) + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result { + let n = usize::try_from(self.len.saturating_sub(offset).min(buf.len() as u64)).unwrap(); + let buf = &mut buf[..n]; + buf.fill(0); + let base_n = usize::try_from(self.base_len.saturating_sub(offset).min(n as u64)).unwrap(); + match &self.source { + Source::Memory(data) => { + if base_n > 0 { + buf[..base_n].copy_from_slice(&data[offset as usize..offset as usize + base_n]); + } + } + Source::Native(file) => { + if base_n > 0 { + let mut file = lock(file); + file.seek(SeekFrom::Start(offset))?; + file.read_exact(&mut buf[..base_n])?; + } + } + } + for patch in self.patches.iter() { + if patch.len < offset + n as u64 { + let start = patch.len.saturating_sub(offset).min(n as u64) as usize; + buf[start..].fill(0); + } + let lo = offset.max(patch.offset); + let hi = (offset + n as u64) + .min(patch.offset + patch.data.len() as u64) + .min(patch.len); + if hi > lo { + buf[(lo - offset) as usize..(hi - offset) as usize].copy_from_slice( + &patch.data[(lo - patch.offset) as usize..(hi - patch.offset) as usize], + ); + } + } + Ok(n) + } + fn write_to(&self, file: &mut dyn BackendFile) -> io::Result<()> { + let mut buf = Zeroizing::new([0u8; 64 * 1024]); + let mut offset = 0; + while offset < self.len { + let n = self.read_at(offset, &mut *buf)?; + file.write_all(&buf[..n])?; + offset += n as u64; + } + Ok(()) + } +} +struct Object { + image: Image, + meta: Metadata, + path: Option, + dirty: bool, +} +impl Object { + fn memory(data: Payload, meta: Metadata) -> Self { + Self { + image: Image::memory(data), + meta, + path: None, + dirty: true, + } + } + fn native(file: Box, path: PathBuf) -> io::Result { + let meta = Metadata::disk(file.metadata()?, file.identity()?); + Ok(Self { + image: Image::native(file)?, + meta, + path: Some(path), + dirty: false, + }) + } +} +type Obj = Arc>; +struct Entry { + path: PathBuf, + object: Option, +} +#[derive(Clone)] +pub struct Fs { + service: Arc, + lease: Option>, +} +struct Service { + backend: Arc, + state: Mutex, + max_bytes: usize, + max_operations: usize, +} +struct State { + entries: Vec, + redirects: Vec<(PathBuf, PathBuf)>, + objects: Vec>>, + pending: VecDeque, + next: u64, + exhausted: bool, + leases: Vec<( + PathBuf, + std::sync::Weak, + std::sync::Weak, + )>, +} +#[derive(Debug)] +pub struct Status { + pub pending_operations: usize, + pub retained_bytes: usize, + pub exhausted: bool, +} +#[derive(Debug)] +pub struct RecoveryReport { + pub completed_operations: usize, + pub remaining_operations: usize, + pub blocked: Option, +} +pub struct Lease { + inner: Arc, +} +struct LeaseCaller { + authority: Arc, +} +impl std::ops::Deref for LeaseCaller { + type Target = LeaseInner; + fn deref(&self) -> &LeaseInner { + &self.authority + } +} +struct LeaseInner { + fenced: Mutex>, + native: Box, + scope: PathBuf, + service: std::sync::Weak, +} +impl LeaseInner { + fn check(&self) -> io::Result<()> { + let mut fenced = lock(&self.fenced); + if let Some(kind) = *fenced { + if kind == io::ErrorKind::NotFound { + // Keep Missing while the name is absent, but report a replaced + // owner as PermissionDenied. A restored old inode stays fenced. + match self.native.check() { + Err(e) if e.kind() == io::ErrorKind::NotFound => return Err(e), + _ => { + *fenced = Some(io::ErrorKind::PermissionDenied); + return Err(io::ErrorKind::PermissionDenied.into()); + } + } + } + return Err(kind.into()); + } + let result = self.native.check(); + if let Err(e) = &result { + *fenced = Some(e.kind()); + } + result + } +} +impl Lease { + pub fn check(&self) -> io::Result<()> { + self.inner.check() + } +} +enum Action { + Put { + path: PathBuf, + temp: PathBuf, + temp_file: Option, + parent_identity: Option, + image: Image, + permissions: Permissions, + stage: u8, + }, + Mkdir { + path: PathBuf, + private: bool, + stage: u8, + }, + Unlink { + path: PathBuf, + dir: bool, + stage: u8, + }, + Rename { + from: PathBuf, + to: PathBuf, + stage: u8, + }, + Chmod { + path: PathBuf, + permissions: Permissions, + }, + Sync { + path: PathBuf, + }, +} +struct Pending { + action: Action, + lease: Option>, +} +impl Action { + fn bytes(&self) -> usize { + match self { + Self::Put { image, .. } => image.payload_bytes(), + _ => 0, + } + } + fn touches(&self, p: &Path) -> bool { + match self { + Self::Rename { from, to, .. } => { + from.starts_with(p) || p.starts_with(from) || to.starts_with(p) || p.starts_with(to) + } + Self::Put { path, .. } + | Self::Mkdir { path, .. } + | Self::Unlink { path, .. } + | Self::Chmod { path, .. } + | Self::Sync { path } => path.starts_with(p) || p.starts_with(path), + } + } +} +static GLOBAL: OnceLock = OnceLock::new(); +pub fn initialize_global(fs: Fs) -> Result<(), Fs> { + GLOBAL.set(fs) +} +pub fn global() -> &'static Fs { + GLOBAL.get_or_init(|| Fs::new(Arc::new(DiskBackend))) +} +impl Fs { + pub fn new(backend: Arc) -> Self { + Self::with_budget(backend, 64 * 1024 * 1024, 4096) + } + pub fn with_budget(backend: Arc, max_bytes: usize, max_operations: usize) -> Self { + Self { + service: Arc::new(Service { + backend, + state: Mutex::new(State { + entries: Vec::new(), + redirects: Vec::new(), + objects: Vec::new(), + pending: VecDeque::new(), + next: 0, + exhausted: false, + leases: Vec::new(), + }), + max_bytes, + max_operations, + }), + lease: None, + } + } + pub fn guarded(&self, lease: &Lease) -> io::Result { + lease.check()?; + if !lease.inner.service.ptr_eq(&Arc::downgrade(&self.service)) { + return Err(error( + io::ErrorKind::PermissionDenied, + "lease belongs to another filesystem", + )); + } + Ok(Self { + service: self.service.clone(), + lease: Some(lease.inner.clone()), + }) + } + pub fn acquire_lease, Q: AsRef>( + &self, + path: P, + scope: Q, + mode: LeaseMode, + ) -> io::Result { + let cleanup = matches!(mode, LeaseMode::CreateNew); + self.acquire_lease_with_cleanup(path, scope, mode, cleanup) + } + pub fn acquire_lease_with_cleanup, Q: AsRef>( + &self, + path: P, + scope: Q, + mode: LeaseMode, + remove_on_drop: bool, + ) -> io::Result { + let path = self.norm(path.as_ref())?; + let scope = self.norm(scope.as_ref())?; + self.require_disk(&path)?; + let mut state = lock(&self.service.state); + state + .leases + .retain(|(_, authority, _)| authority.strong_count() > 0); + if let Some(index) = state.leases.iter().position(|(p, _, _)| *p == path) + && let Some(authority) = state.leases[index].1.upgrade() + { + let valid = authority.check(); + let dirty = state + .pending + .iter() + .any(|p| p.lease.as_ref().is_some_and(|l| Arc::ptr_eq(l, &authority))); + if valid.is_err() && !dirty { + // A lost *clean* lease does not reserve a namespace forever. + // The old observer remains fenced; reacquisition is real native IO. + state.leases.remove(index); + } else { + valid?; + if state.leases[index].2.upgrade().is_some() { + return Err(error( + io::ErrorKind::WouldBlock, + "lease has a live observer", + )); + } + if authority.scope != scope { + return Err(error( + io::ErrorKind::PermissionDenied, + "retained lease scope mismatch", + )); + } + let owner = Arc::new(LeaseCaller { authority }); + state.leases[index].2 = Arc::downgrade(&owner); + return Ok(Lease { inner: owner }); + } + } + state.leases.try_reserve(1).map_err(|_| allocation_oom())?; + let native = self.service.backend.acquire_lease(&LeaseRequest { + path: path.clone(), + scope: scope.clone(), + mode, + remove_on_drop, + })?; + let authority = Arc::new(LeaseInner { + fenced: Mutex::new(None), + native, + scope, + service: Arc::downgrade(&self.service), + }); + let caller = Arc::new(LeaseCaller { + authority: authority.clone(), + }); + state + .leases + .push((path, Arc::downgrade(&authority), Arc::downgrade(&caller))); + Ok(Lease { inner: caller }) + } + fn norm(&self, path: &Path) -> io::Result { + // Canonicalize a real ancestor, never collapse `..` through a symlink. + let absolute = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir()?.join(path) + }; + if absolute + .components() + .any(|c| matches!(c, Component::ParentDir)) + { + return self.service.backend.canonicalize(&absolute); + } + let mut ancestor = absolute.parent().unwrap_or(&absolute); + let mut suffix = Vec::new(); + if let Some(name) = absolute.file_name() { + suffix.push(name.to_os_string()); + } + loop { + match self.service.backend.canonicalize(ancestor) { + Ok(mut base) => { + for part in suffix.iter().rev() { + base.push(part); + } + return Ok(base); + } + Err(e) if e.kind() == io::ErrorKind::NotFound => { + if let Some(name) = ancestor.file_name() { + suffix.push(name.to_os_string()); + } + ancestor = ancestor.parent().ok_or(e)?; + } + Err(e) => return Err(e), + } + } + } + fn authority(&self, path: &Path) -> io::Result<()> { + if let Some(l) = &self.lease { + l.check()?; + if !path.starts_with(&l.scope) { + return Err(error( + io::ErrorKind::PermissionDenied, + "mutation outside lease scope", + )); + } + } + Ok(()) + } + fn secure_path(&self, s: &State, path: &Path, final_link: bool) -> io::Result<()> { + let mut cur = PathBuf::new(); + for c in path.components() { + cur.push(c.as_os_str()); + if final_link && cur == path { + break; + } + if let Some(e) = s.entries.iter().find(|e| e.path == cur) { + if let Some(o) = &e.object { + if lock(o).meta.kind.symlink { + return Err(error( + io::ErrorKind::PermissionDenied, + "symlink in managed path", + )); + } + continue; + } else { + continue; + } + } + match self + .service + .backend + .metadata(&Self::disk_path(s, &cur), false) + { + Ok(m) if m.file_type().is_symlink() => { + return Err(error( + io::ErrorKind::PermissionDenied, + "symlink in managed path", + )); + } + Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e), + _ => {} + } + } + Ok(()) + } + fn retained(s: &State) -> usize { + let objects = s + .objects + .iter() + .filter_map(|o| o.upgrade()) + .map(|o| lock(&o).image.payload_bytes()) + .sum::(); + objects.saturating_add(s.pending.iter().map(|p| p.action.bytes()).sum::()) + } + pub fn status(&self) -> Status { + let s = lock(&self.service.state); + Status { + pending_operations: s.pending.len(), + retained_bytes: Self::retained(&s), + exhausted: s.exhausted + || ALLOCATION_EXHAUSTED.load(std::sync::atomic::Ordering::Acquire), + } + } + fn reserve(&self, s: &mut State, additional: usize, entries: usize) -> io::Result<()> { + if s.pending.len() >= self.service.max_operations + || Self::retained(s).saturating_add(additional) > self.service.max_bytes + || s.entries.len().saturating_add(entries) + > self.service.max_operations.saturating_mul(4) + { + s.exhausted = true; + return Err(oom()); + } + s.objects.retain(|w| w.strong_count() > 0); + if s.pending.try_reserve(1).is_err() + || s.entries.try_reserve(entries).is_err() + || s.objects.try_reserve(entries).is_err() + { + s.exhausted = true; + return Err(allocation_oom()); + } + Ok(()) + } + fn disk_path(s: &State, path: &Path) -> PathBuf { + let mut mapped = if let Some((to, from)) = s + .redirects + .iter() + .filter(|(to, _)| path.starts_with(to)) + .max_by_key(|(to, _)| to.components().count()) + { + if path == to { + from.clone() + } else { + from.join(path.strip_prefix(to).unwrap()) + } + } else { + path.to_path_buf() + }; + for p in &s.pending { + if let Action::Rename { from, to, stage: 1 } = &p.action + && mapped.starts_with(from) + { + mapped = if mapped == *from { + to.clone() + } else { + to.join(mapped.strip_prefix(from).unwrap()) + }; + } + } + mapped + } + fn prepare(s: &mut State, entries: usize) -> io::Result<()> { + s.entries + .try_reserve(entries) + .map_err(|_| allocation_oom())?; + s.objects + .try_reserve(entries) + .map_err(|_| allocation_oom())?; + s.redirects + .try_reserve(entries) + .map_err(|_| allocation_oom())?; + s.pending.try_reserve(1).map_err(|_| allocation_oom())?; + Ok(()) + } + fn lookup(&self, s: &State, path: &Path) -> io::Result { + if let Some(e) = s + .entries + .iter() + .rev() + .find(|e| e.path == path && s.pending.iter().any(|p| p.action.touches(path))) + { + return e + .object + .as_ref() + .map(|o| lock(o).meta.clone()) + .ok_or_else(|| error(io::ErrorKind::NotFound, "removed path")); + } + if s.entries.iter().any(|e| { + e.object.is_none() + && path.starts_with(&e.path) + && s.pending.iter().any(|p| p.action.touches(&e.path)) + }) { + return Err(error(io::ErrorKind::NotFound, "removed ancestor")); + } + let path = Self::disk_path(s, path); + Ok(Metadata::disk( + self.service.backend.metadata(&path, false)?, + self.service.backend.identity(&path, false)?, + )) + } + fn object(&self, s: &mut State, path: &Path) -> io::Result { + if let Some(e) = s.entries.iter().find(|e| e.path == path) { + return e + .object + .clone() + .ok_or_else(|| error(io::ErrorKind::NotFound, "removed path")); + } + let meta = self.lookup(s, path)?; + if !meta.is_file() { + return Err(error(io::ErrorKind::InvalidInput, "not a regular file")); + } + let native = self.service.backend.open( + &Self::disk_path(s, path), + &DiskOpenOptions { + read: true, + ..Default::default() + }, + )?; + let object = Arc::new(Mutex::new(Object::native(native, path.to_path_buf())?)); + s.objects.try_reserve(1).map_err(|_| allocation_oom())?; + s.objects.push(Arc::downgrade(&object)); + Ok(object) + } + fn entry(s: &mut State, path: PathBuf, object: Option) { + if let Some(o) = &object { + lock(o).path = Some(path.clone()); + } + if let Some(o) = &object + && !s.objects.iter().any(|w| w.ptr_eq(&Arc::downgrade(o))) + { + s.objects.push(Arc::downgrade(o)); + } + if let Some(e) = s.entries.iter_mut().find(|e| e.path == path) { + e.object = object; + } else { + s.entries.push(Entry { path, object }); + } + } + fn new_permissions( + &self, + s: &State, + path: &Path, + private: bool, + dir: bool, + ) -> io::Result { + #[cfg(unix)] + { + let _ = (s, path); + Ok(permissions(private, dir)) + } + #[cfg(not(unix))] + { + let _ = (private, dir); + let parent = path + .parent() + .ok_or_else(|| error(io::ErrorKind::InvalidInput, "no parent"))?; + let mut p = self.lookup(s, parent)?.permissions(); + // This branch is non-Unix: clear the Windows readonly attribute, + // never broaden Unix mode bits. ACL policy remains in DiskBackend. + #[allow(clippy::permissions_set_readonly_false)] + p.set_readonly(false); + Ok(p) + } + } + fn parent(&self, s: &State, path: &Path) -> io::Result<()> { + let p = path + .parent() + .ok_or_else(|| error(io::ErrorKind::InvalidInput, "no parent"))?; + let parent = self.lookup(s, p)?; + if parent.permissions().readonly() { + return Err(error( + io::ErrorKind::PermissionDenied, + "parent directory is read-only", + )); + } + if !parent.is_dir() { + return Err(error( + io::ErrorKind::NotADirectory, + "parent is not a directory", + )); + } + Ok(()) + } + fn preflight(&self, s: &State, path: &Path) -> io::Result<()> { + self.authority(path)?; + self.secure_path(s, path, false)?; + self.parent(s, path)?; + let existing = match self.lookup(s, path) { + Ok(m) => Some(m), + Err(e) if e.kind() == io::ErrorKind::NotFound => None, + Err(e) => return Err(e), + }; + if let Some(m) = existing { + if !m.is_file() { + return Err(error(io::ErrorKind::InvalidInput, "not a regular file")); + } + #[cfg(unix)] + if let Some(d) = m.disk_metadata() { + use std::os::unix::fs::MetadataExt; + if d.nlink() > 1 { + return Err(error( + io::ErrorKind::Unsupported, + "hard-linked mutation unsupported", + )); + } + } + if m.permissions().readonly() { + return Err(error(io::ErrorKind::PermissionDenied, "read-only file")); + } + match self.service.backend.open( + path, + &DiskOpenOptions { + write: true, + ..Default::default() + }, + ) { + Ok(_) => {} + Err(e) if capacity(&e) => {} + Err(e) + if e.kind() == io::ErrorKind::NotFound + && s.entries + .iter() + .any(|e| e.path == path && e.object.is_some()) => {} + Err(e) => return Err(e), + } + } + Ok(()) + } + fn put_action(s: &mut State, path: &Path, image: Image, permissions: Permissions) -> Action { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); + s.next = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + static START: OnceLock = OnceLock::new(); + let start = START.get_or_init(|| { + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + }); + let temp = path.with_file_name(format!( + ".kit-resilient-{}-{}-{}.tmp", + std::process::id(), + start, + s.next + )); + Action::Put { + path: path.to_path_buf(), + temp, + temp_file: None, + parent_identity: None, + image, + permissions, + stage: 0, + } + } + // Healthy IO never consumes the configured fallback budget. Only a + // complete, unpublished obligation enters the bounded write-back queue. + fn submit(&self, s: &mut State, mut action: Action) -> (bool, io::Result<()>) { + if s.pending.try_reserve(1).is_err() { + return (false, Err(allocation_oom())); + } + if !s.pending.is_empty() { + if let Err(e) = self.reserve(s, action.bytes().saturating_mul(2), 1) { + return (false, Err(e)); + } + return self.enqueue(s, action); + } + let result = self + .lease + .as_ref() + .map_or(Ok(()), |l| l.check()) + .and_then(|_| self.replay(&mut action)); + match result { + Ok(()) => (true, Ok(())), + Err(e) => { + let published = matches!( + action, + Action::Put { stage: 3, .. } + | Action::Rename { stage: 1, .. } + | Action::Mkdir { stage: 1, .. } + | Action::Unlink { stage: 1, .. } + ); + if published && let Action::Put { image, .. } = &mut action { + *image = Image::memory(Arc::new(Zeroizing::new(Vec::new()))); + } + if !capacity(&e) && !published { + self.abandon(&action); + return (false, Err(e)); + } + if !published && let Err(e) = self.reserve(s, action.bytes().saturating_mul(2), 1) { + self.abandon(&action); + return (false, Err(e)); + } + s.pending.push_back(Pending { + action, + lease: self.lease.as_ref().map(|c| c.authority.clone()), + }); + (true, Ok(())) + } + } + } + fn abandon(&self, action: &Action) { + if let Action::Put { + temp, + temp_file: Some(file), + stage: 1 | 2, + .. + } = action + && let Ok(held) = lock(file).identity() + && let Ok(named) = self.service.backend.identity(temp, false) + && same_disk_identity(held, named) + { + let _ = self.service.backend.remove_file(temp); + } + } + fn rebase(&self, s: &mut State) { + for object in s.objects.iter().filter_map(|w| w.upgrade()) { + let mut object = lock(&object); + let Some(path) = object.path.as_ref() else { + continue; + }; + if !object.dirty || !object.meta.is_file() { + continue; + } + if s.pending.iter().any(|p| { + p.action.touches(path) + && !matches!(p.action, Action::Put { stage: 3, .. } | Action::Sync { .. }) + }) { + continue; + } + let published = s.pending.iter().rev().find_map(|p| match &p.action { + Action::Put { + path: p, + temp_file: Some(file), + stage: 3, + .. + } if p == path => Some(file.clone()), + _ => None, + }); + if let Some(file) = published { + let snapshot = { + let held = lock(&file); + held.metadata() + .and_then(|meta| Ok((meta, held.identity()?))) + }; + if let Ok((meta, disk_identity)) = snapshot { + let len = meta.len(); + object.image = Image { + source: Source::Native(file), + base_len: len, + len, + patches: Arc::new(Vec::new()), + }; + object.meta.disk_identity = disk_identity; + object.meta.disk = Some(meta); + object.dirty = false; + } + } else if let Ok(native) = self.service.backend.open( + path, + &DiskOpenOptions { + read: true, + ..Default::default() + }, + ) && let Ok(meta) = native.metadata() + && let Ok(disk_identity) = native.identity() + && let Ok(image) = Image::native(native) + { + object.image = image; + object.meta.disk_identity = disk_identity; + object.meta.disk = Some(meta); + object.dirty = false; + } + } + } + fn enqueue(&self, s: &mut State, action: Action) -> (bool, io::Result<()>) { + s.pending.push_back(Pending { + action, + lease: self.lease.as_ref().map(|c| c.authority.clone()), + }); + let r = self.recover_locked(s); + match r.blocked { + Some(e) if !capacity(&e) => { + // An unpublished temporary image can be abandoned safely. An + // already published rename retains its directory-sync obligation. + let published = s.pending.len() == 1 + && s.pending.back().is_some_and(|p| { + matches!( + p.action, + Action::Put { stage: 3, .. } + | Action::Rename { stage: 1, .. } + | Action::Mkdir { stage: 1, .. } + | Action::Unlink { stage: 1, .. } + ) + }); + if !published && let Some(p) = s.pending.pop_back() { + self.abandon(&p.action); + } + if published { + (true, Ok(())) + } else { + (false, Err(e)) + } + } + _ => (true, Ok(())), + } + } + pub fn recover(&self) -> RecoveryReport { + let mut s = lock(&self.service.state); + let report = self.recover_locked(&mut s); + self.rebase(&mut s); + Self::prune(&mut s); + report + } + fn recover_locked(&self, s: &mut State) -> RecoveryReport { + let mut completed = 0; + let mut blocked = None; + for _ in 0..64 { + let Some(p) = s.pending.front_mut() else { + break; + }; + let result = p + .lease + .as_ref() + .map_or(Ok(()), |l| l.check()) + .and_then(|_| self.replay(&mut p.action)); + match result { + Ok(()) => { + if let Some(Pending { + action: Action::Rename { from, to, .. }, + .. + }) = s.pending.pop_front() + { + s.redirects.retain(|(path, _)| *path != to); + for (_, source) in &mut s.redirects { + if source.starts_with(&from) { + *source = if *source == from { + to.clone() + } else { + to.join(source.strip_prefix(&from).unwrap()) + }; + } + } + } + completed += 1; + } + Err(e) => { + blocked = Some(e); + break; + } + } + } + RecoveryReport { + completed_operations: completed, + remaining_operations: s.pending.len(), + blocked, + } + } + fn replay(&self, a: &mut Action) -> io::Result<()> { + let b = &self.service.backend; + match a { + Action::Put { + path, + temp, + temp_file, + parent_identity, + image, + permissions, + stage, + } => { + if *stage == 0 { + let parent = b.metadata(path.parent().unwrap(), false)?; + if !parent.is_dir() { + return Err(error( + io::ErrorKind::NotADirectory, + "replacement parent changed", + )); + } + *parent_identity = b.identity(path.parent().unwrap(), false)?; + if parent_identity.is_none() { + return Err(error( + io::ErrorKind::PermissionDenied, + "replacement parent identity unavailable", + )); + } + let file = b.open( + temp, + &DiskOpenOptions { + read: true, + write: true, + create_new: true, + private: true, + ..Default::default() + }, + )?; + *temp_file = Some(Arc::new(Mutex::new(file))); + *stage = 1; + } + if *stage < 3 { + let file = temp_file.as_ref().ok_or_else(|| { + error(io::ErrorKind::InvalidData, "missing temporary descriptor") + })?; + let mut file = lock(file); + #[cfg(unix)] + let named = b.metadata(temp, false)?; + if !same_disk_identity(file.identity()?, b.identity(temp, false)?) + || !same_disk_identity( + *parent_identity, + b.identity(path.parent().unwrap(), false)?, + ) + { + return Err(error( + io::ErrorKind::PermissionDenied, + "temporary file or parent identity changed", + )); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if named.nlink() != 1 { + return Err(error( + io::ErrorKind::PermissionDenied, + "temporary file is hard-linked", + )); + } + } + if *stage == 1 { + file.set_len(0)?; + file.seek(SeekFrom::Start(0))?; + file.set_permissions(permissions.clone())?; + image.write_to(&mut **file)?; + file.sync_all()?; + *stage = 2; + } + if *stage == 2 { + b.rename(temp, path)?; + *stage = 3; + } + } + b.sync_directory(path.parent().unwrap()) + } + Action::Mkdir { + path, + private, + stage, + } => { + if *stage == 0 { + b.create_dir(path, *private)?; + *stage = 1; + } + b.sync_directory(path.parent().unwrap()) + } + Action::Unlink { path, dir, stage } => { + if *stage == 0 { + match if *dir { + b.remove_dir(path) + } else { + b.remove_file(path) + } { + Err(e) if e.kind() == io::ErrorKind::NotFound => {} + r => r?, + } + *stage = 1; + } + b.sync_directory(path.parent().unwrap()) + } + Action::Rename { from, to, stage } => { + if *stage == 0 { + b.rename(from, to)?; + *stage = 1; + } + b.sync_directory(to.parent().unwrap())?; + if from.parent() != to.parent() { + b.sync_directory(from.parent().unwrap())?; + } + Ok(()) + } + Action::Chmod { path, permissions } => b.set_permissions(path, permissions.clone()), + Action::Sync { path } => b.sync_directory(path), + } + } + pub fn require_disk>(&self, path: P) -> io::Result<()> { + let path = self.norm(path.as_ref())?; + let mut s = lock(&self.service.state); + let report = self.recover_locked(&mut s); + self.rebase(&mut s); + Self::prune(&mut s); + if s.pending.iter().any(|p| p.action.touches(&path)) { + return Err(report.blocked.unwrap_or_else(|| { + error( + io::ErrorKind::WouldBlock, + "bounded recovery has pending work", + ) + })); + } + Ok(()) + } + pub fn metadata>(&self, path: P) -> io::Result { + let path = self.norm(path.as_ref())?; + let s = lock(&self.service.state); + self.secure_path(&s, &path, false)?; + self.lookup(&s, &path) + } + pub fn symlink_metadata>(&self, path: P) -> io::Result { + let path = self.norm(path.as_ref())?; + let s = lock(&self.service.state); + self.secure_path(&s, &path, true)?; + self.lookup(&s, &path) + } + pub fn try_exists>(&self, path: P) -> io::Result { + match self.metadata(path) { + Ok(_) => Ok(true), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(e), + } + } + pub fn read>(&self, path: P) -> io::Result> { + let file = self.open(path)?; + let image = lock(&file.object).image.clone(); + let len = usize::try_from(image.len).map_err(|_| allocation_oom())?; + let mut data = Zeroizing::new(Vec::new()); + data.try_reserve_exact(len).map_err(|_| allocation_oom())?; + data.resize(len, 0); + image.read_at(0, &mut data)?; + Ok(std::mem::take(&mut *data)) + } + pub fn read_to_string>(&self, path: P) -> io::Result { + String::from_utf8(self.read(path)?) + .map_err(|e| error(io::ErrorKind::InvalidData, &e.to_string())) + } + pub fn write, C: AsRef<[u8]>>(&self, path: P, contents: C) -> io::Result<()> { + self.replace_impl(path.as_ref(), contents.as_ref(), false, false) + } + pub fn replace>(&self, path: P, contents: &[u8]) -> io::Result<()> { + self.replace_impl(path.as_ref(), contents, false, true) + } + pub fn replace_private>(&self, path: P, contents: &[u8]) -> io::Result<()> { + self.replace_impl(path.as_ref(), contents, true, true) + } + fn replace_impl( + &self, + path: &Path, + contents: &[u8], + private: bool, + new_object: bool, + ) -> io::Result<()> { + let path = self.norm(path)?; + let mut s = lock(&self.service.state); + self.recover_before(&mut s)?; + self.preflight(&s, &path)?; + s.entries.try_reserve(1).map_err(|_| allocation_oom())?; + s.objects.try_reserve(1).map_err(|_| allocation_oom())?; + let perms = if private { + self.new_permissions(&s, &path, true, false)? + } else { + self.lookup(&s, &path) + .map(|m| m.permissions()) + .map_or_else( + |e| { + if e.kind() == io::ErrorKind::NotFound { + self.new_permissions(&s, &path, false, false) + } else { + Err(e) + } + }, + Ok, + )? + }; + let data = Arc::new(bytes(contents)?); + let object = if !new_object { + s.entries + .iter() + .find(|e| e.path == path) + .and_then(|e| e.object.clone()) + } else { + None + }; + let meta = Metadata { + identity: next_identity(), + disk_identity: None, + disk: None, + kind: FileType { + file: true, + dir: false, + symlink: false, + }, + len: data.len() as u64, + permissions: perms.clone(), + modified: SystemTime::now(), + }; + let a = Self::put_action(&mut s, &path, Image::memory(data.clone()), perms); + let (accepted, result) = self.submit(&mut s, a); + if !accepted { + return result; + } + let object = if let Some(o) = object { + *lock(&o) = Object::memory(data.clone(), meta); + o + } else { + Arc::new(Mutex::new(Object::memory(data.clone(), meta))) + }; + Self::entry(&mut s, path.clone(), Some(object)); + self.rebase(&mut s); + result + } + fn prune(s: &mut State) { + s.entries + .retain(|entry| s.pending.iter().any(|p| p.action.touches(&entry.path))); + s.objects.retain(|w| w.strong_count() > 0); + s.redirects + .retain(|(path, _)| s.pending.iter().any(|p| p.action.touches(path))); + } + fn recover_before(&self, s: &mut State) -> io::Result<()> { + let report = self.recover_locked(s); + self.rebase(s); + Self::prune(s); + match report.blocked { + Some(e) if !capacity(&e) => Err(e), + _ => Ok(()), + } + } + pub fn open>(&self, path: P) -> io::Result { + OpenOptions::new().read(true).open_in(self, path) + } + pub fn create>(&self, path: P) -> io::Result { + OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open_in(self, path) + } + pub fn read_link>(&self, path: P) -> io::Result { + let p = self.norm(path.as_ref())?; + let s = lock(&self.service.state); + self.secure_path(&s, &p, true)?; + if !self.lookup(&s, &p)?.file_type().is_symlink() { + return Err(error(io::ErrorKind::InvalidInput, "not a symlink")); + } + self.service.backend.read_link(&p) + } + pub fn canonicalize>(&self, path: P) -> io::Result { + let p = self.norm(path.as_ref())?; + match self.service.backend.canonicalize(&p) { + Ok(p) => Ok(p), + Err(e) if e.kind() == io::ErrorKind::NotFound => { + self.metadata(&p)?; + Ok(p) + } + Err(e) => Err(e), + } + } +} +#[derive(Clone)] +pub struct DirEntry { + fs: Fs, + path: PathBuf, + key: PathBuf, +} +impl DirEntry { + pub fn path(&self) -> PathBuf { + self.path.clone() + } + pub fn file_name(&self) -> OsString { + self.path.file_name().unwrap_or_default().to_os_string() + } + pub fn metadata(&self) -> io::Result { + self.fs.symlink_metadata(&self.key) + } + pub fn file_type(&self) -> io::Result { + Ok(self.metadata()?.file_type()) + } +} +pub struct ReadDir { + entries: std::vec::IntoIter>, +} +impl Iterator for ReadDir { + type Item = io::Result; + fn next(&mut self) -> Option { + self.entries.next() + } +} +impl Fs { + fn list(&self, s: &State, path: &Path) -> io::Result> { + if !self.lookup(s, path)?.is_dir() { + return Err(error(io::ErrorKind::NotADirectory, "not a directory")); + } + let mut paths = Vec::new(); + match self.service.backend.read_dir(&Self::disk_path(s, path)) { + Ok(entries) => { + paths + .try_reserve(entries.len()) + .map_err(|_| allocation_oom())?; + for e in entries { + let e = DiskEntry { + path: path.join(e.file_name), + file_name: OsString::new(), + }; + if s.pending + .iter() + .any(|p| matches!(&p.action,Action::Put{temp,..} if *temp==e.path)) + { + continue; + } + match self.lookup(s, &e.path) { + Ok(_) => paths.push(e.path), + Err(e) if e.kind() == io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } + } + } + Err(e) + if e.kind() == io::ErrorKind::NotFound + && s.entries + .iter() + .any(|e| e.path == path && e.object.is_some()) => {} + Err(e) => return Err(e), + } + for e in &s.entries { + if e.path.parent() == Some(path) + && e.object.is_some() + && s.pending.iter().any(|p| p.action.touches(&e.path)) + && !paths.contains(&e.path) + { + paths.try_reserve(1).map_err(|_| allocation_oom())?; + paths.push(e.path.clone()); + } + } + paths.sort(); + Ok(paths) + } + pub fn read_dir>(&self, path: P) -> io::Result { + let p = self.norm(path.as_ref())?; + let s = lock(&self.service.state); + self.secure_path(&s, &p, false)?; + let paths = self.list(&s, &p)?; + let mut entries = Vec::new(); + entries + .try_reserve_exact(paths.len()) + .map_err(|_| allocation_oom())?; + for key in paths { + let display = path.as_ref().join(key.file_name().unwrap_or_default()); + entries.push(Ok(DirEntry { + fs: self.clone(), + path: display, + key, + })); + } + Ok(ReadDir { + entries: entries.into_iter(), + }) + } + pub fn create_dir>(&self, path: P) -> io::Result<()> { + self.mkdir(path.as_ref(), false) + } + fn mkdir(&self, path: &Path, private: bool) -> io::Result<()> { + let p = self.norm(path)?; + let mut s = lock(&self.service.state); + self.recover_before(&mut s)?; + self.authority(&p)?; + self.secure_path(&s, &p, false)?; + self.parent(&s, &p)?; + match self.lookup(&s, &p) { + Ok(_) => return Err(error(io::ErrorKind::AlreadyExists, "path exists")), + Err(e) if e.kind() == io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } + Self::prepare(&mut s, 1)?; + let meta = Metadata { + identity: next_identity(), + disk_identity: None, + disk: None, + kind: FileType { + file: false, + dir: true, + symlink: false, + }, + len: 0, + permissions: self.new_permissions(&s, &p, private, true)?, + modified: SystemTime::now(), + }; + let (accepted, result) = self.submit( + &mut s, + Action::Mkdir { + path: p.clone(), + private, + stage: 0, + }, + ); + if !accepted { + return result; + } + Self::entry( + &mut s, + p.clone(), + Some(Arc::new(Mutex::new(Object::memory( + Arc::new(Zeroizing::new(Vec::new())), + meta, + )))), + ); + result + } + pub fn create_dir_all>(&self, path: P) -> io::Result<()> { + self.mkdir_all(path.as_ref(), false) + } + pub fn create_private_dir_all>(&self, path: P) -> io::Result<()> { + self.mkdir_all(path.as_ref(), true) + } + fn mkdir_all(&self, path: &Path, private: bool) -> io::Result<()> { + let p = self.norm(path)?; + let mut cur = PathBuf::new(); + for c in p.components() { + cur.push(c.as_os_str()); + match self.metadata(&cur) { + Ok(m) if m.is_dir() => {} + Ok(_) => { + return Err(error( + io::ErrorKind::NotADirectory, + "ancestor is not a directory", + )); + } + Err(e) if e.kind() == io::ErrorKind::NotFound => match self.mkdir(&cur, private) { + Err(e) + if e.kind() == io::ErrorKind::AlreadyExists + && self.metadata(&cur)?.is_dir() => {} + r => r?, + }, + Err(e) => return Err(e), + } + } + #[cfg(unix)] + if private { + self.set_permissions(&p, permissions(true, true))?; + } + Ok(()) + } + pub fn remove_file>(&self, path: P) -> io::Result<()> { + self.unlink(path.as_ref(), false) + } + pub fn remove_dir>(&self, path: P) -> io::Result<()> { + self.unlink(path.as_ref(), true) + } + fn unlink(&self, path: &Path, dir: bool) -> io::Result<()> { + let p = self.norm(path)?; + let mut s = lock(&self.service.state); + self.recover_before(&mut s)?; + self.authority(&p)?; + self.secure_path(&s, &p, false)?; + let m = self.lookup(&s, &p)?; + if m.is_dir() != dir { + return Err(error(io::ErrorKind::InvalidInput, "incorrect removal type")); + } + if dir && !self.list(&s, &p)?.is_empty() { + return Err(error( + io::ErrorKind::DirectoryNotEmpty, + "directory not empty", + )); + } + Self::prepare(&mut s, 1)?; + let (accepted, result) = self.submit( + &mut s, + Action::Unlink { + path: p.clone(), + dir, + stage: 0, + }, + ); + if accepted { + for object in s.objects.iter().filter_map(|w| w.upgrade()) { + let mut object = lock(&object); + if object.path.as_ref() == Some(&p) { + object.path = None; + } + } + Self::entry(&mut s, p, None); + } + result + } + pub fn remove_dir_all>(&self, path: P) -> io::Result<()> { + let p = self.norm(path.as_ref())?; + for e in self.read_dir(&p)? { + let e = e?; + if e.file_type()?.is_dir() { + self.remove_dir_all(e.path())?; + } else { + self.remove_file(e.path())?; + } + } + self.remove_dir(p) + } + pub fn rename, Q: AsRef>(&self, from: P, to: Q) -> io::Result<()> { + let from = self.norm(from.as_ref())?; + let to = self.norm(to.as_ref())?; + let mut s = lock(&self.service.state); + self.recover_before(&mut s)?; + self.authority(&from)?; + self.authority(&to)?; + self.secure_path(&s, &from, false)?; + self.secure_path(&s, &to, false)?; + let meta = self.lookup(&s, &from)?; + if from == to { + return Ok(()); + } + if to.starts_with(&from) { + return Err(error( + io::ErrorKind::InvalidInput, + "rename into own subtree", + )); + } + self.parent(&s, &to)?; + match self.lookup(&s, &to) { + Ok(dest) => { + if dest.is_dir() != meta.is_dir() { + return Err(error(io::ErrorKind::InvalidInput, "rename type mismatch")); + } + if dest.is_dir() && !self.list(&s, &to)?.is_empty() { + return Err(error( + io::ErrorKind::DirectoryNotEmpty, + "destination not empty", + )); + } + } + Err(e) if e.kind() == io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } + let count = s + .entries + .iter() + .filter(|e| e.path.starts_with(&from)) + .count(); + Self::prepare(&mut s, count + 2)?; + let source = Self::disk_path(&s, &from); + // Capture only a descriptor and metadata, never directory descendants or + // file payloads. A redirect merges unchanged descendants during fallback. + let root = if meta.is_file() { + self.object(&mut s, &from)? + } else { + Arc::new(Mutex::new(Object::memory( + Arc::new(Zeroizing::new(Vec::new())), + meta, + ))) + }; + let mut moved = Vec::new(); + moved + .try_reserve_exact(count + 1) + .map_err(|_| allocation_oom())?; + moved.push((to.clone(), Some(root.clone()))); + for e in &s.entries { + if e.path.starts_with(&from) && e.path != from { + moved.push(( + to.join(e.path.strip_prefix(&from).unwrap()), + e.object.clone(), + )); + } + } + let (accepted, result) = self.submit( + &mut s, + Action::Rename { + from: from.clone(), + to: to.clone(), + stage: 0, + }, + ); + if accepted { + for object in s.objects.iter().filter_map(|w| w.upgrade()) { + let mut object = lock(&object); + if let Some(path) = &object.path { + if path.starts_with(&from) { + object.path = Some(if *path == from { + to.clone() + } else { + to.join(path.strip_prefix(&from).unwrap()) + }); + } else if path.starts_with(&to) { + object.path = None; + } + } + } + for e in &mut s.entries { + if e.path.starts_with(&from) { + e.object = None; + } + } + Self::entry(&mut s, from.clone(), None); + for (path, object) in moved { + Self::entry(&mut s, path, object); + } + s.redirects + .retain(|(path, _)| !path.starts_with(&from) && !path.starts_with(&to)); + if s.pending.iter().any(|p| p.action.touches(&to)) { + s.redirects.push((to, source)); + } + } + result + } + pub fn copy, Q: AsRef>(&self, from: P, to: Q) -> io::Result { + let data = Zeroizing::new(self.read(&from)?); + let p = self.metadata(&from)?.permissions(); + self.write(&to, &*data)?; + self.set_permissions(to, p)?; + Ok(data.len() as u64) + } + pub fn set_permissions>( + &self, + path: P, + permissions: Permissions, + ) -> io::Result<()> { + let p = self.norm(path.as_ref())?; + let mut s = lock(&self.service.state); + self.recover_before(&mut s)?; + self.authority(&p)?; + self.secure_path(&s, &p, false)?; + self.capture_shallow(&mut s, &p)?; + Self::prepare(&mut s, 0)?; + let (accepted, result) = self.submit( + &mut s, + Action::Chmod { + path: p.clone(), + permissions: permissions.clone(), + }, + ); + if !accepted { + return result; + } + if let Some(o) = s + .entries + .iter() + .find(|e| e.path == p) + .and_then(|e| e.object.as_ref()) + { + let mut o = lock(o); + o.meta.permissions = permissions.clone(); + o.meta.disk = None; + } + result + } + fn capture_shallow(&self, s: &mut State, p: &Path) -> io::Result<()> { + let meta = self.lookup(s, p)?; + if meta.is_file() { + self.object(s, p)?; + } else if meta.is_dir() && !s.entries.iter().any(|e| e.path == p) { + Self::prepare(s, 1)?; + Self::entry( + s, + p.to_path_buf(), + Some(Arc::new(Mutex::new(Object::memory( + Arc::new(Zeroizing::new(Vec::new())), + meta, + )))), + ); + } + Ok(()) + } + pub fn sync_directory>(&self, path: P) -> io::Result<()> { + let p = self.norm(path.as_ref())?; + let mut s = lock(&self.service.state); + self.recover_before(&mut s)?; + self.authority(&p)?; + self.secure_path(&s, &p, false)?; + if !self.lookup(&s, &p)?.is_dir() { + return Err(error(io::ErrorKind::NotADirectory, "not a directory")); + } + Self::prepare(&mut s, 0)?; + self.submit(&mut s, Action::Sync { path: p }).1 + } + pub fn open_beneath, Q: AsRef>( + &self, + root: P, + relative: Q, + ) -> io::Result { + if relative.as_ref().as_os_str().is_empty() + || relative + .as_ref() + .components() + .any(|c| !matches!(c, Component::Normal(_))) + { + return Err(error( + io::ErrorKind::PermissionDenied, + "relative path must contain normal components only", + )); + } + let root = self.norm(root.as_ref())?; + let p = root.join(relative); + let mut s = lock(&self.service.state); + let _ = self.recover_locked(&mut s); + self.rebase(&mut s); + Self::prune(&mut s); + self.secure_path(&s, &p, false)?; + if s.entries.iter().any(|e| e.path == p) { + let object = self.object(&mut s, &p)?; + return Ok(File { + fs: self.clone(), + object, + cursor: Arc::new(Mutex::new(0)), + read: true, + write: false, + append: false, + }); + } + let native = self + .service + .backend + .open_beneath(&Self::disk_path(&s, &root), p.strip_prefix(&root).unwrap())?; + if !native.metadata()?.is_file() { + return Err(error(io::ErrorKind::InvalidInput, "not a regular file")); + } + let object = Arc::new(Mutex::new(Object::native(native, p)?)); + s.objects.try_reserve(1).map_err(|_| allocation_oom())?; + s.objects.push(Arc::downgrade(&object)); + + Ok(File { + fs: self.clone(), + object, + cursor: Arc::new(Mutex::new(0)), + read: true, + write: false, + append: false, + }) + } +} +#[derive(Clone, Default, Debug)] +pub struct OpenOptions { + options: DiskOpenOptions, +} +impl OpenOptions { + pub fn new() -> Self { + Self::default() + } + pub fn read(&mut self, v: bool) -> &mut Self { + self.options.read = v; + self + } + pub fn write(&mut self, v: bool) -> &mut Self { + self.options.write = v; + self + } + pub fn append(&mut self, v: bool) -> &mut Self { + self.options.append = v; + self + } + pub fn truncate(&mut self, v: bool) -> &mut Self { + self.options.truncate = v; + self + } + pub fn create(&mut self, v: bool) -> &mut Self { + self.options.create = v; + self + } + pub fn create_new(&mut self, v: bool) -> &mut Self { + self.options.create_new = v; + self + } + pub fn private(&mut self, v: bool) -> &mut Self { + self.options.private = v; + self + } + pub fn open>(&self, path: P) -> io::Result { + self.open_in(global(), path) + } + pub fn open_in>(&self, fs: &Fs, path: P) -> io::Result { + let p = fs.norm(path.as_ref())?; + let o = &self.options; + let writable = o.write || o.append; + if (o.create_new || o.create || o.truncate || !o.read) && !writable + || o.truncate && o.append && !o.create_new + { + return Err(error(io::ErrorKind::InvalidInput, "invalid open options")); + } + let mut s = lock(&fs.service.state); + if writable { + fs.recover_before(&mut s)?; + } else { + let _ = fs.recover_locked(&mut s); + fs.rebase(&mut s); + Fs::prune(&mut s); + } + fs.secure_path(&s, &p, false)?; + let exists = match fs.lookup(&s, &p) { + Ok(m) => { + if !m.is_file() { + return Err(error(io::ErrorKind::InvalidInput, "not a regular file")); + } + true + } + Err(e) if e.kind() == io::ErrorKind::NotFound => false, + Err(e) => return Err(e), + }; + if exists && o.create_new { + return Err(error(io::ErrorKind::AlreadyExists, "path exists")); + } + if !exists && !o.create && !o.create_new { + return Err(error(io::ErrorKind::NotFound, "path does not exist")); + } + if writable { + fs.preflight(&s, &p)?; + } + if !exists || o.truncate { + s.entries.try_reserve(1).map_err(|_| allocation_oom())?; + s.objects.try_reserve(1).map_err(|_| allocation_oom())?; + let perms = if o.private { + fs.new_permissions(&s, &p, true, false)? + } else { + fs.lookup(&s, &p).map(|m| m.permissions()).map_or_else( + |e| { + if e.kind() == io::ErrorKind::NotFound { + fs.new_permissions(&s, &p, false, false) + } else { + Err(e) + } + }, + Ok, + )? + }; + let data = Arc::new(Zeroizing::new(Vec::new())); + let meta = Metadata { + identity: next_identity(), + disk_identity: None, + disk: None, + kind: FileType { + file: true, + dir: false, + symlink: false, + }, + len: 0, + permissions: perms.clone(), + modified: SystemTime::now(), + }; + let obj = s + .entries + .iter() + .find(|e| e.path == p) + .and_then(|e| e.object.clone()); + let a = Fs::put_action(&mut s, &p, Image::memory(data.clone()), perms); + let (accepted, result) = fs.submit(&mut s, a); + if !accepted { + result?; + unreachable!(); + } + let object = if let Some(obj) = obj { + *lock(&obj) = Object::memory(data.clone(), meta); + obj + } else { + Arc::new(Mutex::new(Object::memory(data.clone(), meta))) + }; + Fs::entry(&mut s, p.clone(), Some(object)); + result?; + } + fs.rebase(&mut s); + let object = fs.object(&mut s, &p)?; + Ok(File { + fs: fs.clone(), + object, + cursor: Arc::new(Mutex::new(0)), + read: o.read, + write: writable, + append: o.append, + }) + } +} +pub struct File { + fs: Fs, + object: Obj, + cursor: Arc>, + read: bool, + write: bool, + append: bool, +} +impl File { + pub fn open>(path: P) -> io::Result { + global().open(path) + } + pub fn create>(path: P) -> io::Result { + global().create(path) + } + pub fn open_in>(fs: &Fs, path: P) -> io::Result { + fs.open(path) + } + pub fn create_in>(fs: &Fs, path: P) -> io::Result { + fs.create(path) + } + pub fn try_clone(&self) -> io::Result { + Ok(Self { + fs: self.fs.clone(), + object: self.object.clone(), + cursor: self.cursor.clone(), + read: self.read, + write: self.write, + append: self.append, + }) + } + pub fn metadata(&self) -> io::Result { + let object = lock(&self.object); + if object.image.patches.is_empty() + && let Source::Native(file) = &object.image.source + { + let file = lock(file); + let mut meta = Metadata::disk(file.metadata()?, file.identity()?); + meta.identity = object.meta.identity; + Ok(meta) + } else { + Ok(object.meta.clone()) + } + } + fn mutate(&self, offset: Option, data: &[u8], size: Option) -> io::Result { + if !self.write { + return Err(error( + io::ErrorKind::PermissionDenied, + "handle is not writable", + )); + } + if data.is_empty() && size.is_none() { + return Ok(0); + } + let mut s = lock(&self.fs.service.state); + self.fs.recover_before(&mut s)?; + let mut cursor = lock(&self.cursor); + let mut object = lock(&self.object); + if !object.dirty + && let Source::Native(native) = &object.image.source + { + let (meta, disk_identity) = { + let native = lock(native); + (native.metadata()?, native.identity()?) + }; + if disk_identity.is_none() { + return Err(error( + io::ErrorKind::PermissionDenied, + "native file identity unavailable", + )); + } + if let Some(path) = &object.path + && !s.pending.iter().any(|p| p.action.touches(path)) + { + match self.fs.service.backend.identity(path, false) { + Ok(named) if same_disk_identity(disk_identity, named) => {} + Ok(None) => { + return Err(error( + io::ErrorKind::PermissionDenied, + "native identity unavailable", + )); + } + Ok(_) => object.path = None, + Err(e) if e.kind() == io::ErrorKind::NotFound => object.path = None, + Err(e) => return Err(e), + } + } + object.image.base_len = meta.len(); + object.image.len = meta.len(); + let identity = object.meta.identity; + object.meta = Metadata::disk(meta, disk_identity); + object.meta.identity = identity; + } + let path = object.path.clone(); + let start = if self.append && size.is_none() { + object.image.len + } else { + offset.unwrap_or(*cursor) + }; + let end = start + .checked_add(data.len() as u64) + .ok_or_else(allocation_oom)?; + let len = size.unwrap_or(end.max(object.image.len)); + let image = object.image.patched(start, data, len)?; + let perms = object.meta.permissions(); + drop(object); + if let Some(path) = &path { + self.fs.preflight(&s, path)?; + } else if let Some(lease) = &self.fs.lease { + lease.check()?; + } + let (accepted, result) = if let Some(path) = &path { + s.entries.try_reserve(1).map_err(|_| allocation_oom())?; + s.objects.try_reserve(1).map_err(|_| allocation_oom())?; + let action = Fs::put_action(&mut s, path, image.clone(), perms); + self.fs.submit(&mut s, action) + } else { + self.fs.reserve(&mut s, image.payload_bytes(), 0)?; + (true, Ok(())) + }; + if !accepted { + result?; + unreachable!(); + } + { + let mut object = lock(&self.object); + object.image = image; + object.meta.len = len; + object.meta.modified = SystemTime::now(); + object.meta.disk = None; + if object.meta.identity == 0 { + object.meta.identity = next_identity(); + } + object.dirty = true; + } + if let Some(path) = path { + Fs::entry(&mut s, path, Some(self.object.clone())); + } + if size.is_none() { + *cursor = end; + } + self.fs.rebase(&mut s); + result?; + Ok(data.len()) + } + pub fn set_len(&self, size: u64) -> io::Result<()> { + self.mutate(None, &[], Some(size)).map(|_| ()) + } + pub fn sync_data(&self) -> io::Result<()> { + let r = self.fs.recover(); + match r.blocked { + Some(e) if !capacity(&e) => Err(e), + _ => Ok(()), + } + } + pub fn sync_all(&self) -> io::Result<()> { + self.sync_data() + } + pub fn set_permissions(&self, p: Permissions) -> io::Result<()> { + let mut s = lock(&self.fs.service.state); + self.fs.recover_before(&mut s)?; + let path = lock(&self.object).path.clone(); + let (accepted, result) = if let Some(path) = path { + self.fs.authority(&path)?; + self.fs.secure_path(&s, &path, false)?; + Fs::prepare(&mut s, 0)?; + self.fs.enqueue( + &mut s, + Action::Chmod { + path, + permissions: p.clone(), + }, + ) + } else { + if let Some(lease) = &self.fs.lease { + lease.check()?; + } + (true, Ok(())) + }; + if accepted { + let mut object = lock(&self.object); + object.meta.permissions = p; + object.meta.disk = None; + } + result + } +} +impl Read for File { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if !self.read { + return Err(error( + io::ErrorKind::PermissionDenied, + "handle is not readable", + )); + } + let mut cursor = lock(&self.cursor); + let image = lock(&self.object).image.clone(); + let n = if image.patches.is_empty() + && let Source::Native(file) = &image.source + { + let mut file = lock(file); + file.seek(SeekFrom::Start(*cursor))?; + file.read(buf)? + } else { + image.read_at(*cursor, buf)? + }; + *cursor += n as u64; + Ok(n) + } +} +impl Write for File { + fn write(&mut self, data: &[u8]) -> io::Result { + self.mutate(None, data, None) + } + fn flush(&mut self) -> io::Result<()> { + self.sync_data() + } +} +impl Seek for File { + fn seek(&mut self, from: SeekFrom) -> io::Result { + let mut cursor = lock(&self.cursor); + let next = match from { + SeekFrom::Start(n) => n as i128, + SeekFrom::Current(n) => *cursor as i128 + n as i128, + SeekFrom::End(n) => self.metadata()?.len() as i128 + n as i128, + }; + if !(0..=u64::MAX as i128).contains(&next) { + return Err(error(io::ErrorKind::InvalidInput, "invalid seek")); + } + *cursor = next as u64; + Ok(*cursor) + } +} +macro_rules! forward {($($name:ident -> $out:ty;)+)=>{$(pub fn $name>(path:P)->io::Result<$out>{global().$name(path)})+};} +forward! {read->Vec;read_to_string->String;create_dir->();create_dir_all->();create_private_dir_all->();remove_file->();remove_dir->();remove_dir_all->();metadata->Metadata;symlink_metadata->Metadata;read_dir->ReadDir;read_link->PathBuf;canonicalize->PathBuf;try_exists->bool;sync_directory->();require_disk->();} +pub fn write, C: AsRef<[u8]>>(path: P, contents: C) -> io::Result<()> { + global().write(path, contents) +} +pub fn replace>(path: P, contents: &[u8]) -> io::Result<()> { + global().replace(path, contents) +} +pub fn replace_private>(path: P, contents: &[u8]) -> io::Result<()> { + global().replace_private(path, contents) +} +pub fn set_permissions>(path: P, p: Permissions) -> io::Result<()> { + global().set_permissions(path, p) +} +pub fn rename, Q: AsRef>(from: P, to: Q) -> io::Result<()> { + global().rename(from, to) +} +pub fn copy, Q: AsRef>(from: P, to: Q) -> io::Result { + global().copy(from, to) +} +pub fn open_beneath, Q: AsRef>(root: P, relative: Q) -> io::Result { + global().open_beneath(root, relative) +} + +#[cfg(test)] +mod tests; + +#[derive(Default, Debug)] +pub struct DirBuilder { + mode: Option, +} +impl DirBuilder { + pub fn new() -> Self { + Self::default() + } + pub fn mode(&mut self, mode: u32) -> &mut Self { + self.mode = Some(mode); + self + } + pub fn create>(&self, path: P) -> io::Result<()> { + match self.mode { + Some(0o700) => global().mkdir(path.as_ref(), true), + None => global().create_dir(path), + Some(_) => Err(error( + io::ErrorKind::Unsupported, + "only private directory mode 0700 is supported", + )), + } + } +} +impl std::fmt::Debug for File { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("File") + .field("read", &self.read) + .field("write", &self.write) + .finish_non_exhaustive() + } +} +impl std::fmt::Debug for Fs { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Fs").finish_non_exhaustive() + } +} diff --git a/src/resilient_fs/tests.rs b/src/resilient_fs/tests.rs new file mode 100644 index 00000000..58c83cb9 --- /dev/null +++ b/src/resilient_fs/tests.rs @@ -0,0 +1,1037 @@ +//! Black-box fault injection around the production disk backend. +use super::*; +use std::fs as native; +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Point { + Open, + Write, + Sync, + Rename, + Mkdir, + DirectorySync, +} +#[derive(Default)] +struct Faults(Mutex>); +impl Faults { + fn arm(&self, point: Point, code: i32) { + *self.0.lock().unwrap() = Some((point, code)); + } + fn clear(&self) { + *self.0.lock().unwrap() = None; + } + fn check(&self, point: Point) -> io::Result<()> { + match *self.0.lock().unwrap() { + Some((p, code)) if p == point => Err(io::Error::from_raw_os_error(code)), + _ => Ok(()), + } + } +} +struct Injected { + disk: DiskBackend, + faults: Arc, +} +struct InjectedFile { + disk: Box, + faults: Arc, + wrote_prefix: bool, +} +impl Read for InjectedFile { + fn read(&mut self, b: &mut [u8]) -> io::Result { + self.disk.read(b) + } +} +impl Seek for InjectedFile { + fn seek(&mut self, p: SeekFrom) -> io::Result { + self.disk.seek(p) + } +} +impl Write for InjectedFile { + fn write(&mut self, b: &[u8]) -> io::Result { + if self.faults.check(Point::Write).is_err() && !b.is_empty() { + if self.wrote_prefix { + self.faults.check(Point::Write)?; + } + self.wrote_prefix = true; + return self.disk.write(&b[..b.len().min(3)]); + } + self.disk.write(b) + } + fn flush(&mut self) -> io::Result<()> { + self.disk.flush() + } +} +impl BackendFile for InjectedFile { + fn identity(&self) -> io::Result> { + self.disk.identity() + } + fn metadata(&self) -> io::Result { + self.disk.metadata() + } + fn set_len(&self, n: u64) -> io::Result<()> { + self.disk.set_len(n) + } + fn sync_data(&self) -> io::Result<()> { + self.faults.check(Point::Sync)?; + self.disk.sync_data() + } + fn sync_all(&self) -> io::Result<()> { + self.faults.check(Point::Sync)?; + self.disk.sync_all() + } + fn set_permissions(&self, p: Permissions) -> io::Result<()> { + self.disk.set_permissions(p) + } +} +impl Backend for Injected { + fn identity(&self, path: &Path, follow: bool) -> io::Result> { + self.disk.identity(path, follow) + } + fn open(&self, p: &Path, o: &DiskOpenOptions) -> io::Result> { + if o.write || o.append { + self.faults.check(Point::Open)?; + } + Ok(Box::new(InjectedFile { + disk: self.disk.open(p, o)?, + faults: self.faults.clone(), + wrote_prefix: false, + })) + } + fn metadata(&self, p: &Path, follow: bool) -> io::Result { + self.disk.metadata(p, follow) + } + fn read_dir(&self, p: &Path) -> io::Result> { + self.disk.read_dir(p) + } + fn read_link(&self, p: &Path) -> io::Result { + self.disk.read_link(p) + } + fn canonicalize(&self, p: &Path) -> io::Result { + self.disk.canonicalize(p) + } + fn create_dir(&self, p: &Path, private: bool) -> io::Result<()> { + self.faults.check(Point::Mkdir)?; + self.disk.create_dir(p, private) + } + fn remove_file(&self, p: &Path) -> io::Result<()> { + self.disk.remove_file(p) + } + fn remove_dir(&self, p: &Path) -> io::Result<()> { + self.disk.remove_dir(p) + } + fn rename(&self, a: &Path, b: &Path) -> io::Result<()> { + self.faults.check(Point::Rename)?; + self.disk.rename(a, b) + } + fn set_permissions(&self, p: &Path, mode: Permissions) -> io::Result<()> { + self.disk.set_permissions(p, mode) + } + fn sync_directory(&self, p: &Path) -> io::Result<()> { + self.faults.check(Point::DirectorySync)?; + self.disk.sync_directory(p) + } + fn acquire_lease(&self, r: &LeaseRequest) -> io::Result> { + self.disk.acquire_lease(r) + } + fn open_beneath(&self, root: &Path, p: &Path) -> io::Result> { + self.disk.open_beneath(root, p) + } +} +struct Fixture { + root: PathBuf, + fs: Fs, + faults: Arc, +} +impl Fixture { + fn new() -> Self { + Self::budget(1024 * 1024, 4096) + } + fn budget(bytes: usize, ops: usize) -> Self { + static NEXT: AtomicUsize = AtomicUsize::new(0); + let root = std::env::temp_dir().join(format!( + "kit-resilient-test-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + native::create_dir(&root).unwrap(); + let faults = Arc::new(Faults::default()); + let fs = Fs::with_budget( + Arc::new(Injected { + disk: DiskBackend, + faults: faults.clone(), + }), + bytes, + ops, + ); + Self { root, fs, faults } + } + fn path(&self, name: &str) -> PathBuf { + self.root.join(name) + } + fn settle(&self) { + self.faults.clear(); + self.fs.require_disk(&self.root).unwrap(); + let report = self.fs.recover(); + assert_eq!(report.remaining_operations, 0); + assert!(report.blocked.is_none()); + } +} +impl Drop for Fixture { + fn drop(&mut self) { + let _ = native::remove_dir_all(&self.root); + } +} +fn names(fs: &Fs, p: &Path) -> Vec { + let mut names: Vec<_> = fs + .read_dir(p) + .unwrap() + .map(|e| e.unwrap().file_name()) + .collect(); + names.sort(); + names +} + +#[test] +#[cfg(unix)] +fn capacity_at_each_commit_stage_is_accepted_but_not_durable() { + for code in [libc::ENOSPC, libc::EDQUOT] { + for point in [ + Point::Open, + Point::Write, + Point::Sync, + Point::Rename, + Point::DirectorySync, + ] { + let t = Fixture::new(); + let p = t.path("value"); + native::write(&p, b"old disk content").unwrap(); + t.faults.arm(point, code); + t.fs.replace(&p, b"complete replacement bytes").unwrap(); + assert_eq!( + t.fs.read(&p).unwrap(), + b"complete replacement bytes", + "{point:?}" + ); + for _ in 0..3 { + let r = t.fs.recover(); + assert!(r.remaining_operations > 0, "{point:?}"); + assert_eq!(r.blocked.unwrap().raw_os_error(), Some(code)); + assert_eq!( + t.fs.require_disk(&p).unwrap_err().raw_os_error(), + Some(code) + ); + } + t.settle(); + assert_eq!(native::read(&p).unwrap(), b"complete replacement bytes"); + let disk_names: Vec<_> = native::read_dir(&t.root) + .unwrap() + .map(|e| e.unwrap().file_name()) + .collect(); + assert_eq!( + disk_names, + vec![OsString::from("value")], + "temporary file leaked at {point:?}" + ); + } + } +} + +#[test] +#[cfg(unix)] +fn handles_close_reopen_append_seek_clone_and_truncate_during_outage() { + let t = Fixture::new(); + let p = t.path("log"); + t.faults.arm(Point::Open, libc::ENOSPC); + let mut f = OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open_in(&t.fs, &p) + .unwrap(); + f.write_all(b"abcdef").unwrap(); + f.sync_data().unwrap(); + f.seek(SeekFrom::Start(2)).unwrap(); + let mut clone = f.try_clone().unwrap(); + clone.write_all(b"XY").unwrap(); + assert_eq!(f.stream_position().unwrap(), 4); + drop(clone); + drop(f); + assert_eq!(t.fs.read(&p).unwrap(), b"abXYef"); + let mut append = OpenOptions::new().append(true).open_in(&t.fs, &p).unwrap(); + append.seek(SeekFrom::Start(0)).unwrap(); + append.write_all(b"!").unwrap(); + append.sync_all().unwrap(); + drop(append); + assert_eq!(t.fs.read(&p).unwrap(), b"abXYef!"); + let f = OpenOptions::new().write(true).open_in(&t.fs, &p).unwrap(); + f.set_len(4).unwrap(); + f.set_len(6).unwrap(); + drop(f); + assert_eq!(t.fs.read(&p).unwrap(), b"abXY\0\0"); + t.settle(); + assert_eq!(native::read(&p).unwrap(), b"abXY\0\0"); + let mut f = OpenOptions::new() + .write(true) + .truncate(true) + .open_in(&t.fs, &p) + .unwrap(); + f.write_all(b"new").unwrap(); + drop(f); + t.settle(); + assert_eq!(native::read(&p).unwrap(), b"new"); +} + +#[test] +#[cfg(unix)] +fn overlay_directory_tree_rename_delete_and_recreate() { + let t = Fixture::new(); + for code in [libc::ENOSPC, libc::EDQUOT] { + t.faults.arm(Point::Mkdir, code); + let a = t.path("a"); + let b = t.path("b"); + t.fs.create_dir_all(a.join("nested")).unwrap(); + t.fs.write(a.join("nested/one"), b"one").unwrap(); + t.fs.write(a.join("two"), b"two").unwrap(); + assert!(!a.exists()); + assert!(t.fs.metadata(&a).unwrap().is_dir()); + assert_eq!( + names(&t.fs, &a), + vec![OsString::from("nested"), OsString::from("two")] + ); + t.fs.rename(&a, &b).unwrap(); + assert!(!t.fs.try_exists(&a).unwrap()); + assert_eq!(t.fs.read(b.join("nested/one")).unwrap(), b"one"); + t.fs.remove_file(b.join("two")).unwrap(); + t.fs.write(b.join("two"), b"reborn").unwrap(); + t.fs.remove_dir_all(b.join("nested")).unwrap(); + t.fs.create_dir(b.join("nested")).unwrap(); + t.fs.write(b.join("nested/new"), b"new").unwrap(); + t.settle(); + assert!(!a.exists()); + assert_eq!(native::read(b.join("two")).unwrap(), b"reborn"); + assert!(!b.join("nested/one").exists()); + assert_eq!(native::read(b.join("nested/new")).unwrap(), b"new"); + t.fs.remove_dir_all(&b).unwrap(); + t.settle(); + assert!(!b.exists()); + } +} + +#[test] +#[cfg(unix)] +fn noncapacity_errors_are_not_reported_as_success() { + for code in [libc::EACCES, libc::EIO, libc::EROFS] { + for point in [ + Point::Open, + Point::Write, + Point::Sync, + Point::Rename, + Point::DirectorySync, + Point::Mkdir, + ] { + let t = Fixture::new(); + t.faults.arm(point, code); + let result = if point == Point::Mkdir { + t.fs.create_dir(t.path("dir")) + } else { + t.fs.replace(t.path("file"), b"long enough for partial write") + }; + if point == Point::DirectorySync { + assert!(result.is_ok()); + assert!(t.fs.recover().blocked.is_some()); + } else { + assert_eq!(result.unwrap_err().raw_os_error(), Some(code), "{point:?}"); + } + } + } +} + +#[test] +#[cfg(unix)] +fn private_modes_survive_fallback_and_recovery() { + use std::os::unix::fs::PermissionsExt; + let t = Fixture::new(); + t.faults.arm(Point::Mkdir, libc::EDQUOT); + let dir = t.path("private"); + let p = dir.join("secret"); + t.fs.create_private_dir_all(&dir).unwrap(); + t.fs.replace_private(&p, b"secret").unwrap(); + assert_eq!( + t.fs.metadata(&dir).unwrap().permissions().mode() & 0o777, + 0o700 + ); + assert_eq!( + t.fs.metadata(&p).unwrap().permissions().mode() & 0o777, + 0o600 + ); + t.settle(); + assert_eq!( + native::metadata(&dir).unwrap().permissions().mode() & 0o777, + 0o700 + ); + assert_eq!( + native::metadata(&p).unwrap().permissions().mode() & 0o777, + 0o600 + ); + native::set_permissions(&p, Permissions::from_mode(0o640)).unwrap(); + let fs = Fs::new(Arc::new(DiskBackend)); + fs.replace(&p, b"updated").unwrap(); + fs.require_disk(&p).unwrap(); + assert_eq!( + native::metadata(&p).unwrap().permissions().mode() & 0o777, + 0o640 + ); +} + +#[test] +#[cfg(unix)] +fn budgets_reject_without_losing_previously_accepted_data() { + for (bytes, ops) in [(16, 100), (1024, 1)] { + let t = Fixture::budget(bytes, ops); + t.faults.arm(Point::Open, libc::ENOSPC); + let p = t.path("accepted"); + t.fs.replace(&p, b"kept").unwrap(); + let rejected = t.path("rejected"); + let payload = vec![b'x'; if bytes == 16 { 32 } else { 4 }]; + assert_eq!( + t.fs.replace(&rejected, &payload).unwrap_err().kind(), + io::ErrorKind::OutOfMemory + ); + assert_eq!(t.fs.read(&p).unwrap(), b"kept"); + assert!(!t.fs.try_exists(&rejected).unwrap()); + t.settle(); + assert_eq!(native::read(&p).unwrap(), b"kept"); + assert!(!rejected.exists()); + } +} + +#[test] +fn open_modes_reject_invalid_access_and_exclusive_recreation() { + let t = Fixture::new(); + let p = t.path("file"); + assert!(OpenOptions::new().open_in(&t.fs, &p).is_err()); + assert!( + OpenOptions::new() + .read(true) + .create(true) + .open_in(&t.fs, &p) + .is_err() + ); + t.fs.write(&p, b"contents").unwrap(); + assert_eq!( + OpenOptions::new() + .write(true) + .create_new(true) + .open_in(&t.fs, &p) + .err() + .unwrap() + .kind(), + io::ErrorKind::AlreadyExists + ); + let mut read = t.fs.open(&p).unwrap(); + assert!(read.write_all(b"bad").is_err()); + assert!(read.set_len(0).is_err()); + let mut write = OpenOptions::new().write(true).open_in(&t.fs, &p).unwrap(); + assert!(write.read(&mut [0]).is_err()); + assert_eq!(t.fs.read(&p).unwrap(), b"contents"); +} + +#[test] +#[cfg(unix)] +fn pending_work_retains_native_lease_until_recovery() { + let t = Fixture::new(); + let lock_path = t.path("owner.lock"); + let lease = + t.fs.acquire_lease(&lock_path, &t.root, LeaseMode::CreateNew) + .unwrap(); + let guarded = t.fs.guarded(&lease).unwrap(); + assert!(Fs::new(Arc::new(DiskBackend)).guarded(&lease).is_err()); + t.faults.arm(Point::Open, libc::ENOSPC); + guarded + .replace(t.path("value"), b"owned pending bytes") + .unwrap(); + drop(guarded); + drop(lease); + assert!(lock_path.exists()); + let competitor = Fs::new(Arc::new(DiskBackend)); + assert!( + competitor + .acquire_lease(&lock_path, &t.root, LeaseMode::ExistingOrNew) + .is_err() + ); + t.settle(); + assert_eq!( + native::read(t.path("value")).unwrap(), + b"owned pending bytes" + ); + assert!(!lock_path.exists()); + let next = competitor + .acquire_lease(&lock_path, &t.root, LeaseMode::CreateNew) + .unwrap(); + drop(next); + assert!(!lock_path.exists()); +} + +#[test] +#[cfg(unix)] +fn changed_lease_identity_blocks_replay_and_does_not_unlink_replacement() { + let t = Fixture::new(); + let lock_path = t.path("owner.lock"); + let lease = + t.fs.acquire_lease(&lock_path, &t.root, LeaseMode::CreateNew) + .unwrap(); + let guarded = t.fs.guarded(&lease).unwrap(); + t.faults.arm(Point::Open, libc::ENOSPC); + guarded + .replace(t.path("value"), b"must not replay") + .unwrap(); + native::rename(&lock_path, t.path("old.lock")).unwrap(); + native::write(&lock_path, b"different owner").unwrap(); + t.faults.clear(); + let r = t.fs.recover(); + assert!(r.remaining_operations > 0); + assert!(r.blocked.is_some()); + assert!(t.fs.require_disk(t.path("value")).is_err()); + assert!(!t.path("value").exists()); + drop(guarded); + drop(lease); + assert_eq!(native::read(&lock_path).unwrap(), b"different owner"); +} + +#[test] +#[cfg(unix)] +fn open_handle_tracks_rename_but_not_deleted_path_recreation() { + let t = Fixture::new(); + let a = t.path("a"); + let b = t.path("b"); + t.fs.write(&a, b"original").unwrap(); + let mut held = OpenOptions::new() + .read(true) + .write(true) + .open_in(&t.fs, &a) + .unwrap(); + t.faults.arm(Point::Rename, libc::ENOSPC); + t.fs.rename(&a, &b).unwrap(); + held.seek(SeekFrom::Start(0)).unwrap(); + held.write_all(b"renamed!").unwrap(); + assert_eq!(t.fs.read(&b).unwrap(), b"renamed!"); + t.fs.remove_file(&b).unwrap(); + t.fs.write(&b, b"replacement").unwrap(); + held.seek(SeekFrom::Start(0)).unwrap(); + let mut old = Vec::new(); + held.read_to_end(&mut old).unwrap(); + assert_eq!(old, b"renamed!"); + held.seek(SeekFrom::Start(0)).unwrap(); + held.write_all(b"detached").unwrap(); + assert_eq!(t.fs.read(&b).unwrap(), b"replacement"); + drop(held); + t.settle(); + assert!(!a.exists()); + assert_eq!(native::read(&b).unwrap(), b"replacement"); +} + +#[test] +#[cfg(unix)] +fn dropping_service_releases_pending_lease_without_claiming_durability() { + let t = Fixture::new(); + let fs = Fs::new(Arc::new(Injected { + disk: DiskBackend, + faults: t.faults.clone(), + })); + let lock_path = t.path("owner.lock"); + let lease = fs + .acquire_lease(&lock_path, &t.root, LeaseMode::CreateNew) + .unwrap(); + let guarded = fs.guarded(&lease).unwrap(); + t.faults.arm(Point::Open, libc::EDQUOT); + guarded.replace(t.path("pending"), b"memory only").unwrap(); + drop(lease); + drop(guarded); + assert!(lock_path.exists()); + drop(fs); + assert!(!lock_path.exists()); + assert!(!t.path("pending").exists()); +} + +#[test] +#[cfg(unix)] +fn secure_beneath_checks_disk_links_even_for_memory_entries() { + use std::os::unix::fs::symlink; + let t = Fixture::new(); + native::create_dir(t.path("dir")).unwrap(); + native::write(t.path("dir/disk"), b"disk").unwrap(); + let mut f = t.fs.open_beneath(&t.root, "dir/disk").unwrap(); + let mut data = String::new(); + f.read_to_string(&mut data).unwrap(); + assert_eq!(data, "disk"); + t.faults.arm(Point::Open, libc::ENOSPC); + t.fs.write(t.path("dir/memory"), b"memory").unwrap(); + assert!(t.fs.open_beneath(&t.root, "dir/memory").is_ok()); + native::rename(t.path("dir"), t.path("moved")).unwrap(); + symlink(t.path("moved"), t.path("dir")).unwrap(); + assert!(t.fs.open_beneath(&t.root, "dir/memory").is_err()); + assert!(t.fs.open_beneath(&t.root, "dir/disk").is_err()); + assert!(t.fs.open_beneath(&t.root, "../outside").is_err()); +} + +#[test] +#[cfg(unix)] +fn directory_sync_failure_keeps_namespace_and_retry_stages() { + let t = Fixture::new(); + t.faults.arm(Point::DirectorySync, libc::ENOSPC); + t.fs.create_dir(t.path("dir")).unwrap(); + assert!(t.fs.metadata(t.path("dir")).unwrap().is_dir()); + assert!(t.fs.recover().blocked.is_some()); + t.settle(); + t.faults.arm(Point::DirectorySync, libc::ENOSPC); + t.fs.remove_dir(t.path("dir")).unwrap(); + assert!(!t.fs.try_exists(t.path("dir")).unwrap()); + assert!(t.fs.recover().blocked.is_some()); + t.settle(); + assert!(!t.path("dir").exists()); +} + +#[test] +#[cfg(unix)] +fn rejected_capacity_payload_does_not_publish_and_orphans_remain_budgeted() { + let t = Fixture::new(); + let fs = Fs::with_budget( + Arc::new(Injected { + disk: DiskBackend, + faults: t.faults.clone(), + }), + 32, + 16, + ); + fs.write(t.path("old"), b"12345678").unwrap(); + let held = fs.open(t.path("old")).unwrap(); + fs.remove_file(t.path("old")).unwrap(); + fs.recover(); + assert_eq!(fs.status().retained_bytes, 0); // healthy detached descriptor is streaming + t.faults.arm(Point::Open, libc::ENOSPC); + assert_eq!( + fs.write(t.path("large"), [0; 20]).unwrap_err().kind(), + io::ErrorKind::OutOfMemory + ); + assert!(!fs.try_exists(t.path("large")).unwrap()); + assert!(fs.status().exhausted); + drop(held); + fs.recover(); + assert_eq!(fs.status().retained_bytes, 0); +} + +#[test] +#[cfg(unix)] +fn noncapacity_partial_image_is_abandoned_not_replayed() { + let t = Fixture::new(); + t.fs.write(t.path("value"), b"old").unwrap(); + t.faults.arm(Point::Write, libc::EIO); + assert_eq!( + t.fs.write(t.path("value"), b"not accepted") + .unwrap_err() + .raw_os_error(), + Some(libc::EIO) + ); + assert_eq!(t.fs.read(t.path("value")).unwrap(), b"old"); + assert_eq!(t.fs.status().pending_operations, 0); + t.settle(); + assert_eq!(native::read(t.path("value")).unwrap(), b"old"); + assert_eq!(names(&t.fs, &t.root), vec![OsString::from("value")]); +} + +#[test] +#[cfg(unix)] +fn bounded_recovery_does_not_discard_later_work() { + let t = Fixture::new(); + t.faults.arm(Point::Open, libc::ENOSPC); + for index in 0..80 { + t.fs.write(t.path(&format!("item-{index}")), [index as u8]) + .unwrap(); + } + assert_eq!(t.fs.status().pending_operations, 80); + t.faults.clear(); + let first = t.fs.recover(); + assert_eq!(first.completed_operations, 64); + assert_eq!(first.remaining_operations, 16); + assert!(first.blocked.is_none()); + let second = t.fs.recover(); + assert_eq!(second.completed_operations, 16); + assert_eq!(second.remaining_operations, 0); + assert_eq!(t.fs.recover().completed_operations, 0); + for index in 0..80 { + assert_eq!( + native::read(t.path(&format!("item-{index}"))).unwrap(), + [index as u8] + ); + } +} + +#[test] +#[cfg(unix)] +fn temporary_images_are_not_namespace_entries() { + let t = Fixture::new(); + t.faults.arm(Point::Write, libc::ENOSPC); + t.fs.write(t.path("visible"), b"intended complete contents") + .unwrap(); + assert_eq!(names(&t.fs, &t.root), vec![OsString::from("visible")]); + assert_eq!( + t.fs.read(t.path("visible")).unwrap(), + b"intended complete contents" + ); + t.settle(); +} + +#[test] +fn healthy_large_files_and_directory_renames_do_not_use_fallback_budget() { + let t = Fixture::new(); + native::create_dir(t.path("large-dir")).unwrap(); + let path = t.path("large-dir/file"); + let native_file = native::File::create(&path).unwrap(); + native_file.set_len(4 * 1024 * 1024).unwrap(); + drop(native_file); + let fs = Fs::with_budget(Arc::new(DiskBackend), 16, 1); + let mut old = fs.open(&path).unwrap(); + old.seek(SeekFrom::End(-1)).unwrap(); + assert_eq!(old.read(&mut [1]).unwrap(), 1); + let secure = fs.open_beneath(t.path("large-dir"), "file").unwrap(); + assert_eq!(secure.metadata().unwrap().len(), 4 * 1024 * 1024); + fs.rename(t.path("large-dir"), t.path("moved")).unwrap(); + let mut writer = OpenOptions::new() + .append(true) + .open_in(&fs, t.path("moved/file")) + .unwrap(); + writer.write_all(b"tail").unwrap(); + assert_eq!(writer.metadata().unwrap().len(), 4 * 1024 * 1024 + 4); + assert_eq!(fs.status().retained_bytes, 0); + assert!(!fs.status().exhausted); + fs.write(t.path("new-large"), [7u8; 8192]).unwrap(); + assert_eq!(fs.status().retained_bytes, 0); +} + +#[test] +fn fresh_reads_do_not_use_clean_live_handle_cache() { + let t = Fixture::new(); + t.fs.write(t.path("value"), b"old").unwrap(); + let mut held = t.fs.open(t.path("value")).unwrap(); + native::write(t.path("replacement"), b"new and longer").unwrap(); + native::rename(t.path("replacement"), t.path("value")).unwrap(); + assert_eq!(t.fs.read(t.path("value")).unwrap(), b"new and longer"); + let mut old = String::new(); + held.read_to_string(&mut old).unwrap(); + assert_eq!(old, "old"); + assert_eq!(t.fs.metadata(t.path("value")).unwrap().len(), 14); +} + +#[test] +#[cfg(unix)] +fn dirty_lease_handoff_reopens_but_rejects_live_callers() { + let t = Fixture::new(); + let lock_path = t.path("session.lock"); + let lease = + t.fs.acquire_lease(&lock_path, &t.root, LeaseMode::CreateNew) + .unwrap(); + let guarded = t.fs.guarded(&lease).unwrap(); + let mut writer = OpenOptions::new() + .append(true) + .create(true) + .private(true) + .open_in(&guarded, t.path("session")) + .unwrap(); + writer.write_all(b"first\n").unwrap(); + t.faults.arm(Point::Open, libc::ENOSPC); + writer.write_all(b"second\n").unwrap(); + assert_eq!( + t.fs.acquire_lease(&lock_path, &t.root, LeaseMode::ExistingOrNew) + .err() + .unwrap() + .kind(), + io::ErrorKind::WouldBlock + ); + drop(lease); + drop(guarded); + assert_eq!( + t.fs.acquire_lease(&lock_path, &t.root, LeaseMode::ExistingOrNew) + .err() + .unwrap() + .kind(), + io::ErrorKind::WouldBlock + ); + drop(writer); + let resumed = + t.fs.acquire_lease(&lock_path, &t.root, LeaseMode::ExistingOrNew) + .unwrap(); + let guarded = t.fs.guarded(&resumed).unwrap(); + assert_eq!(guarded.read(t.path("session")).unwrap(), b"first\nsecond\n"); + let mut writer = OpenOptions::new() + .append(true) + .open_in(&guarded, t.path("session")) + .unwrap(); + writer.write_all(b"third\n").unwrap(); + drop(writer); + drop(guarded); + drop(resumed); + t.settle(); + assert_eq!( + native::read(t.path("session")).unwrap(), + b"first\nsecond\nthird\n" + ); + assert!(!lock_path.exists()); +} + +#[test] +#[cfg(unix)] +fn large_native_baseline_accepts_small_bounded_delta() { + let t = Fixture::new(); + let path = t.path("large"); + let file = native::File::create(&path).unwrap(); + file.set_len(1024 * 1024).unwrap(); + drop(file); + let fs = Fs::with_budget( + Arc::new(Injected { + disk: DiskBackend, + faults: t.faults.clone(), + }), + 64, + 16, + ); + let mut writer = OpenOptions::new().append(true).open_in(&fs, &path).unwrap(); + t.faults.arm(Point::Write, libc::ENOSPC); + writer.write_all(b"delta").unwrap(); + drop(writer); + assert!(fs.status().retained_bytes <= 64); + let mut read = fs.open(&path).unwrap(); + read.seek(SeekFrom::End(-5)).unwrap(); + let mut tail = [0; 5]; + read.read_exact(&mut tail).unwrap(); + assert_eq!(&tail, b"delta"); + drop(read); + t.faults.clear(); + fs.require_disk(&path).unwrap(); + assert_eq!(native::metadata(&path).unwrap().len(), 1024 * 1024 + 5); +} + +#[test] +#[cfg(unix)] +fn rejected_delta_does_not_replay_or_advance_cursor() { + let t = Fixture::new(); + t.fs.write(t.path("value"), b"old").unwrap(); + let mut file = OpenOptions::new() + .read(true) + .write(true) + .open_in(&t.fs, t.path("value")) + .unwrap(); + t.faults.arm(Point::Write, libc::EIO); + assert!(file.write_all(b"rejected delta").is_err()); + assert_eq!(file.stream_position().unwrap(), 0); + assert_eq!(t.fs.status().pending_operations, 0); + assert_eq!(t.fs.read(t.path("value")).unwrap(), b"old"); + t.settle(); + assert_eq!(native::read(t.path("value")).unwrap(), b"old"); +} + +#[test] +#[cfg(unix)] +fn recovery_does_not_truncate_a_replaced_or_hardlinked_temp() { + use std::os::unix::fs::PermissionsExt; + for hardlink in [false, true] { + let t = Fixture::new(); + t.faults.arm(Point::Write, libc::ENOSPC); + t.fs.write(t.path("value"), b"accepted data longer than prefix") + .unwrap(); + let temp = native::read_dir(&t.root) + .unwrap() + .map(Result::unwrap) + .find(|e| { + e.file_name() + .to_string_lossy() + .starts_with(".kit-resilient-") + }) + .unwrap() + .path(); + if hardlink { + native::hard_link(&temp, t.path("linked")).unwrap(); + } else { + native::remove_file(&temp).unwrap(); + native::write(&temp, b"other owner").unwrap(); + native::set_permissions(&temp, Permissions::from_mode(0o600)).unwrap(); + } + let before = native::read(&temp).unwrap(); + t.faults.clear(); + assert_eq!( + t.fs.recover().blocked.unwrap().kind(), + io::ErrorKind::PermissionDenied + ); + assert_eq!(native::read(&temp).unwrap(), before); + assert_eq!( + t.fs.read(t.path("value")).unwrap(), + b"accepted data longer than prefix" + ); + } +} + +#[test] +#[cfg(unix)] +fn renamed_directory_redirects_follow_partially_completed_recovery() { + let t = Fixture::new(); + native::create_dir(t.path("a")).unwrap(); + native::write(t.path("a/child"), b"unchanged baseline").unwrap(); + t.faults.arm(Point::Rename, libc::ENOSPC); + t.fs.rename(t.path("a"), t.path("b")).unwrap(); + t.fs.rename(t.path("b"), t.path("c")).unwrap(); + assert_eq!(t.fs.read(t.path("c/child")).unwrap(), b"unchanged baseline"); + t.faults.arm(Point::DirectorySync, libc::ENOSPC); + assert!(t.fs.recover().blocked.is_some()); + assert_eq!(t.fs.read(t.path("c/child")).unwrap(), b"unchanged baseline"); + t.settle(); + assert_eq!( + native::read(t.path("c/child")).unwrap(), + b"unchanged baseline" + ); +} + +#[test] +#[cfg(unix)] +fn public_directory_entries_preserve_symlink_alias_spelling() { + use std::os::unix::fs::symlink; + let t = Fixture::new(); + native::create_dir(t.path("actual")).unwrap(); + native::write(t.path("actual/disk"), b"disk").unwrap(); + symlink(t.path("actual"), t.path("alias")).unwrap(); + // Use an alias in an ancestor, as /var -> /private/var on macOS does. + native::create_dir(t.path("actual/listed")).unwrap(); + native::write(t.path("actual/listed/disk"), b"disk").unwrap(); + let requested = t.path("alias/listed"); + let entry = t.fs.read_dir(&requested).unwrap().next().unwrap().unwrap(); + assert_eq!(entry.path(), requested.join("disk")); + assert_eq!( + entry.path().strip_prefix(&requested).unwrap(), + Path::new("disk") + ); + assert!(entry.metadata().unwrap().is_file()); + t.faults.arm(Point::Open, libc::ENOSPC); + t.fs.write(requested.join("memory"), b"memory").unwrap(); + for entry in t.fs.read_dir(&requested).unwrap() { + assert!(entry.unwrap().path().starts_with(&requested)); + } + t.settle(); +} + +#[test] +#[cfg(unix)] +fn clean_lost_lease_can_be_reacquired_while_old_observer_stays_fenced() { + let t = Fixture::new(); + let path = t.path("lease.lock"); + let old = + t.fs.acquire_lease(&path, &t.root, LeaseMode::CreateNew) + .unwrap(); + let guarded = t.fs.guarded(&old).unwrap(); + guarded.write(t.path("value"), b"durable").unwrap(); + native::remove_file(&path).unwrap(); + assert_eq!(old.check().unwrap_err().kind(), io::ErrorKind::NotFound); + assert_eq!(old.check().unwrap_err().kind(), io::ErrorKind::NotFound); + let new = + t.fs.acquire_lease(&path, &t.root, LeaseMode::CreateNew) + .unwrap(); + new.check().unwrap(); + assert_eq!( + old.check().unwrap_err().kind(), + io::ErrorKind::PermissionDenied + ); + assert!(guarded.write(t.path("value"), b"forbidden").is_err()); + assert_eq!(t.fs.read(t.path("value")).unwrap(), b"durable"); + drop(old); + drop(guarded); + new.check().unwrap(); +} + +#[test] +#[cfg(unix)] +fn dirty_lost_lease_cannot_acquire_new_authority_for_old_pending_bytes() { + let t = Fixture::new(); + let path = t.path("lease.lock"); + let lease = + t.fs.acquire_lease(&path, &t.root, LeaseMode::CreateNew) + .unwrap(); + let guarded = t.fs.guarded(&lease).unwrap(); + t.faults.arm(Point::Open, libc::ENOSPC); + guarded + .write(t.path("value"), b"must remain pending") + .unwrap(); + drop(guarded); + drop(lease); + native::remove_file(&path).unwrap(); + t.faults.clear(); + assert!( + t.fs.acquire_lease(&path, &t.root, LeaseMode::CreateNew) + .is_err() + ); + assert!(!path.exists()); + assert!(!t.path("value").exists()); + assert_eq!(t.fs.status().pending_operations, 1); + assert_eq!(t.fs.read(t.path("value")).unwrap(), b"must remain pending"); +} + +#[test] +#[cfg(unix)] +fn private_directory_creation_tightens_only_existing_target() { + use std::os::unix::fs::PermissionsExt; + let t = Fixture::new(); + native::create_dir(t.path("parent")).unwrap(); + native::create_dir(t.path("parent/target")).unwrap(); + native::set_permissions(t.path("parent"), Permissions::from_mode(0o755)).unwrap(); + native::set_permissions(t.path("parent/target"), Permissions::from_mode(0o755)).unwrap(); + t.fs.create_private_dir_all(t.path("parent/target")) + .unwrap(); + assert_eq!( + native::metadata(t.path("parent")) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o755 + ); + assert_eq!( + native::metadata(t.path("parent/target")) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o700 + ); +} + +#[test] +fn metadata_identity_is_physical_or_stable_overlay_not_file_contents() { + let t = Fixture::new(); + t.fs.write(t.path("one"), b"same").unwrap(); + t.fs.write(t.path("two"), b"same").unwrap(); + let one = t.fs.metadata(t.path("one")).unwrap(); + let one_file = t.fs.open(t.path("one")).unwrap().metadata().unwrap(); + let two = t.fs.metadata(t.path("two")).unwrap(); + assert!(one.same_identity(&one_file)); + assert!(!one.same_identity(&two)); + t.faults.arm(Point::Open, if cfg!(unix) { 28 } else { 112 }); + t.fs.write(t.path("pending"), b"memory").unwrap(); + let first = t.fs.metadata(t.path("pending")).unwrap(); + let second = t.fs.metadata(t.path("pending")).unwrap(); + assert!(first.same_identity(&second)); +} + +#[test] +fn allocator_failure_invokes_exit_hook_before_returning_an_error() { + const CHILD: &str = "KIT_FS_ALLOCATOR_FAILURE_CHILD"; + if std::env::var_os(CHILD).is_some() { + set_allocation_failure_handler(|| std::process::exit(73)).unwrap(); + let _ = allocation_oom(); + panic!("allocator failure returned past the exit hook"); + } + let module = module_path!().split_once("::").unwrap().1; + let name = format!("{module}::allocator_failure_invokes_exit_hook_before_returning_an_error"); + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", &name, "--nocapture"]) + .env(CHILD, "1") + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(73), "{output:?}"); +} diff --git a/src/runtime.rs b/src/runtime.rs index cd1d7ce7..428929f9 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -10,7 +10,6 @@ use std::{ }; use agentkit_acp::{AcpIntegration, AcpRuntimeError}; -use agentkit_context::{AgentsMd, ContextLoader}; use agentkit_core::{ CancellationController, CancellationHandle, FinishReason, Item, ItemKind, MetadataMap, Part, ToolOutput, ToolResultPart, @@ -38,8 +37,9 @@ use crate::{ ModelSelection, ProviderKind, ReasoningEffort, SelectableAdapter, SelectableSession, }, tools::{ - A2aTool, AuthTool, CloseTool, DocsTool, EditTool, ForkTool, McpTool, Observed, PromptTool, - ShellTool, SubagentTool, Subagents, SubagentsTool, ToolSearch, observe_shared, + A2aTool, ArtifactTool, AuthTool, CloseTool, DocsTool, EditTool, ForkTool, McpTool, + Observed, PromptTool, ShellTool, SubagentTool, Subagents, SubagentsTool, ToolSearch, + observe_shared, }, }; @@ -380,11 +380,10 @@ impl Runtime { reasoning_effort: Option, openrouter_api_key: Option, ) -> Result, String> { - let root = root - .as_ref() - .canonicalize() + crate::resilient_fs::start_recovery_worker(); + let root = crate::resilient_fs::canonicalize(root.as_ref()) .map_err(|error| format!("could not open working directory: {error}"))?; - if !root.is_dir() { + if !crate::resilient_fs::metadata(&root).is_ok_and(|metadata| metadata.is_dir()) { return Err(format!( "working directory is not a directory: {}", root.display() @@ -952,6 +951,9 @@ impl Runtime { skills: Arc, ) -> ComposeOnly { let mut children = agentkit_tools_core::ToolRegistry::new() + .with(Observed::new(ArtifactTool::new(crate::artifacts::base( + &self.root, + )))) .with(Observed::new(DocsTool::new())) .with(Observed::new(ShellTool::new(self.root.clone()))) .with(Observed::new(EditTool::new(self.root.clone()))); @@ -1152,6 +1154,13 @@ impl Runtime { depth: usize, cancellation: Option, ) -> Result { + if crate::resilient_fs::shutdown_token().is_cancelled() { + return Err(LoopError::InvalidState( + "storage shutdown in progress".into(), + )); + } + let controller = CancellationController::new(); + let _shutdown_bridge = StorageCancellationBridge::new(controller.clone(), cancellation); if self.plugin_runtime.is_some() { self.mcp.refresh().await.map_err(LoopError::InvalidState)?; } @@ -1169,7 +1178,7 @@ impl Runtime { ) .map_err(LoopError::InvalidState)?; let subagents = self.subagents.fresh(); - let mut builder = Agent::builder() + let builder = Agent::builder() .model(self.adapter.clone()) .telemetry(self.agentkit_telemetry()) .add_tool_source(self.compose_with_jobs( @@ -1182,9 +1191,7 @@ impl Runtime { .mutator(compactor) .transcript(transcript) .input(vec![Item::text(ItemKind::User, prompt)]); - if let Some(cancellation) = cancellation { - builder = builder.cancellation(cancellation); - } + let builder = builder.cancellation(controller.handle()); let mut driver = builder .build()? .start(SessionConfig::new(session).without_cache()) @@ -1263,9 +1270,7 @@ impl Runtime { where I: LoopObserver + Clone + 'static, { - let cwd = context - .cwd - .canonicalize() + let cwd = crate::resilient_fs::canonicalize(&context.cwd) .map_err(|error| AcpRuntimeError::Loop(error.to_string()))?; if cwd != self.root || !context.additional_directories.is_empty() { return Err(AcpRuntimeError::Loop(format!( @@ -1591,20 +1596,23 @@ fn build_skill_registry( let default_roots = default_skill_roots(root); let canonical_defaults = default_roots .iter() - .map(|path| path.canonicalize().unwrap_or_else(|_| path.clone())) + .map(|path| crate::resilient_fs::canonicalize(path).unwrap_or_else(|_| path.clone())) .collect::>(); let canonical_package_roots = package_roots .iter() - .filter_map(|path| path.canonicalize().ok()) + .filter_map(|path| crate::resilient_fs::canonicalize(path).ok()) .collect::>(); let canonical_plugin_skills = skill_directories .iter() - .filter_map(|path| path.canonicalize().ok()) + .filter_map(|path| crate::resilient_fs::canonicalize(path).ok()) .collect::>(); let mut roots = default_roots; roots.extend(skill_directories.iter().cloned()); + // This dependency is a native path reader. Do not hand it stale disk paths + // for a pending generation; it can be discovered on a later refresh. + roots.retain(|path| crate::resilient_fs::global().require_disk(path).is_ok()); SkillRegistry::from_paths(roots).with_filter(move |skill: &agentkit_tool_skills::Skill| { - let Ok(base) = skill.base_dir.canonicalize() else { + let Ok(base) = crate::resilient_fs::canonicalize(&skill.base_dir) else { return false; }; if canonical_defaults @@ -1613,7 +1621,7 @@ fn build_skill_registry( { return true; } - let Ok(location) = skill.location.canonicalize() else { + let Ok(location) = crate::resilient_fs::canonicalize(&skill.location) else { return false; }; canonical_plugin_skills.contains(&base) @@ -2130,12 +2138,18 @@ impl BackgroundableCompose { } self.background_jobs.changed(&mut jobs); } - if let Some(cancellation) = foreground_cancellation { + { + let shutdown = crate::resilient_fs::shutdown_token().child_token(); let jobs = self.background_jobs.clone(); let relay_call_id = call_id.clone(); let relay = tokio::spawn(async move { - cancellation.cancelled().await; - jobs.propagate_foreground_cancellation(&relay_call_id); + tokio::select! { + _ = shutdown.cancelled() => { jobs.cancel_running(&relay_call_id.0); }, + _ = async { + if let Some(cancellation) = foreground_cancellation { cancellation.cancelled().await; } + else { std::future::pending::<()>().await; } + } => jobs.propagate_foreground_cancellation(&relay_call_id), + } }) .abort_handle(); if let Ok(mut jobs) = self.background_jobs.state.lock() @@ -2273,14 +2287,44 @@ impl ComposeBackend for HiddenRunletBackend { } async fn load_initial_transcript(root: &Path, system_prompt: String) -> Result, String> { - let mut transcript = vec![Item::text(ItemKind::System, system_prompt)]; - let context = ContextLoader::new() - .with_source(AgentsMd::discover_all(root)) - .load() - .await - .map_err(|error| format!("could not load AGENTS.md context: {error}"))?; - transcript.extend(context); - Ok(transcript) + let root = root.to_path_buf(); + tokio::task::spawn_blocking(move || { + let mut transcript = vec![Item::text(ItemKind::System, system_prompt)]; + // Preserve agentkit-context's outermost-to-innermost ordering and item + // metadata, while reading through the same view as other internal IO. + let ancestors = root.ancestors().collect::>(); + for directory in ancestors.into_iter().rev() { + let path = directory.join("AGENTS.md"); + let body = match crate::resilient_fs::read_to_string(&path) { + Ok(body) => body, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => { + return Err(format!( + "could not load AGENTS.md context: could not read {}: {error}", + path.display() + )); + } + }; + let mut item = Item::text( + ItemKind::Context, + format!( + "[Loaded AGENTS]\nPath: {}\n\n{}", + path.display(), + body.trim_end() + ), + ); + item.metadata + .insert("agentkit.context.source".into(), json!("agents_md")); + item.metadata.insert( + "agentkit.context.path".into(), + json!(path.display().to_string()), + ); + transcript.push(item); + } + Ok(transcript) + }) + .await + .map_err(|error| format!("could not load AGENTS.md context: {error}"))? } fn record_runtime_failure( @@ -2341,3 +2385,37 @@ async fn drive(driver: &mut LoopDriver) -> Result); + +impl StorageCancellationBridge { + pub(crate) fn new( + controller: CancellationController, + external: Option, + ) -> Self { + let shutdown = crate::resilient_fs::shutdown_token().child_token(); + let external = external.map(|handle| handle.checkpoint()); + Self(tokio::spawn(async move { + tokio::select! { + _ = shutdown.cancelled() => {}, + _ = async { + if let Some(external) = external { external.cancelled().await; } + else { std::future::pending::<()>().await; } + } => {}, + } + // A generation snapshot can be created during async startup after + // the first interrupt. Keep shutdown asserted until the owner closes. + loop { + controller.interrupt(); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + })) + } +} + +impl Drop for StorageCancellationBridge { + fn drop(&mut self) { + self.0.abort(); + } +} diff --git a/src/session.rs b/src/session.rs index df2f50b7..aff39381 100644 --- a/src/session.rs +++ b/src/session.rs @@ -1,8 +1,11 @@ -//! Durable, append-only session transcripts and their filesystem lock. +//! Append-only session transcripts and their filesystem ownership lease. +//! +//! Writes are accepted by the shared resilient filesystem. During capacity +//! failures, accepted records and their ownership remain process-resident until +//! recovery persists them; acceptance is not a process-crash durability guarantee. use std::{ env, - fs::{self, File, OpenOptions}, io::{self, BufRead, BufReader, Seek, SeekFrom, Write}, path::{Path, PathBuf}, sync::{ @@ -12,6 +15,8 @@ use std::{ time::{SystemTime, UNIX_EPOCH}, }; +use crate::resilient_fs::{self as fs, File, Fs, Lease, LeaseMode, OpenOptions}; + use agentkit_core::{Item, ItemKind, Part, Timestamp}; use agentkit_loop::{TranscriptEvent, TranscriptObserver}; use serde::{Deserialize, Serialize}; @@ -85,23 +90,16 @@ struct Writer { file: File, lock: SessionLock, created: Option, - // Once degraded, the on-disk prefix is never appended to again. - memory: Option, - allow_memory: bool, - #[cfg(test)] - append_failure: Option<(usize, io::ErrorKind)>, } struct SessionLock { path: PathBuf, - token: String, - // Retaining the OS lock closes the token check/write race. The option lets - // Drop close the handle before removing the path on Windows. - file: Option, + lease: Lease, } /// Removes an incompletely bootstrapped new transcript unless opening commits. struct CreatedTranscript { + filesystem: Fs, path: PathBuf, keep: bool, } @@ -113,8 +111,12 @@ struct InitialTranscriptOptions { } impl CreatedTranscript { - fn new(path: PathBuf) -> Self { - Self { path, keep: false } + fn new(path: PathBuf, filesystem: Fs) -> Self { + Self { + path, + filesystem, + keep: false, + } } fn keep(mut self) { @@ -125,7 +127,7 @@ impl CreatedTranscript { impl Drop for CreatedTranscript { fn drop(&mut self) { if !self.keep { - let _ = fs::remove_file(&self.path); + let _ = self.filesystem.remove_file(&self.path); } } } @@ -217,8 +219,7 @@ pub(crate) fn remove_stale_lock_in( &workspace_storage_directory(directory, &workspace_root), session_id, ); - let path = if scoped - .try_exists() + let path = if fs::try_exists(&scoped) .map_err(|error| format!("could not inspect {}: {error}", scoped.display()))? { scoped @@ -229,21 +230,18 @@ pub(crate) fn remove_stale_lock_in( } else { return Ok(()); }; - let file = match OpenOptions::new().read(true).write(true).open(&path) { - Ok(file) => file, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(error) => return Err(format!("could not inspect session lock: {error}")), - }; - file.try_lock() - .map_err(|_| "session lock is still held by a live Kit instance".to_string())?; - drop(file); - fs::remove_file(&path).map_err(|error| format!("could not remove stale session lock: {error}")) + if !fs::try_exists(&path).map_err(|error| format!("could not inspect session lock: {error}"))? { + return Ok(()); + } + // Cleanup requires real interprocess authority, never an overlay-only lock. + drop(SessionLock::acquire(path, true)?); + Ok(()) } /// Opens a new or resumed transcript and takes its mutation lock. /// /// `resume` requires the transcript to exist. `force` fences an abandoned lock; -/// an older process checks the lock token before every append and can no longer +/// an older process checks the ownership lease before every append and can no longer /// write after it has been replaced. pub fn open( root: &Path, @@ -323,9 +321,31 @@ fn open_with_initial_timestamps_in( .map_err(|error| format!("could not create session directory: {error}"))?; let path = transcript_path(&scoped_directory, session_id); let lock = SessionLock::acquire(lock_path(&scoped_directory, session_id), force)?; - let _migration_locks = lock_migration_sources(directory, &workspace_root, session_id)?; - recover_torn_migration_writes(&path, directory, &workspace_root, session_id)?; - let authority = select_authority(directory, &workspace_root, session_id)?; + let filesystem = lock.filesystem()?; + let (migration_locks, authority) = loop { + let sources = lock_migration_sources(directory, &workspace_root, session_id)?; + recover_torn_migration_writes( + &path, + directory, + &workspace_root, + session_id, + &lock, + &sources, + )?; + let authority = select_authority(directory, &workspace_root, session_id)?; + // A legacy creator can publish a history while sources are discovered. + // Never use a newly discovered mutable history without its real lock. + let all_locked = authority.as_ref().is_none_or(|authority| { + authority.legacy_histories.iter().all(|path| { + sources + .iter() + .any(|source| source.path == path.with_extension("lock")) + }) + }); + if all_locked { + break (sources, authority); + } + }; if resume { let authority = authority.ok_or_else(|| format!("session {session_id:?} does not exist"))?; @@ -334,6 +354,7 @@ fn open_with_initial_timestamps_in( .iter() .find_map(|items| first_user_item_index(items).map(|index| items[..=index].to_vec())); establish_scoped_authority( + &filesystem, &path, session_id, &workspace_root, @@ -341,7 +362,8 @@ fn open_with_initial_timestamps_in( title_seed.as_deref(), )?; for legacy in authority.legacy_histories { - redirect_legacy_transcript(&legacy, &path, session_id, &workspace_root)?; + let source_fs = migration_filesystem(&legacy, &lock, &migration_locks)?; + redirect_legacy_transcript(&source_fs, &legacy, &path, session_id, &workspace_root)?; } } else if authority.is_some() { return Err(format!( @@ -374,9 +396,9 @@ fn open_with_initial_timestamps_in( options.create_new(true); } let file = options - .open(&path) + .open_in(&filesystem, &path) .map_err(|error| format!("could not open {}: {error}", path.display()))?; - let created = (!resume).then(|| CreatedTranscript::new(path.clone())); + let created = (!resume).then(|| CreatedTranscript::new(path.clone(), filesystem.clone())); let mut writer = Writer { session_id: session_id.into(), generation, @@ -385,10 +407,6 @@ fn open_with_initial_timestamps_in( file, lock, created, - memory: None, - allow_memory: false, - #[cfg(test)] - append_failure: None, }; if resume && stored_workspace.is_none() { writer.replace(&transcript)?; @@ -419,9 +437,6 @@ fn open_with_initial_timestamps_in( if initial_options.commit_creation { writer.commit_creation(); } - // Opening/repairing and cloning a session must still establish durable - // history. Degradation is limited to subsequent active-session mutations. - writer.allow_memory = true; Ok(OpenSession { transcript, observer: SessionObserver(Arc::new(Mutex::new(writer))), @@ -436,10 +451,9 @@ impl SessionObserver { .commit_creation(); } - /// Records a complete transcript replacement produced by a mutator. + /// Durably records a complete transcript replacement produced by a mutator. /// Existing append records remain intact, while readers treat this record as - /// a new canonical snapshot. After a capacity failure, replacements share - /// the writer's memory-only tail rather than claiming disk durability. + /// a new canonical snapshot. pub fn replace(&self, transcript: &[Item]) -> Result<(), String> { if transcript.is_empty() { return Err("cannot persist an empty transcript replacement".into()); @@ -455,9 +469,13 @@ impl TranscriptObserver for SessionObserver { fn on_transcript_event(&self, event: TranscriptEvent<'_>) { let mut writer = self.0.lock().expect("session transcript writer poisoned"); if let Err(error) = writer.append(event.item) { + if fs::global().status().exhausted || fs::shutdown_token().is_cancelled() { + fs::request_shutdown(); + return; + } // The loop invokes observers before committing the item in memory. - // Capacity failures use bounded volatile storage. Other failures - // still refuse the mutation rather than bypassing integrity checks. + // Refusing that mutation is safer than continuing with history that + // was not durably recorded and cannot be resumed faithfully. panic!("session persistence failed: {error}"); } } @@ -483,7 +501,16 @@ impl Writer { .generation .checked_add(1) .ok_or_else(|| "session generation overflowed".to_string())?; - self.write_record(Some(item), None, generation) + let record = Record { + schema_version: SCHEMA_VERSION, + session_id: self.session_id.clone(), + generation, + workspace_root: Some(self.workspace_root.clone()), + item: Some(item.clone()), + replacement: None, + redirect: None, + }; + self.write_record(record, generation) } fn replace(&mut self, transcript: &[Item]) -> Result<(), String> { @@ -492,112 +519,41 @@ impl Writer { .generation .checked_add(1) .ok_or_else(|| "session generation overflowed".to_string())?; - self.write_record(None, Some(transcript), generation) - } - - fn write_record( - &mut self, - item: Option<&Item>, - replacement: Option<&[Item]>, - generation: u64, - ) -> Result<(), String> { - // Borrow replacements rather than cloning the entire transcript before - // the fallible writer has a chance to enforce its budget. - #[derive(Serialize)] - struct AppendRecord<'a> { - schema_version: u32, - session_id: &'a str, - generation: u64, - workspace_root: &'a Path, - #[serde(skip_serializing_if = "Option::is_none")] - item: Option<&'a Item>, - #[serde(skip_serializing_if = "Option::is_none")] - replacement: Option<&'a [Item]>, - } - let record = AppendRecord { + let record = Record { schema_version: SCHEMA_VERSION, - session_id: &self.session_id, + session_id: self.session_id.clone(), generation, - workspace_root: &self.workspace_root, - item, - replacement, + workspace_root: Some(self.workspace_root.clone()), + item: None, + replacement: Some(transcript.to_vec()), + redirect: None, }; - if self.memory.is_none() { - // Encode before touching disk; only append/sync capacity errors - // permit degradation, not serialization or metadata failures. - let mut encoded = serde_json::to_vec(&record) - .map_err(|error| format!("could not encode transcript record: {error}"))?; - encoded.push(b'\n'); - let offset = self - .file - .metadata() - .map_err(|error| format!("could not inspect transcript length: {error}"))? - .len(); - #[cfg(test)] - let result = if let Some((bytes, kind)) = self.append_failure.take() { - self.file - .write_all(&encoded[..bytes.min(encoded.len())]) - .and_then(|_| Err(io::Error::from(kind))) - } else { - self.file - .write_all(&encoded) - .and_then(|_| self.file.sync_data()) - }; - #[cfg(not(test))] - let result = self - .file - .write_all(&encoded) - .and_then(|_| self.file.sync_data()); - if let Err(error) = result { - // Sync failure can leave a complete record, not just a partial - // write. Remove either before accepting the memory-only tail. - self.file.set_len(offset).map_err(|rollback| { - format!("could not roll back transcript append after {error}: {rollback}") - })?; - if !self.allow_memory || !crate::storage::is_capacity_error(&error) { - return Err(format!("could not persist transcript record: {error}")); - } - // Sync itself may be failing: truncation restores the logical - // prefix, but its crash durability cannot be promised here. - self.memory = Some(crate::storage::MemoryBuffer::default()); - let _ = io::stderr().write_all( - b"kit: session storage is full; subsequent transcript changes are memory-only and will be lost when the session closes.\n", - ); - } else { - self.generation = generation; - return Ok(()); - } - } - // Validate without retaining bytes before mutating the memory log. A - // rejected timestamp/serialization must not leave a partial record if - // a caller handles the error and attempts another replacement. - serde_json::to_writer(io::sink(), &record) - .map_err(|error| format!("could not encode transcript record: {error}"))?; - let mut memory = self - .memory - .as_mut() - .expect("memory fallback initialized") - .writer_or_exit(); - // Handle allocation refusal inside the writer, before serde_json can - // allocate a boxed I/O error while the allocator may be exhausted. - serde_json::to_writer(&mut memory, &record) + self.write_record(record, generation) + } + + fn write_record(&mut self, record: Record, generation: u64) -> Result<(), String> { + // Encode before touching the append-only file, so serialization + // failures can never leave a partial JSON record behind. + let mut encoded = serde_json::to_vec(&record) .map_err(|error| format!("could not encode transcript record: {error}"))?; - if memory.write_all(b"\n").is_err() { - crate::storage::exit_exhausted(); - } + encoded.push(b'\n'); + self.file + .write_all(&encoded) + .and_then(|_| self.file.sync_data()) + .map_err(|error| { + if error.kind() == io::ErrorKind::OutOfMemory { + fs::request_shutdown(); + } + format!("could not persist transcript record: {error}") + })?; self.generation = generation; Ok(()) } fn ensure_lock(&mut self) -> Result<(), String> { - if self.memory.is_some() { - // Never reconstruct disk history from a prefix missing the memory - // tail, nor fabricate authority if the degraded writer loses its lock. - return self.lock.check().map_err(|error| error.to_string()); - } match self.lock.check() { Ok(()) => { - if self.path.try_exists().map_err(|error| { + if fs::try_exists(&self.path).map_err(|error| { format!("could not inspect {}: {error}", self.path.display()) })? { Ok(()) @@ -618,8 +574,8 @@ impl Writer { fs::create_dir_all(directory) .map_err(|error| format!("could not recreate session directory: {error}"))?; let lock = SessionLock::acquire(lock_path(directory, &self.session_id), false)?; - self.reconstruct()?; self.lock = lock; + self.reconstruct()?; Ok(()) } @@ -636,10 +592,10 @@ impl Writer { .read(true) .append(true) .create_new(true) - .open(&self.path) + .open_in(&self.lock.filesystem()?, &self.path) .map_err(|error| format!("could not reconstruct {}: {error}", self.path.display()))?; if let Err(error) = io::copy(&mut source, &mut file).and_then(|_| file.sync_all()) { - let _ = fs::remove_file(&self.path); + let _ = self.lock.filesystem()?.remove_file(&self.path); return Err(format!( "could not reconstruct {}: {error}", self.path.display() @@ -666,93 +622,36 @@ impl std::fmt::Display for LockError { impl SessionLock { fn acquire(path: PathBuf, force: bool) -> Result { - Self::acquire_with(path, force, |file, token| { - file.set_len(0) - .and_then(|_| file.write_all(token.as_bytes())) - .and_then(|_| file.sync_all()) - }) - } - - fn acquire_with( - path: PathBuf, - force: bool, - initialize: impl FnOnce(&mut File, &str) -> io::Result<()>, - ) -> Result { - Self::acquire_with_hook(path, force, || {}, initialize) - } - - fn acquire_with_hook( - path: PathBuf, - force: bool, - before_lock: impl FnOnce(), - initialize: impl FnOnce(&mut File, &str) -> io::Result<()>, - ) -> Result { - let token = format!("{}:{}:{}", std::process::id(), new_id(), SCHEMA_VERSION); - let mut options = OpenOptions::new(); - options.read(true).write(true); - #[cfg(windows)] - { - use std::os::windows::fs::OpenOptionsExt as _; - - // FILE_SHARE_READ | FILE_SHARE_WRITE. Omitting FILE_SHARE_DELETE - // prevents the lock pathname from being renamed or replaced while - // token checks read through this handle. - options.share_mode(0x0000_0001 | 0x0000_0002); - } - if force { - options.create(true); + let scope = path + .parent() + .ok_or_else(|| "session lock has no parent".to_string())?; + let mode = if force { + LeaseMode::ExistingOrNew } else { - options.create_new(true); - } - let mut file = options.open(&path).map_err(|error| { - if error.kind() == std::io::ErrorKind::AlreadyExists { - format!("session is locked by another Kit instance ({}); use --force to override a stale lock", path.display()) - } else { - format!("could not acquire session lock {}: {error}", path.display()) - } - })?; - before_lock(); - if file.try_lock().is_err() { - // A forced opener can take the OS lock after this process creates - // the pathname but before this call. It now owns that pathname, so - // the loser must close only and must never unlink it. - drop(file); - return Err(format!( - "session is actively locked by another Kit instance ({})", - path.display() - )); - } - if let Err(error) = initialize(&mut file, &token) { - // The OS lock proves this process owns mutation authority even for - // a forced stale-lock takeover. Close before removing so a failed - // token write cannot leave a corrupted path on Windows. - remove_failed_lock(&path, file); - return Err(format!("could not write session lock: {error}")); - } - Ok(Self { - path, - token, - file: Some(file), - }) + LeaseMode::CreateNew + }; + let lease = fs::global() + .acquire_lease_with_cleanup(&path, scope, mode, true) + .map_err(|error| { + if error.kind() == io::ErrorKind::AlreadyExists { + format!("session is locked by another Kit instance ({}); use --force to override a stale lock", path.display()) + } else if error.kind() == io::ErrorKind::WouldBlock { + format!("session is actively locked by another Kit instance ({})", path.display()) + } else { + format!("could not acquire session lock {}: {error}", path.display()) + } + })?; + Ok(Self { path, lease }) } - fn check(&self) -> Result<(), LockError> { - let current = self.read_token()?; - if current == self.token { - Ok(()) - } else { - Err(LockError::Other( - "session lock was overridden by another Kit instance".into(), - )) - } + fn filesystem(&self) -> Result { + fs::global() + .guarded(&self.lease) + .map_err(|error| format!("session lock was lost: {error}")) } - // Reads the lock file's token to confirm this process still owns it. On - // Unix the advisory lock lets a fresh handle read the path, which also - // reports the file being unlinked as `Missing` so the writer can recover. - #[cfg(not(windows))] - fn read_token(&self) -> Result { - fs::read_to_string(&self.path).map_err(|error| { + fn check(&self) -> Result<(), LockError> { + self.lease.check().map_err(|error| { if error.kind() == io::ErrorKind::NotFound { LockError::Missing } else { @@ -760,42 +659,6 @@ impl SessionLock { } }) } - - // On Windows `File::try_lock` takes a mandatory exclusive lock, so - // re-opening the path to read the token fails with a sharing violation - // (os error 33). Read it through the lock-owning handle instead, which the - // lock owner is always permitted to do. The lock file is opened without - // FILE_SHARE_DELETE, so it cannot be unlinked while this handle is held; a - // `Missing` state therefore cannot arise here and losing the handle is - // itself the lost-lock condition. - #[cfg(windows)] - fn read_token(&self) -> Result { - use std::io::Read; - let Some(file) = self.file.as_ref() else { - return Err(LockError::Missing); - }; - let mut handle: &File = file; - let mut current = String::new(); - handle - .seek(SeekFrom::Start(0)) - .and_then(|_| handle.read_to_string(&mut current)) - .map_err(|error| LockError::Other(format!("session lock was lost: {error}")))?; - Ok(current) - } -} - -fn remove_failed_lock(path: &Path, file: File) { - drop(file); - let _ = fs::remove_file(path); -} - -impl Drop for SessionLock { - fn drop(&mut self) { - if self.check().is_ok() { - drop(self.file.take()); - let _ = fs::remove_file(&self.path); - } - } } fn stamp_item(item: &mut Item, now: Timestamp) { @@ -943,7 +806,7 @@ fn read_record_lines( } fn canonical_workspace(root: &Path) -> PathBuf { - if let Ok(canonical) = root.canonicalize() { + if let Ok(canonical) = fs::canonicalize(root) { return canonical; } let mut ancestor = root.to_path_buf(); @@ -953,7 +816,7 @@ fn canonical_workspace(root: &Path) -> PathBuf { if !ancestor.pop() { return root.to_path_buf(); } - if let Ok(mut canonical) = ancestor.canonicalize() { + if let Ok(mut canonical) = fs::canonicalize(&ancestor) { for component in suffix.iter().rev() { canonical.push(component); } @@ -964,7 +827,7 @@ fn canonical_workspace(root: &Path) -> PathBuf { } fn normalized_absolute(path: &Path) -> Result { - path.canonicalize() + fs::canonicalize(path) .map_err(|error| format!("could not normalize {}: {error}", path.display())) } @@ -1051,13 +914,16 @@ fn set_display_name_in( display_name: Option<&str>, ) -> Result, String> { validate_id(session_id)?; - let root = root.canonicalize().map_err(|error| { + let root = fs::canonicalize(root).map_err(|error| { format!( "could not resolve workspace root {}: {error}", root.display() ) })?; - if !root.is_dir() { + if !fs::metadata(&root) + .map(|metadata| metadata.is_dir()) + .unwrap_or(false) + { return Err(format!( "workspace root {} is not a directory", root.display() @@ -1089,17 +955,12 @@ fn set_display_name_in( ) })?; let path = metadata_path(&directory, session_id); - atomicwrites::AtomicFile::new(&path, atomicwrites::AllowOverwrite) - .write(|file| { - file.write_all(&output)?; - file.sync_all() - }) - .map_err(|error| { - format!( - "could not replace session metadata {}: {error}", - path.display() - ) - })?; + fs::replace_private(&path, &output).map_err(|error| { + format!( + "could not replace session metadata {}: {error}", + path.display() + ) + })?; Ok(effective_title) } @@ -1153,13 +1014,16 @@ fn catalog_for_workspace( root: &Path, global_directory: &Path, ) -> Result, String> { - let root = root.canonicalize().map_err(|error| { + let root = fs::canonicalize(root).map_err(|error| { format!( "could not resolve workspace root {}: {error}", root.display() ) })?; - if !root.is_dir() { + if !fs::metadata(&root) + .map(|metadata| metadata.is_dir()) + .unwrap_or(false) + { return Err(format!( "workspace root {} is not a directory", root.display() @@ -1512,8 +1376,7 @@ fn select_authority_with( (&global, true, true), (&local, false, true), ] { - if !path - .try_exists() + if !fs::try_exists(path) .map_err(|error| format!("could not inspect {}: {error}", path.display()))? { continue; @@ -1637,19 +1500,43 @@ fn migration_source_workspace(path: &Path, session_id: &str) -> Result Result { + let lock_path = path.with_extension("lock"); + std::iter::once(scoped) + .chain(sources) + .find(|lock| lock.path == lock_path) + .ok_or_else(|| format!("no mutation authority for {}", path.display()))? + .filesystem() +} + fn recover_torn_migration_writes( scoped: &Path, directory: &Path, root: &Path, session_id: &str, + scoped_lock: &SessionLock, + source_locks: &[SessionLock], ) -> Result<(), String> { let mut paths = applicable_migration_sources(directory, root, session_id)?; paths.push(scoped.to_path_buf()); paths.sort(); paths.dedup(); for path in paths { - let exists = path - .try_exists() + // Redirects are sealed and absent paths need no migration. They do not + // justify creating fresh native lock files during a read-only resume. + if path != scoped + && !source_locks + .iter() + .any(|source| source.path == path.with_extension("lock")) + { + continue; + } + let filesystem = migration_filesystem(&path, scoped_lock, source_locks)?; + let exists = fs::try_exists(&path) .map_err(|error| format!("could not inspect {}: {error}", path.display()))?; if !exists { continue; @@ -1657,22 +1544,24 @@ fn recover_torn_migration_writes( let bytes = fs::read(&path) .map_err(|error| format!("could not read {}: {error}", path.display()))?; if bytes.is_empty() && path == scoped { - fs::remove_file(&path) + filesystem + .remove_file(&path) .map_err(|error| format!("could not remove {}: {error}", path.display()))?; - sync_parent_directory(&path)?; + sync_parent_directory(&filesystem, &path)?; continue; } let Some(complete) = torn_migration_tail_start(&bytes) else { continue; }; if complete == 0 && path == scoped { - fs::remove_file(&path) + filesystem + .remove_file(&path) .map_err(|error| format!("could not remove {}: {error}", path.display()))?; - sync_parent_directory(&path)?; + sync_parent_directory(&filesystem, &path)?; } else { let file = OpenOptions::new() .write(true) - .open(&path) + .open_in(&filesystem, &path) .map_err(|error| format!("could not open {}: {error}", path.display()))?; file.set_len(complete as u64) .and_then(|_| file.sync_all()) @@ -1687,24 +1576,16 @@ fn recover_torn_migration_writes( Ok(()) } -#[cfg(unix)] -fn sync_parent_directory(path: &Path) -> Result<(), String> { +fn sync_parent_directory(filesystem: &Fs, path: &Path) -> Result<(), String> { let parent = path .parent() .ok_or_else(|| format!("{} has no parent directory", path.display()))?; - File::open(parent) - .and_then(|directory| directory.sync_all()) - .map_err(|error| { - format!( - "could not sync session directory {}: {error}", - parent.display() - ) - }) -} - -#[cfg(not(unix))] -fn sync_parent_directory(_path: &Path) -> Result<(), String> { - Ok(()) + filesystem.sync_directory(parent).map_err(|error| { + format!( + "could not sync session directory {}: {error}", + parent.display() + ) + }) } fn applicable_migration_sources( @@ -1714,8 +1595,7 @@ fn applicable_migration_sources( ) -> Result, String> { let global = transcript_path(directory, session_id); let mut sources = vec![legacy_transcript(root, session_id)]; - let global_exists = global - .try_exists() + let global_exists = fs::try_exists(&global) .map_err(|error| format!("could not inspect {}: {error}", global.display()))?; if !global_exists || migration_source_workspace(&global, session_id)? @@ -1738,11 +1618,25 @@ fn lock_migration_sources( let mut locked_paths = Vec::new(); loop { let sources = applicable_migration_sources(directory, root, session_id)?; - let pending = sources - .into_iter() - .map(|path| path.with_extension("lock")) - .filter(|path| !locked_paths.contains(path)) - .collect::>(); + let mut pending = Vec::new(); + for source in sources { + let path = source.with_extension("lock"); + if locked_paths.contains(&path) { + continue; + } + let lock_exists = fs::try_exists(&path) + .map_err(|error| format!("could not inspect {}: {error}", path.display()))?; + let exists = fs::try_exists(&source) + .map_err(|error| format!("could not inspect {}: {error}", source.display()))?; + let sealed = exists + && matches!( + read_records_direct(&source, session_id), + Ok(StoredTranscript::Redirect(_)) + ); + if lock_exists || exists && !sealed { + pending.push(path); + } + } if pending.is_empty() { return Ok(locks); } @@ -1759,13 +1653,19 @@ fn lock_migration_sources( } } -fn write_migration_record(path: &Path, record: &Record, create: bool) -> Result<(), String> { - write_migration_record_with(path, record, create, |file, encoded| { +fn write_migration_record( + filesystem: &Fs, + path: &Path, + record: &Record, + create: bool, +) -> Result<(), String> { + write_migration_record_with(filesystem, path, record, create, |file, encoded| { file.write_all(encoded) }) } fn write_migration_record_with( + filesystem: &Fs, path: &Path, record: &Record, create: bool, @@ -1780,7 +1680,7 @@ fn write_migration_record_with( options.create_new(true); } let mut file = options - .open(path) + .open_in(filesystem, path) .map_err(|error| format!("could not open {}: {error}", path.display()))?; let original_len = file .metadata() @@ -1793,7 +1693,7 @@ fn write_migration_record_with( if let Err(error) = write(&mut file, &encoded).and_then(|_| file.sync_all()) { let rollback = if create { drop(file); - fs::remove_file(path) + filesystem.remove_file(path) } else { file.set_len(original_len).and_then(|_| file.sync_all()) }; @@ -1805,20 +1705,20 @@ fn write_migration_record_with( }; } if create { - sync_parent_directory(path)?; + sync_parent_directory(filesystem, path)?; } Ok(()) } fn establish_scoped_authority( + filesystem: &Fs, path: &Path, session_id: &str, root: &Path, items: &[Item], title_seed: Option<&[Item]>, ) -> Result<(), String> { - let exists = path - .try_exists() + let exists = fs::try_exists(path) .map_err(|error| format!("could not inspect {}: {error}", path.display()))?; let (mut generation, existing_items) = if exists { match read_records_direct(path, session_id)? { @@ -1850,6 +1750,7 @@ fn establish_scoped_authority( let mut create = !exists; if let Some(title_seed) = title_seed { write_migration_record( + filesystem, path, &Record { schema_version: SCHEMA_VERSION, @@ -1868,6 +1769,7 @@ fn establish_scoped_authority( create = false; } write_migration_record( + filesystem, path, &Record { schema_version: SCHEMA_VERSION, @@ -1883,6 +1785,7 @@ fn establish_scoped_authority( } fn redirect_legacy_transcript( + filesystem: &Fs, path: &Path, target: &Path, session_id: &str, @@ -1902,6 +1805,7 @@ fn redirect_legacy_transcript( } }; write_migration_record( + filesystem, path, &Record { schema_version: REDIRECT_SCHEMA_VERSION, @@ -1922,7 +1826,8 @@ fn legacy_transcript_for_workspace( session_id: &str, ) -> Result, String> { let global = transcript_path(directory, session_id); - if global.exists() + if fs::try_exists(&global) + .map_err(|error| format!("could not inspect {}: {error}", global.display()))? && transcript_workspace(&global, session_id)? .as_deref() .is_none_or(|stored| stored == root) @@ -1930,7 +1835,9 @@ fn legacy_transcript_for_workspace( return Ok(Some(global)); } let local = legacy_transcript(root, session_id); - if local.exists() { + if fs::try_exists(&local) + .map_err(|error| format!("could not inspect {}: {error}", local.display()))? + { read_records(&local, session_id)?; Ok(Some(local)) } else { @@ -1980,166 +1887,6 @@ mod tests { use agentkit_core::{ItemKind, MetadataMap, Part, ReasoningPart}; use serde_json::json; - #[test] - fn capacity_append_fallback_rolls_back_and_preserves_generations() { - // A short write and a sync failure (all bytes written) must both leave - // exactly the original disk prefix, including for quota exhaustion. - for kind in [io::ErrorKind::StorageFull, io::ErrorKind::QuotaExceeded] { - for bytes in [7, usize::MAX] { - let root = tempfile::tempdir().unwrap(); - let opened = open( - root.path(), - "fallback", - false, - false, - vec![Item::text(ItemKind::System, "system")], - ) - .unwrap(); - let mut writer = opened.observer.0.lock().unwrap(); - let prefix = fs::read(&writer.path).unwrap(); - let item = Item::text(ItemKind::User, "memory").with_created_at(Timestamp(123)); - writer.append_failure = Some((bytes, kind)); - writer.append(&item).unwrap(); - assert_eq!(writer.generation, 2); - writer.replace(std::slice::from_ref(&item)).unwrap(); - writer.append(&item).unwrap(); - assert_eq!(writer.generation, 4); - assert_eq!(fs::read(&writer.path).unwrap(), prefix); - let records: Vec = writer - .memory - .as_ref() - .unwrap() - .as_slice() - .split(|byte| *byte == b'\n') - .filter(|line| !line.is_empty()) - .map(|line| serde_json::from_slice(line).unwrap()) - .collect(); - assert_eq!( - records - .iter() - .map(|record| record.generation) - .collect::>(), - vec![2, 3, 4] - ); - assert!(records[0].item.is_some()); - assert_eq!(records[1].replacement.as_ref().unwrap().len(), 1); - assert!(records[1].item.is_none()); - assert!(records[2].item.is_some()); - } - } - } - - #[test] - fn observer_accepts_capacity_failure_without_panicking() { - let root = tempfile::tempdir().unwrap(); - let opened = open( - root.path(), - "observer", - false, - false, - vec![Item::text(ItemKind::System, "system")], - ) - .unwrap(); - opened.observer.0.lock().unwrap().append_failure = Some((7, io::ErrorKind::StorageFull)); - opened.observer.on_transcript_event(TranscriptEvent { - session_id: &agentkit_core::SessionId::new("observer"), - item: &Item::text(ItemKind::User, "still running").with_created_at(Timestamp(123)), - }); - let writer = opened.observer.0.lock().unwrap(); - assert_eq!(writer.generation, 2); - assert!(writer.memory.is_some()); - assert_eq!(read_records(&writer.path, "observer").unwrap().1, 1); - } - - #[test] - fn non_capacity_append_errors_never_enable_memory_fallback() { - for kind in [io::ErrorKind::PermissionDenied, io::ErrorKind::Other] { - for bytes in [7, usize::MAX] { - let root = tempfile::tempdir().unwrap(); - let opened = open( - root.path(), - "failure", - false, - false, - vec![Item::text(ItemKind::System, "system")], - ) - .unwrap(); - let mut writer = opened.observer.0.lock().unwrap(); - let prefix = fs::read(&writer.path).unwrap(); - writer.append_failure = Some((bytes, kind)); - assert!( - writer - .append( - &Item::text(ItemKind::User, "refused").with_created_at(Timestamp(123)) - ) - .is_err() - ); - assert!(writer.memory.is_none()); - assert_eq!(writer.generation, 1); - assert_eq!(fs::read(&writer.path).unwrap(), prefix); - } - } - } - - #[test] - fn capacity_fallback_does_not_relax_startup_or_rollback_failures() { - let root = tempfile::tempdir().unwrap(); - let opened = open( - root.path(), - "startup", - false, - false, - vec![Item::text(ItemKind::System, "system")], - ) - .unwrap(); - let mut writer = opened.observer.0.lock().unwrap(); - let prefix = fs::read(&writer.path).unwrap(); - let item = Item::text(ItemKind::User, "refused").with_created_at(Timestamp(123)); - writer.allow_memory = false; - writer.append_failure = Some((7, io::ErrorKind::StorageFull)); - assert!(writer.append(&item).is_err()); - assert!(writer.memory.is_none()); - assert_eq!(fs::read(&writer.path).unwrap(), prefix); - writer.allow_memory = true; - // A read-only handle deterministically rejects truncation. Inject the - // capacity error before writing to exercise failed rollback itself. - writer.file = File::open(&writer.path).unwrap(); - writer.append_failure = Some((0, io::ErrorKind::StorageFull)); - assert!(writer.append(&item).unwrap_err().contains("roll back")); - assert!(writer.memory.is_none()); - assert_eq!(writer.generation, 1); - } - - #[test] - fn memory_fallback_preserves_validation_and_lock_checks() { - let root = tempfile::tempdir().unwrap(); - let opened = open( - root.path(), - "locks", - false, - false, - vec![Item::text(ItemKind::System, "system")], - ) - .unwrap(); - let mut writer = opened.observer.0.lock().unwrap(); - let item = Item::text(ItemKind::User, "memory").with_created_at(Timestamp(123)); - writer.append_failure = Some((7, io::ErrorKind::StorageFull)); - writer.append(&item).unwrap(); - let memory_len = writer.memory.as_ref().unwrap().as_slice().len(); - assert!( - writer - .append(&Item::text(ItemKind::User, "no timestamp")) - .is_err() - ); - // Alter the expected token instead of unlinking an OS-locked file, - // which Windows correctly forbids. - writer.lock.token.push_str("-no-longer-owner"); - assert!(writer.append(&item).is_err()); - assert!(writer.replace(std::slice::from_ref(&item)).is_err()); - assert_eq!(writer.generation, 2); - assert_eq!(writer.memory.as_ref().unwrap().as_slice().len(), memory_len); - } - fn session_directory(root: &Path) -> PathBuf { root.join("sessions") } @@ -2261,53 +2008,15 @@ mod tests { let root = tempfile::tempdir().unwrap(); let failed = root.path().join("failed.jsonl"); fs::write(&failed, "partial").unwrap(); - drop(CreatedTranscript::new(failed.clone())); + drop(CreatedTranscript::new(failed.clone(), fs::global().clone())); assert!(!failed.exists()); let committed = root.path().join("committed.jsonl"); fs::write(&committed, "complete").unwrap(); - CreatedTranscript::new(committed.clone()).keep(); + CreatedTranscript::new(committed.clone(), fs::global().clone()).keep(); assert!(committed.exists()); } - #[test] - fn failed_lock_token_initialization_removes_new_lock_path() { - let root = tempfile::tempdir().unwrap(); - let path = root.path().join("failed.lock"); - let error = SessionLock::acquire_with(path.clone(), false, |_, _| { - Err(io::Error::other("injected token failure")) - }) - .err() - .expect("injected token initialization unexpectedly succeeded"); - assert!(error.contains("injected token failure")); - assert!(!path.exists(), "failed initialization left a lock path"); - - drop(SessionLock::acquire(path.clone(), false).unwrap()); - assert!(!path.exists()); - } - - #[test] - fn failed_forced_lock_token_initialization_removes_corrupted_path() { - let root = tempfile::tempdir().unwrap(); - let path = root.path().join("failed-force.lock"); - fs::write(&path, "stale owner").unwrap(); - - let error = SessionLock::acquire_with(path.clone(), true, |file, _| { - file.set_len(0)?; - Err(io::Error::other("injected forced token failure")) - }) - .err() - .expect("injected forced initialization unexpectedly succeeded"); - - assert!(error.contains("injected forced token failure")); - assert!( - !path.exists(), - "failed forced initialization left a lock path" - ); - drop(SessionLock::acquire(path.clone(), false).unwrap()); - assert!(!path.exists()); - } - #[cfg(windows)] #[test] fn windows_lock_path_cannot_be_renamed_while_held() { @@ -2325,30 +2034,6 @@ mod tests { assert!(!path.exists()); } - #[test] - fn non_force_lock_loser_does_not_unlink_forced_owner() { - let root = tempfile::tempdir().unwrap(); - let path = root.path().join("takeover.lock"); - let takeover = std::cell::RefCell::new(None); - - let error = SessionLock::acquire_with_hook( - path.clone(), - false, - || { - *takeover.borrow_mut() = Some(SessionLock::acquire(path.clone(), true).unwrap()); - }, - |file, token| file.write_all(token.as_bytes()), - ) - .err() - .expect("non-force opener unexpectedly retained the OS lock"); - - assert!(error.contains("actively locked")); - assert!(path.exists(), "lock loser unlinked the forced owner's path"); - assert!(takeover.borrow().as_ref().unwrap().check().is_ok()); - drop(takeover.into_inner()); - assert!(!path.exists()); - } - #[test] fn appends_versioned_generations_and_resumes() { let root = tempfile::tempdir().unwrap(); @@ -2783,7 +2468,7 @@ mod tests { assert_eq!(load(root.path(), "abc").unwrap(), vec![item.clone()]); assert!(!transcript_path(root.path(), "abc").exists()); - let legacy_lock = OpenOptions::new() + let legacy_lock = std::fs::OpenOptions::new() .read(true) .write(true) .create_new(true) @@ -2863,7 +2548,7 @@ mod tests { Some(canonical_workspace(&first)), ); let global_lock_path = global.with_extension("lock"); - let global_lock = OpenOptions::new() + let global_lock = std::fs::OpenOptions::new() .read(true) .write(true) .create_new(true) @@ -2905,10 +2590,16 @@ mod tests { redirect: Some(root.path().join("scoped/abc.jsonl")), }; - let error = write_migration_record_with(&path, &record, false, |file, encoded| { - file.write_all(&encoded[..encoded.len() / 2])?; - Err(io::Error::other("injected tombstone failure")) - }) + let error = write_migration_record_with( + &_lock.filesystem().unwrap(), + &path, + &record, + false, + |file, encoded| { + file.write_all(&encoded[..encoded.len() / 2])?; + Err(io::Error::other("injected tombstone failure")) + }, + ) .unwrap_err(); assert!(error.contains("injected tombstone failure")); @@ -2998,6 +2689,7 @@ mod tests { Item::text(ItemKind::Context, "compacted newer state").with_created_at(Timestamp(9)), ]; write_migration_record( + fs::global(), &scoped, &Record { schema_version: SCHEMA_VERSION, @@ -3113,6 +2805,7 @@ mod tests { generation = record.generation; } write_migration_record( + fs::global(), &global, &Record { schema_version: SCHEMA_VERSION, @@ -3142,7 +2835,7 @@ mod tests { write_history(&local, LEGACY_SCHEMA_VERSION, "abc", &["same"], None); for lock_path in [global.with_extension("lock"), local.with_extension("lock")] { - let lock = OpenOptions::new() + let lock = std::fs::OpenOptions::new() .read(true) .write(true) .create_new(true) @@ -4228,7 +3921,7 @@ mod tests { let error = opened.observer.0.lock().unwrap().append(&item).unwrap_err(); - assert!(error.contains("overridden by another Kit instance")); + assert!(error.contains("session lock was lost"), "{error}"); assert!(!transcript_path(root.path(), "abc").exists()); drop(other); } diff --git a/src/storage.rs b/src/storage.rs deleted file mode 100644 index 97f8bdf2..00000000 --- a/src/storage.rs +++ /dev/null @@ -1,210 +0,0 @@ -//! Bounded volatile storage for internal persistence failures. -//! -//! Capacity errors are distinct from permission, locking and integrity errors: -//! only the former may relax durability. This is not a virtual filesystem and -//! must not be used to claim a user-requested file edit succeeded. - -use std::{ - io::{self, Write}, - sync::atomic::{AtomicUsize, Ordering}, -}; - -const MAX_FALLBACK_BYTES: usize = 64 * 1024 * 1024; -static RESERVED_BYTES: Budget = Budget(AtomicUsize::new(0)); - -pub(crate) fn is_capacity_error(error: &io::Error) -> bool { - matches!( - error.kind(), - io::ErrorKind::StorageFull | io::ErrorKind::QuotaExceeded - ) -} - -struct Budget(AtomicUsize); - -impl Budget { - fn reserve(&self, additional: usize, limit: usize) -> io::Result<()> { - self.0 - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |used| { - used.checked_add(additional).filter(|next| *next <= limit) - }) - .map(|_| ()) - .map_err(|_| io::Error::from(io::ErrorKind::OutOfMemory)) - } - - fn release(&self, bytes: usize) { - self.0.fetch_sub(bytes, Ordering::Relaxed); - } -} - -/// An append-only buffer sharing a process-wide allocation budget. -/// -/// Growth is fallible, including during serde serialization through `Write`. -/// Dropping a buffer releases its reservation. No disk retry occurs implicitly: -/// callers must never append beyond a missing durable record. -#[derive(Default)] -pub(crate) struct MemoryBuffer { - bytes: Vec, - reserved: usize, -} - -impl MemoryBuffer { - /// Exit before a serializer can heap-allocate an error wrapper on OOM. - pub(crate) fn writer_or_exit(&mut self) -> impl Write + '_ { - ExitOnFailure(self) - } - - #[cfg(test)] - pub(crate) fn as_slice(&self) -> &[u8] { - &self.bytes - } - - fn reserve(&mut self, needed: usize) -> io::Result<()> { - if needed > self.reserved { - // Geometric growth avoids reallocating for every serializer token. - let target = needed - .checked_next_power_of_two() - .unwrap_or(needed) - .min(MAX_FALLBACK_BYTES) - .max(needed); - let additional = target - self.reserved; - RESERVED_BYTES.reserve(additional, MAX_FALLBACK_BYTES)?; - if self - .bytes - .try_reserve_exact(target - self.bytes.len()) - .is_err() - { - RESERVED_BYTES.release(additional); - return Err(io::ErrorKind::OutOfMemory.into()); - } - self.reserved = target; - } - Ok(()) - } -} - -impl Write for MemoryBuffer { - fn write(&mut self, bytes: &[u8]) -> io::Result { - let needed = self - .bytes - .len() - .checked_add(bytes.len()) - .ok_or(io::ErrorKind::OutOfMemory)?; - self.reserve(needed)?; - self.bytes.extend_from_slice(bytes); - Ok(bytes.len()) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -struct ExitOnFailure<'a>(&'a mut MemoryBuffer); - -impl Write for ExitOnFailure<'_> { - fn write(&mut self, bytes: &[u8]) -> io::Result { - match self.0.write(bytes) { - Ok(written) => Ok(written), - Err(_) => exit_exhausted(), - } - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -impl Drop for MemoryBuffer { - fn drop(&mut self) { - // Free the allocation before another thread can reuse its budget. - drop(std::mem::take(&mut self.bytes)); - RESERVED_BYTES.release(self.reserved); - } -} - -/// The observer API cannot return a persistence error to its caller. Stop -/// without unwinding or allocating an error report when volatile storage fills. -/// This cannot recover arbitrary allocator aborts elsewhere in the process. -pub(crate) fn exit_exhausted() -> ! { - crate::tui::restore_after_storage_failure(); - let _ = io::stderr().write_all( - b"kit: disk persistence failed and the memory fallback is exhausted; exiting. Unsaved session records will be lost.\n", - ); - std::process::exit(1) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn only_capacity_errors_allow_fallback() { - for kind in [io::ErrorKind::StorageFull, io::ErrorKind::QuotaExceeded] { - assert!(is_capacity_error(&kind.into())); - } - for kind in [ - io::ErrorKind::PermissionDenied, - io::ErrorKind::NotFound, - io::ErrorKind::WriteZero, - io::ErrorKind::Other, - ] { - assert!(!is_capacity_error(&kind.into())); - } - } - - #[test] - fn budget_rejects_overflow_and_releases_reservations() { - let budget = Budget(AtomicUsize::new(0)); - budget.reserve(8, 10).unwrap(); - assert!(budget.reserve(3, 10).is_err()); - assert!(budget.reserve(usize::MAX, usize::MAX).is_err()); - assert_eq!(budget.0.load(Ordering::Relaxed), 8); - budget.release(8); - budget.reserve(10, 10).unwrap(); - } - - #[test] - fn buffer_writes_and_failed_growth_preserves_content() { - let mut buffer = MemoryBuffer::default(); - buffer.write_all(b"hello").unwrap(); - buffer.write_all(b" world").unwrap(); - assert_eq!(buffer.as_slice(), b"hello world"); - // Exercise a budget refusal without allocating a huge input or relying - // on other concurrently running tests' reservations. - let reserved = buffer.reserved; - assert_eq!( - buffer.reserve(MAX_FALLBACK_BYTES + 1).unwrap_err().kind(), - io::ErrorKind::OutOfMemory - ); - assert_eq!(buffer.reserved, reserved); - assert_eq!(buffer.as_slice(), b"hello world"); - } - - #[test] - fn exhaustion_exits_without_a_panic() { - const CHILD_FLAG: &str = "KIT_TEST_STORAGE_EXHAUSTION_CHILD"; - if std::env::var_os(CHILD_FLAG).is_some() { - // Reserve the budget without actually filling memory. Exercise the - // serialization adapter's exit, not just the exit helper itself. - RESERVED_BYTES - .reserve(MAX_FALLBACK_BYTES, MAX_FALLBACK_BYTES) - .unwrap(); - let mut buffer = MemoryBuffer::default(); - let _ = serde_json::to_writer(buffer.writer_or_exit(), &"no budget left"); - unreachable!("the exhausted writer must exit"); - } - let output = std::process::Command::new(std::env::current_exe().unwrap()) - .args([ - "--exact", - "storage::tests::exhaustion_exits_without_a_panic", - "--nocapture", - ]) - .env(CHILD_FLAG, "1") - .output() - .unwrap(); - assert_eq!(output.status.code(), Some(1)); - let stderr = String::from_utf8_lossy(&output.stderr); - assert!(stderr.contains("memory fallback is exhausted"), "{stderr}"); - assert!(!stderr.contains("panicked"), "{stderr}"); - } -} diff --git a/src/storage_runtime.rs b/src/storage_runtime.rs new file mode 100644 index 00000000..64c6441e --- /dev/null +++ b/src/storage_runtime.rs @@ -0,0 +1,145 @@ +//! Process-lifetime recovery and shutdown coordination for internal storage. + +use std::{io::Write as _, sync::OnceLock, time::Duration}; + +use tokio_util::sync::CancellationToken; + +static SHUTDOWN: OnceLock = OnceLock::new(); +static WORKER: OnceLock<()> = OnceLock::new(); + +pub fn shutdown_token() -> &'static CancellationToken { + SHUTDOWN.get_or_init(CancellationToken::new) +} + +pub fn request_shutdown() { + shutdown_token().cancel(); +} + +/// Actual allocator failure cannot safely reach callers that format errors. +/// +/// Use only static writes and best-effort terminal cleanup, then skip destructors +/// and runtime teardown. This is not the configurable overlay-budget exit path. +fn noallocation_failure_exit() -> ! { + crate::tui::restore_after_allocation_failure(); + let _ = std::io::stderr().write_all( + b"kit: memory allocation failed; exiting immediately. Unpersisted internal storage changes will be lost.\n", + ); + std::process::exit(1); +} + +/// Stop background work and make one final bounded-queue recovery pass. +/// +/// Call synchronously after command work ends, before returning from main. This +/// does not wait for storage capacity to return or retry indefinitely. Passing +/// the service explicitly also permits isolated fault-injection tests. +pub fn finish_recovery(filesystem: &crate::resilient_fs::Fs) -> std::io::Result<()> { + request_shutdown(); + let _ = filesystem.recover(); + let status = filesystem.status(); + if status.pending_operations > 0 { + let _ = std::io::stderr().write_all( + b"kit: exiting with unpersisted internal storage changes; pending in-memory data will be lost at process exit.\n", + ); + return Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "pending internal storage changes could not be persisted before exit", + )); + } + if status.exhausted { + let _ = std::io::stderr().write_all( + b"kit: internal storage memory budget was exhausted; not all requested changes were accepted.\n", + ); + return Err(std::io::Error::other( + "internal storage budget exhausted before exit", + )); + } + Ok(()) +} + +/// Start one recovery worker for the process, independent of session handles. +/// A turn completing or a session closing does not stop this worker or discard +/// the filesystem's pending changes. +pub fn start_recovery_worker() { + // Register before starting any worker or initializing its filesystem. An + // existing application-provided handler takes precedence. + let _ = crate::resilient_fs::set_allocation_failure_handler(noallocation_failure_exit); + WORKER.get_or_init(|| { + let worker = std::thread::Builder::new() + .name("kit-storage-recovery".into()) + .spawn(|| { + let fs = crate::resilient_fs::global(); + let mut delay = 1; + let mut warned = false; + loop { + let status = fs.status(); + if status.exhausted { + let _ = std::io::stderr().write_all( + b"kit: internal storage memory budget exhausted; cancelling work and shutting down. Unpersisted data cannot survive process exit.\n", + ); + request_shutdown(); + } + if status.pending_operations > 0 { + if !warned { + let _ = std::io::stderr().write_all( + b"kit: internal storage is temporarily retained in memory; persistence will resume when disk storage recovers.\n", + ); + warned = true; + } + let report = fs.recover(); + if report.remaining_operations == 0 { + let _ = std::io::stderr().write_all( + b"kit: internal storage recovered; pending changes are persisted.\n", + ); + warned = false; + delay = 1; + } else { + delay = (delay * 2).min(30); + } + } else { + delay = 1; + } + if shutdown_token().is_cancelled() { + // One final best-effort pass; never discard pending data + // merely because a caller or observer was dropped. + let _ = fs.recover(); + return; + } + std::thread::sleep(Duration::from_secs(delay)); + } + }); + if worker.is_err() { + let _ = std::io::stderr().write_all( + b"kit: could not start internal storage recovery; shutting down.\n", + ); + request_shutdown(); + } + }); +} + +#[cfg(test)] +mod tests { + #[test] + fn allocation_failure_exit_is_nonzero_without_protocol_stdout() { + const CHILD: &str = "KIT_ALLOCATION_EXIT_TEST_CHILD"; + if std::env::var_os(CHILD).is_some() { + super::noallocation_failure_exit(); + } + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "storage_runtime::tests::allocation_failure_exit_is_nonzero_without_protocol_stdout", + "--nocapture", + ]) + .env(CHILD, "1") + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(1)); + assert_eq!( + output.stderr, + b"kit: memory allocation failed; exiting immediately. Unpersisted internal storage changes will be lost.\n" + ); + // The test harness writes its own banner, but no terminal escapes may + // contaminate a process that never entered the TUI. + assert!(!output.stdout.contains(&0x1b)); + } +} diff --git a/src/tools/artifact.rs b/src/tools/artifact.rs new file mode 100644 index 00000000..5e28956c --- /dev/null +++ b/src/tools/artifact.rs @@ -0,0 +1,234 @@ +use std::{ + io::{Read as _, Seek as _, SeekFrom}, + path::{Component, PathBuf}, +}; + +use agentkit_core::{ToolOutput, ToolResultPart}; +use agentkit_tools_core::{ + Tool, ToolAnnotations, ToolContext, ToolError, ToolName, ToolRequest, ToolResult, ToolSpec, +}; +use async_trait::async_trait; +use serde::Deserialize; +use serde_json::json; + +/// Reads Kit-owned output through the internal filesystem, not a shell path. +#[derive(Clone)] +pub struct ArtifactTool { + root: PathBuf, + spec: ToolSpec, +} + +impl ArtifactTool { + pub fn new(artifact_root: PathBuf) -> Self { + Self { + root: artifact_root, + spec: ToolSpec::new( + ToolName::new("artifact"), + "Read a UTF-8 Kit output artifact from this session, including artifacts temporarily retained in memory when disk storage fails. Use the artifact path returned by compose. Continue from next_offset to read another bounded chunk. Shell commands cannot read memory-only artifacts.", + json!({ + "type": "object", + "properties": { + "path": {"type": "string", "minLength": 1, "maxLength": 4096}, + "offset": {"type": "integer", "minimum": 0, "default": 0}, + "limit": {"type": "integer", "minimum": 4, "maximum": 1024, "default": 1024} + }, + "required": ["path"], + "additionalProperties": false + }), + ) + .with_output_schema(json!({ + "type": "object", + "properties": { + "content": {"type": "string"}, + "next_offset": {"type": "integer"}, + "total_bytes": {"type": "integer"}, + "eof": {"type": "boolean"} + }, + "required": ["content", "next_offset", "total_bytes", "eof"], + "additionalProperties": false + })) + .with_annotations(ToolAnnotations::read_only()), + } + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Input { + path: PathBuf, + #[serde(default)] + offset: u64, + #[serde(default = "default_limit")] + limit: usize, +} + +const fn default_limit() -> usize { + 1024 +} + +#[async_trait] +impl Tool for ArtifactTool { + fn spec(&self) -> &ToolSpec { + &self.spec + } + + async fn invoke( + &self, + request: ToolRequest, + _context: &mut ToolContext<'_>, + ) -> Result { + let input: Input = serde_json::from_value(request.input) + .map_err(|error| ToolError::InvalidInput(error.to_string()))?; + if !(4..=1024).contains(&input.limit) { + return Err(ToolError::InvalidInput( + "limit must be between 4 and 1024 bytes".into(), + )); + } + let root = crate::artifacts::session_directory(&self.root, &request.session_id.0); + let relative = input + .path + .strip_prefix(&root) + .map_err(|_| ToolError::InvalidInput("artifact must belong to this session".into()))? + .to_path_buf(); + if relative.as_os_str().is_empty() + || relative + .components() + .any(|part| !matches!(part, Component::Normal(_))) + { + return Err(ToolError::InvalidInput("invalid artifact path".into())); + } + let output = tokio::task::spawn_blocking(move || { + // Descriptor-relative no-follow access prevents a path or symlink + // from crossing the session's artifact namespace. + let mut file = crate::resilient_fs::open_beneath(&root, &relative) + .map_err(|error| ToolError::ExecutionFailed(error.to_string()))?; + let metadata = file + .metadata() + .map_err(|error| ToolError::ExecutionFailed(error.to_string()))?; + if !metadata.is_file() || input.offset > metadata.len() { + return Err(ToolError::InvalidInput( + "artifact or offset is invalid".into(), + )); + } + file.seek(SeekFrom::Start(input.offset)) + .map_err(|error| ToolError::ExecutionFailed(error.to_string()))?; + let mut bytes = Vec::with_capacity(input.limit); + file.take(input.limit as u64) + .read_to_end(&mut bytes) + .map_err(|error| ToolError::ExecutionFailed(error.to_string()))?; + let end = match std::str::from_utf8(&bytes) { + Ok(_) => bytes.len(), + Err(error) if error.error_len().is_none() => error.valid_up_to(), + Err(_) => { + return Err(ToolError::InvalidInput( + "artifact is not UTF-8, or offset is not a character boundary".into(), + )); + } + }; + if end < bytes.len() && input.offset + bytes.len() as u64 == metadata.len() { + return Err(ToolError::InvalidInput( + "artifact ends with incomplete UTF-8".into(), + )); + } + bytes.truncate(end); + let content = + String::from_utf8(bytes).map_err(|error| ToolError::Internal(error.to_string()))?; + let next_offset = input.offset + end as u64; + Ok(json!({ + "content": content, + "next_offset": next_offset, + "total_bytes": metadata.len(), + "eof": next_offset == metadata.len() + })) + }) + .await + .map_err(|error| ToolError::Internal(error.to_string()))??; + Ok(ToolResult::new(ToolResultPart::success( + request.call_id, + ToolOutput::structured(output), + ))) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use agentkit_core::{MetadataMap, SessionId, ToolCallId, TurnId}; + use agentkit_tools_core::{AllowAllPermissions, OwnedToolContext}; + + use super::*; + + #[tokio::test] + async fn reads_bounded_utf8_chunks_and_rejects_other_sessions() { + let directory = tempfile::tempdir().unwrap(); + let session = SessionId::new("artifact-session"); + let turn = TurnId::new("turn"); + let root = crate::artifacts::session_directory(directory.path(), &session.0); + let path = crate::artifacts::write(&root.join("call/output.json"), "abcé😀tail".as_bytes()) + .unwrap(); + let tool = ArtifactTool::new(directory.path().to_path_buf()); + let context = OwnedToolContext { + session_id: session.clone(), + turn_id: turn.clone(), + metadata: MetadataMap::new(), + permissions: Arc::new(AllowAllPermissions), + resources: Arc::new(()), + cancellation: None, + execution_scope: None, + approved_request: None, + }; + let request = |session, input| { + ToolRequest::new( + ToolCallId::new("read"), + ToolName::new("artifact"), + input, + session, + turn.clone(), + ) + }; + let result = tool + .invoke( + request(session.clone(), json!({"path":path,"limit":7})), + &mut context.borrowed(), + ) + .await + .unwrap(); + let ToolOutput::Structured(first) = result.result.output else { + panic!("structured output expected") + }; + assert_eq!(first["content"], "abcé"); + assert_eq!(first["next_offset"], 5); + let result = tool + .invoke( + request(session, json!({"path":path,"offset":5})), + &mut context.borrowed(), + ) + .await + .unwrap(); + let ToolOutput::Structured(last) = result.result.output else { + panic!("structured output expected") + }; + assert_eq!(last["content"], "😀tail"); + assert_eq!(last["eof"], true); + assert!( + tool.invoke( + request(SessionId::new("other-session"), json!({"path":path})), + &mut context.borrowed(), + ) + .await + .is_err() + ); + assert!( + tool.invoke( + request( + SessionId::new("artifact-session"), + json!({"path":root.join("../outside")}) + ), + &mut context.borrowed(), + ) + .await + .is_err() + ); + } +} diff --git a/src/tools/mcp.rs b/src/tools/mcp.rs index 8f15081c..6c63d099 100644 --- a/src/tools/mcp.rs +++ b/src/tools/mcp.rs @@ -755,7 +755,7 @@ fn contained_plugin_path(path: PathBuf, root: &Path, context: &str) -> Result break, Err(error) if error.kind() == std::io::ErrorKind::NotFound => { let component = ancestor @@ -772,7 +772,7 @@ fn contained_plugin_path(path: PathBuf, root: &Path, context: &str) -> Result None, }; + crate::resilient_fs::global() + .require_disk(&plugin.root) + .map_err(|error| { + format!("plugin MCP files are not available on disk: {error}") + })?; + crate::resilient_fs::global() + .require_disk(&plugin.data_dir) + .map_err(|error| { + format!("plugin MCP data directory is not available on disk: {error}") + })?; (McpTransportBinding::Stdio(transport), None, false) } PluginMcpTransport::StreamableHttp { url, headers } => { @@ -935,7 +945,11 @@ fn prepare_plugins( } async fn read_source(source: &ConfigSource) -> Result>, String> { - match tokio::fs::read(&source.path).await { + let path = source.path.clone(); + match tokio::task::spawn_blocking(move || crate::resilient_fs::read(path)) + .await + .map_err(|error| format!("MCP config read task failed: {error}"))? + { Ok(bytes) => Ok(Some(bytes)), Err(error) if !source.required && error.kind() == std::io::ErrorKind::NotFound => Ok(None), Err(error) => Err(format!( @@ -1007,7 +1021,11 @@ async fn connect_inner( interactive_oauth_enabled: bool, credential_storage: CredentialStorage, ) -> Result { - let (plugin_prepared, plugin_entries) = prepare_plugins(plugins)?; + let plugins = plugins.to_vec(); + let (plugin_prepared, plugin_entries) = + tokio::task::spawn_blocking(move || prepare_plugins(&plugins)) + .await + .map_err(|error| error.to_string())??; let mut source_states = Vec::with_capacity(sources.len()); for source in sources { let raw = read_source(&source).await?; @@ -1327,7 +1345,12 @@ impl McpRuntime { None => None, }; let (plugin_prepared, plugin_entries) = match &staged_plugins { - Some(staged) => prepare_plugins(&staged.resolved.mcp_plugins)?, + Some(staged) => { + let plugins = staged.resolved.mcp_plugins.clone(); + tokio::task::spawn_blocking(move || prepare_plugins(&plugins)) + .await + .map_err(|error| error.to_string())?? + } None => (current_plugins, current_plugin_entries), }; let mut next_sources = Vec::with_capacity(current_sources.len()); diff --git a/src/tools/mod.rs b/src/tools/mod.rs index 6a10fb0f..e776eff6 100644 --- a/src/tools/mod.rs +++ b/src/tools/mod.rs @@ -1,4 +1,5 @@ mod a2a; +mod artifact; mod docs; mod edit; pub(crate) mod mcp; @@ -8,6 +9,7 @@ mod subagent; pub use crate::credentials::CredentialStorage; pub use a2a::A2aTool; +pub use artifact::ArtifactTool; pub use docs::DocsTool; pub use edit::EditTool; pub use mcp::{AuthTool, McpTool, ToolSearch}; diff --git a/src/tui/mod.rs b/src/tui/mod.rs index 2318c222..712c5491 100644 --- a/src/tui/mod.rs +++ b/src/tui/mod.rs @@ -751,6 +751,18 @@ pub async fn run_with_reasoning_effort_and_openrouter_key( resume_session_id.is_some(), force, )?; + let child_mcp_config = mcp_config.clone(); + tokio::task::spawn_blocking(move || { + if let Some(home) = std::env::var_os("HOME").filter(|home| !home.is_empty()) { + crate::resilient_fs::global() + .require_disk(PathBuf::from(home).join(".kit/config.toml"))?; + } + if let Some(path) = child_mcp_config { + crate::resilient_fs::global().require_disk(path)?; + } + Ok::<_, std::io::Error>(()) + }) + .await??; let auth_invocation = AgentInvocation::from_command(command.as_std()); detach_from_controlling_terminal(&mut command); let mut child = command @@ -1114,12 +1126,14 @@ pub async fn run_with_reasoning_effort_and_openrouter_key( active.id = active_session_id.clone(); } app.start_session(active_session_id.clone()); + let storage_shutdown = crate::resilient_fs::shutdown_token(); let result: Result<(), agent_client_protocol::Error> = async { loop { terminal .draw(|frame| ui::draw(frame, &mut app, &mut images)) .map_err(agent_client_protocol::Error::into_internal_error)?; tokio::select! { + _ = storage_shutdown.cancelled() => return Ok(()), terminal_event = events.next() => { // A paste is a burst: one bracketed-paste event, or // thousands of key events where the terminal cannot @@ -1398,10 +1412,14 @@ pub async fn run_with_reasoning_effort_and_openrouter_key( ); app.note(format!("model changed to {} via {}", choice.model, choice.provider)); if save_defaults { - match save_model_defaults(&choice) { + let saved = { + let choice = choice.clone(); + tokio::task::spawn_blocking(move || save_model_defaults(&choice)).await.map_err(|error| error.to_string()).and_then(|result| result) + }; + match saved { Ok(()) => { saved_model_default = Some(choice.clone()); - app.note("saved model defaults to ~/.kit/config.toml"); + app.note(config_save_message("model defaults")); } Err(error) => app.note(format!("model changed, but defaults were not saved: {error}")), } @@ -1429,9 +1447,13 @@ pub async fn run_with_reasoning_effort_and_openrouter_key( "reasoning effort changed to {effort}" )); if save_defaults { - match save_effort_default(&effort) { + let saved = { + let effort = effort.clone(); + tokio::task::spawn_blocking(move || save_effort_default(&effort)).await.map_err(|error| error.to_string()).and_then(|result| result) + }; + match saved { Ok(()) => app.note( - "saved reasoning effort default to ~/.kit/config.toml", + config_save_message("reasoning effort default"), ), Err(error) => app.note(format!( "reasoning effort changed, but default was not saved: {error}" @@ -1651,6 +1673,14 @@ pub async fn run_with_reasoning_effort_and_openrouter_key( } } +fn config_save_message(setting: &str) -> String { + if crate::resilient_fs::global().status().pending_operations > 0 { + format!("updated {setting} in memory; disk persistence is pending (not durable)") + } else { + format!("saved {setting} to ~/.kit/config.toml") + } +} + fn save_effort_default(effort: &str) -> Result<(), String> { let home = std::env::var_os("HOME") .filter(|value| !value.is_empty()) @@ -1698,9 +1728,7 @@ fn update_config( path: &Path, update: impl FnOnce(&mut toml::map::Map), ) -> Result<(), String> { - use std::io::Write as _; - - let contents = match std::fs::read_to_string(path) { + let contents = match crate::resilient_fs::read_to_string(path) { Ok(value) => value, Err(error) if error.kind() == std::io::ErrorKind::NotFound => String::new(), Err(error) => return Err(format!("could not read {}: {error}", path.display())), @@ -1720,9 +1748,8 @@ fn update_config( let parent = path .parent() .ok_or_else(|| "config path has no parent".to_string())?; - std::fs::create_dir_all(parent).map_err(|error| error.to_string())?; - atomicwrites::AtomicFile::new(path, atomicwrites::AllowOverwrite) - .write(|file| file.write_all(output.as_bytes())) + crate::resilient_fs::create_dir_all(parent).map_err(|error| error.to_string())?; + crate::resilient_fs::replace(path, output.as_bytes()) .map_err(|error| format!("could not save {}: {error}", path.display())) } @@ -1941,6 +1968,7 @@ fn enable_tui_modes() { } fn enter() -> std::io::Result<(DefaultTerminal, image::ImageRuntime)> { + TERMINAL_ACTIVE.store(true, Ordering::Relaxed); let terminal = ratatui::try_init()?; // Query after entering the alternate screen but before the event stream owns // terminal input, as required by ratatui-image. The query has a short bound. @@ -1960,6 +1988,7 @@ fn enter() -> std::io::Result<(DefaultTerminal, image::ImageRuntime)> { } fn resume_terminal(terminal: &mut DefaultTerminal) -> std::io::Result { + TERMINAL_ACTIVE.store(true, Ordering::Relaxed); let resumed = (|| { crossterm::terminal::enable_raw_mode()?; execute!(std::io::stdout(), EnterAlternateScreen)?; @@ -1975,6 +2004,31 @@ fn resume_terminal(terminal: &mut DefaultTerminal) -> std::io::Result( } } -/// Restore terminal state before the allocation-free storage failure exit. -pub(crate) fn restore_after_storage_failure() { - if crossterm::terminal::is_raw_mode_enabled().unwrap_or(false) { - restore_modes(); - let _ = execute!(std::io::stdout(), crossterm::cursor::Show); - let _ = ratatui::try_restore(); - } -} - fn leave(terminal: &mut DefaultTerminal) { restore_modes(); let _ = terminal.show_cursor(); ratatui::restore(); + TERMINAL_ACTIVE.store(false, Ordering::Relaxed); } async fn request_resume( diff --git a/tests/resilient_exit.rs b/tests/resilient_exit.rs new file mode 100644 index 00000000..953d4b17 --- /dev/null +++ b/tests/resilient_exit.rs @@ -0,0 +1,64 @@ +//! Final process-exit recovery must not depend on the background worker waking. +#[path = "support/capacity.rs"] +mod capacity; + +use capacity::{Capacity, CapacityDisk}; +use kit::resilient_fs::{DiskBackend, Fs, finish_recovery}; +use std::{ + fs, io, + path::PathBuf, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +fn pending_replacement() -> (tempfile::TempDir, PathBuf, Arc, Fs) { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("transcript.jsonl"); + fs::write(&path, b"old generation\n").unwrap(); + let backend = Arc::new(Capacity { + exhausted: AtomicBool::new(true), + repaired: directory.path().join("capacity-repaired"), + }); + let filesystem = Fs::new(Arc::new(CapacityDisk(backend.clone()))); + filesystem.replace(&path, b"accepted generation\n").unwrap(); + let status = filesystem.status(); + assert!(status.pending_operations > 0); + assert!(!status.exhausted, "ordinary degradation is below budget"); + assert_eq!(fs::read(&path).unwrap(), b"old generation\n"); + (directory, path, backend, filesystem) +} + +#[test] +fn final_pass_persists_below_budget_changes_after_capacity_returns() { + let (_directory, path, backend, filesystem) = pending_replacement(); + backend.exhausted.store(false, Ordering::SeqCst); + // No facade reads or worker runs between repairing capacity and finalization. + finish_recovery(&filesystem).unwrap(); + assert_eq!(fs::read(path).unwrap(), b"accepted generation\n"); + assert_eq!(filesystem.status().pending_operations, 0); + assert!(kit::resilient_fs::shutdown_token().is_cancelled()); +} + +#[test] +fn final_pass_reports_undurable_changes_even_below_budget() { + let (_directory, path, _backend, filesystem) = pending_replacement(); + let error = finish_recovery(&filesystem).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::WriteZero); + assert!( + error + .to_string() + .contains("could not be persisted before exit") + ); + assert!(filesystem.status().pending_operations > 0); + assert!(!filesystem.status().exhausted); + assert_eq!(fs::read(path).unwrap(), b"old generation\n"); +} + +#[test] +fn final_pass_accepts_an_already_durable_service() { + let filesystem = Fs::new(Arc::new(DiskBackend)); + finish_recovery(&filesystem).unwrap(); + assert_eq!(filesystem.status().pending_operations, 0); +} diff --git a/tests/resilient_session.rs b/tests/resilient_session.rs new file mode 100644 index 00000000..69743fcf --- /dev/null +++ b/tests/resilient_session.rs @@ -0,0 +1,176 @@ +//! Exercise public session and tool APIs against a real-disk backend whose +//! capacity is unavailable until a normal shell tool repairs the condition. +use std::{ + fs, + path::PathBuf, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +use agentkit_core::{Item, ItemKind, MetadataMap, SessionId, Timestamp, ToolCallId, TurnId}; +use agentkit_loop::{TranscriptEvent, TranscriptObserver}; +use agentkit_tools_core::{AllowAllPermissions, OwnedToolContext, Tool, ToolName, ToolRequest}; +use kit::resilient_fs::{self, Fs}; +use serde_json::json; + +#[path = "support/capacity.rs"] +mod capacity; +use capacity::{Capacity, CapacityDisk}; + +#[test] +fn session_survives_outage_close_reopen_and_tool_driven_recovery() { + const CHILD: &str = "KIT_RESILIENT_SESSION_CHILD"; + if std::env::var_os(CHILD).is_none() { + let home = tempfile::tempdir().unwrap(); + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "session_survives_outage_close_reopen_and_tool_driven_recovery", + "--nocapture", + ]) + .env(CHILD, "1") + .env("HOME", home.path()) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return; + } + let home = PathBuf::from(std::env::var_os("HOME").unwrap()); + let root = home.join("project"); + fs::create_dir(&root).unwrap(); + let capacity = Arc::new(Capacity { + exhausted: AtomicBool::new(false), + repaired: home.join("repaired"), + }); + assert!( + resilient_fs::initialize_global(Fs::new(Arc::new(CapacityDisk(capacity.clone())))).is_ok() + ); + let id = SessionId::new("resilience"); + let opened = kit::session::open( + &root, + &id.0, + false, + false, + vec![Item::text(ItemKind::System, "system")], + ) + .unwrap(); + capacity.exhausted.store(true, Ordering::SeqCst); + opened.observer.on_transcript_event(TranscriptEvent { + session_id: &id, + item: &Item::text(ItemKind::User, "accepted during outage").with_created_at(Timestamp(123)), + }); + assert!(resilient_fs::global().status().pending_operations > 0); + drop(opened); + // Reopening uses retained state and retained real mutation authority, not + // a test-only SessionLock branch or a second independent memory transcript. + let reopened = kit::session::open(&root, &id.0, true, false, vec![]).unwrap(); + assert_eq!(reopened.transcript.len(), 2); + assert!( + kit::session::open(&root, &id.0, true, true, vec![]).is_err(), + "live observer must remain exclusive" + ); + let virtual_path = root.join("memory-only.txt"); + resilient_fs::write(&virtual_path, b"original").unwrap(); + assert!(!virtual_path.exists()); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + let turn = TurnId::new("repair"); + let context = OwnedToolContext { + session_id: id.clone(), + turn_id: turn.clone(), + metadata: MetadataMap::new(), + permissions: Arc::new(AllowAllPermissions), + resources: Arc::new(()), + cancellation: None, + execution_scope: None, + approved_request: None, + }; + let request = |name: &str, input| { + ToolRequest::new( + ToolCallId::new(name), + ToolName::new(name), + input, + id.clone(), + turn.clone(), + ) + }; + let edit = kit::tools::EditTool::new(root.clone()); + let result = edit.invoke(request("edit", json!({ + "op":"edit", "path":"memory-only.txt", "hunks":[{"old":"original","new":"changed"}] + })), &mut context.borrowed()).await; + assert!( + result.is_err(), + "explicit edit must fail on the real missing file, not mutate memfs" + ); + assert_eq!(resilient_fs::read(&virtual_path).unwrap(), b"original"); + let shell = kit::tools::ShellTool::new(root.clone()); + shell + .invoke( + request("shell", json!({"command":"echo repaired > ../repaired"})), + &mut context.borrowed(), + ) + .await + .unwrap(); + }); + assert!( + capacity.repaired.exists(), + "a real tool operation repaired the backend condition" + ); + reopened.observer.on_transcript_event(TranscriptEvent { + session_id: &id, + item: &Item::text(ItemKind::User, "accepted after repair").with_created_at(Timestamp(124)), + }); + drop(reopened); + for _ in 0..32 { + let report = resilient_fs::global().recover(); + assert!(report.blocked.is_none(), "{:?}", report.blocked); + if report.remaining_operations == 0 { + break; + } + } + resilient_fs::global().require_disk(&home).unwrap(); + assert_eq!(resilient_fs::global().status().pending_operations, 0); + assert_eq!(fs::read(&virtual_path).unwrap(), b"original"); + assert_eq!(kit::session::load(&root, &id.0).unwrap().len(), 3); + // Inspect actual disk, not the facade, then replay again and prove exactness. + let mut pending = vec![home.join(".kit/sessions")]; + let mut transcript = None; + while let Some(directory) = pending.pop() { + for entry in fs::read_dir(directory).unwrap() { + let entry = entry.unwrap(); + if entry.file_type().unwrap().is_dir() { + pending.push(entry.path()); + } else if entry.file_name() == "resilience.jsonl" { + transcript = Some(entry.path()); + } + } + } + let transcript = transcript.unwrap(); + let durable = fs::read_to_string(&transcript).unwrap(); + let records = durable + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .collect::>(); + assert_eq!(records.len(), 3); + assert_eq!( + records + .iter() + .map(|record| record["generation"].as_u64().unwrap()) + .collect::>(), + vec![1, 2, 3] + ); + assert_eq!(durable.matches("accepted during outage").count(), 1); + assert_eq!(durable.matches("accepted after repair").count(), 1); + assert_eq!(resilient_fs::global().recover().remaining_operations, 0); + assert_eq!(fs::read_to_string(transcript).unwrap(), durable); +} diff --git a/tests/resilient_shutdown.rs b/tests/resilient_shutdown.rs new file mode 100644 index 00000000..cf09559b --- /dev/null +++ b/tests/resilient_shutdown.rs @@ -0,0 +1,33 @@ +//! Keep the process-global exhaustion test separate from ordinary sessions. +use std::{ + io, + sync::{Arc, atomic::AtomicBool}, + time::Duration, +}; + +use kit::resilient_fs::{self as fs, Fs}; +#[path = "support/capacity.rs"] +mod capacity; +use capacity::{Capacity, CapacityDisk}; + +#[tokio::test] +async fn exhausted_storage_requests_orderly_process_shutdown() { + let directory = tempfile::tempdir().unwrap(); + let capacity = Arc::new(Capacity { + exhausted: AtomicBool::new(true), + repaired: directory.path().join("repair"), + }); + fs::initialize_global(Fs::with_budget(Arc::new(CapacityDisk(capacity)), 0, 0)).unwrap(); + fs::start_recovery_worker(); + let path = directory.path().join("cannot-retain"); + let error = fs::write(&path, b"too much for this budget").unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::OutOfMemory); + assert!(fs::global().status().exhausted); + tokio::time::timeout(Duration::from_secs(5), fs::shutdown_token().cancelled()) + .await + .expect("exhaustion must cancel the process, not panic or hang"); + assert!( + !path.exists(), + "a rejected write must not publish partial data" + ); +} diff --git a/tests/support/capacity.rs b/tests/support/capacity.rs new file mode 100644 index 00000000..8ecce63c --- /dev/null +++ b/tests/support/capacity.rs @@ -0,0 +1,143 @@ +use kit::resilient_fs::{ + self, Backend, BackendFile, BackendLease, DiskBackend, DiskEntry, DiskOpenOptions, LeaseRequest, +}; +use std::{ + fs, io, + io::{Read, Seek, SeekFrom, Write}, + path::{Path, PathBuf}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; +pub struct Capacity { + pub exhausted: AtomicBool, + pub repaired: PathBuf, +} +impl Capacity { + fn check(&self) -> io::Result<()> { + if self.exhausted.load(Ordering::SeqCst) && !self.repaired.exists() { + Err(io::ErrorKind::StorageFull.into()) + } else { + Ok(()) + } + } +} +pub struct CapacityDisk(pub Arc); +struct CapacityFile { + inner: Box, + capacity: Arc, +} +impl Read for CapacityFile { + fn read(&mut self, bytes: &mut [u8]) -> io::Result { + self.inner.read(bytes) + } +} +impl Seek for CapacityFile { + fn seek(&mut self, from: SeekFrom) -> io::Result { + self.inner.seek(from) + } +} +impl Write for CapacityFile { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.capacity.check()?; + self.inner.write(bytes) + } + fn flush(&mut self) -> io::Result<()> { + self.capacity.check()?; + self.inner.flush() + } +} +impl BackendFile for CapacityFile { + fn identity(&self) -> io::Result> { + self.inner.identity() + } + fn metadata(&self) -> io::Result { + self.inner.metadata() + } + fn set_len(&self, size: u64) -> io::Result<()> { + self.capacity.check()?; + self.inner.set_len(size) + } + fn sync_data(&self) -> io::Result<()> { + self.capacity.check()?; + self.inner.sync_data() + } + fn sync_all(&self) -> io::Result<()> { + self.capacity.check()?; + self.inner.sync_all() + } + fn set_permissions(&self, p: fs::Permissions) -> io::Result<()> { + self.capacity.check()?; + self.inner.set_permissions(p) + } +} +impl Backend for CapacityDisk { + fn identity( + &self, + path: &Path, + follow: bool, + ) -> io::Result> { + DiskBackend.identity(path, follow) + } + fn open(&self, path: &Path, options: &DiskOpenOptions) -> io::Result> { + if options.write + || options.append + || options.create + || options.create_new + || options.truncate + { + self.0.check()?; + } + Ok(Box::new(CapacityFile { + inner: DiskBackend.open(path, options)?, + capacity: self.0.clone(), + })) + } + fn metadata(&self, path: &Path, follow: bool) -> io::Result { + DiskBackend.metadata(path, follow) + } + fn read_dir(&self, path: &Path) -> io::Result> { + DiskBackend.read_dir(path) + } + fn read_link(&self, path: &Path) -> io::Result { + DiskBackend.read_link(path) + } + fn canonicalize(&self, path: &Path) -> io::Result { + DiskBackend.canonicalize(path) + } + fn create_dir(&self, path: &Path, private: bool) -> io::Result<()> { + self.0.check()?; + DiskBackend.create_dir(path, private) + } + fn remove_file(&self, path: &Path) -> io::Result<()> { + self.0.check()?; + DiskBackend.remove_file(path) + } + fn remove_dir(&self, path: &Path) -> io::Result<()> { + self.0.check()?; + DiskBackend.remove_dir(path) + } + fn rename(&self, from: &Path, to: &Path) -> io::Result<()> { + self.0.check()?; + DiskBackend.rename(from, to) + } + fn set_permissions(&self, path: &Path, p: fs::Permissions) -> io::Result<()> { + self.0.check()?; + DiskBackend.set_permissions(path, p) + } + fn sync_directory(&self, path: &Path) -> io::Result<()> { + self.0.check()?; + DiskBackend.sync_directory(path) + } + fn acquire_lease(&self, request: &LeaseRequest) -> io::Result> { + self.0.check()?; + DiskBackend.acquire_lease(request) + } + fn open_beneath(&self, root: &Path, relative: &Path) -> io::Result> { + Ok(Box::new(CapacityFile { + inner: DiskBackend.open_beneath(root, relative)?, + capacity: self.0.clone(), + })) + } +} From fba308834248422f7930b6c978b15fb62022cdf2 Mon Sep 17 00:00:00 2001 From: daniel Date: Sat, 5 Sep 2026 10:01:54 +0100 Subject: [PATCH 3/4] fix(storage): close recovery and shutdown review gaps --- docs/user/tui-and-sessions.md | 6 +- src/config_files.rs | 34 +++++ src/events.rs | 18 ++- src/lib.rs | 2 + src/main.rs | 83 ++++++++++- src/plugins.rs | 20 ++- src/protocols/acp.rs | 66 ++++++++- src/protocols/acp/v2.rs | 23 +-- src/resilient_fs/backend.rs | 2 + src/resilient_fs/mod.rs | 180 ++++++++++++++++++++---- src/resilient_fs/tests.rs | 255 +++++++++++++++++++++++++++++++++- src/runtime.rs | 2 +- src/runtime/tests.rs | 31 +++++ src/storage_runtime.rs | 21 ++- src/tools/mcp.rs | 94 ++++++++++++- src/tui/app.rs | 38 ++++- src/tui/mod.rs | 94 +++++++++++-- src/tui/ui.rs | 49 +++++++ tests/resilient_exit.rs | 1 + tests/resilient_session.rs | 70 ++++++++++ tests/resilient_shutdown.rs | 1 + tests/support/capacity.rs | 4 + 22 files changed, 1018 insertions(+), 76 deletions(-) create mode 100644 src/config_files.rs diff --git a/docs/user/tui-and-sessions.md b/docs/user/tui-and-sessions.md index 80ee162f..0a0baa83 100644 --- a/docs/user/tui-and-sessions.md +++ b/docs/user/tui-and-sessions.md @@ -27,11 +27,11 @@ A session ID must be 1–128 ASCII letters, digits, `-`, or `_`. `kit prompt` us Kit routes its internal persistence through a shared filesystem service. If a write fails because storage is full or a quota is exceeded, the service retains the pending change in a bounded memory overlay. Internal reads and session listings use the same view, so finishing a turn or closing a session handle does not discard accepted changes. An existing session can be reopened in the same running process while persistence is pending. -Recovery is automatic: internal operations retry pending work, and a process-owned worker retries with bounded backoff while Kit is idle. Once storage recovers, pending changes are persisted in order before newer changes can bypass them. Freeing space through a normal tool call can therefore restore persistence without restarting the session. Kit reports degradation and recovery on stderr. +Recovery is automatic: internal operations retry pending work, and a process-owned worker retries with bounded backoff while Kit is idle. Once storage recovers, pending changes are persisted in order before newer changes can bypass them. Freeing space through a normal tool call can therefore restore persistence without restarting the session. Kit reports degradation and recovery on stderr. The TUI also shows a persistent memory-only warning outside the optional log pane, including after switching sessions, until storage recovers. -**Pending memory state does not survive process termination.** A different process cannot read it. Keep Kit running until recovery completes if you need those changes to be durable. Before normal exit, Kit makes one final recovery pass; if accepted changes remain unpersisted, it warns and exits unsuccessfully. The default service bounds retained data to 64 MiB and pending operations to 4,096; exhaustion requests cancellation and orderly shutdown rather than silently evicting accepted data. If a fallible fallback allocation itself fails, Kit instead makes a best-effort terminal restoration and exits immediately without allocating an error message. +**Pending memory state does not survive process termination.** A different process cannot read it. Keep Kit running until recovery completes if you need those changes to be durable. Before normal exit, Kit makes one final recovery pass; if accepted changes remain unpersisted, it warns and exits unsuccessfully. The TUI waits for its agent process to finish recovery and reports an unsuccessful exit; if graceful shutdown times out, it warns of possible data loss before forcing termination. The default service bounds retained data to 64 MiB and pending operations to 4,096; exhaustion requests cancellation and orderly shutdown rather than silently evicting accepted data. If a fallible fallback allocation itself fails, Kit instead makes a best-effort terminal restoration and exits immediately without allocating an error message. -Explicit `edit` and shell operations still use the real filesystem and report real failures to the model. The fallback does not pretend those operations succeeded. Native consumers, such as plugin subprocesses, require their inputs to be materialized on disk, and interprocess locks still require real ownership. Unsafe paths, lost ownership, and non-capacity errors are not permission to overwrite another process's data. +Explicit `edit` and shell operations still use the real filesystem and report real failures to the model. Native consumers, such as plugin subprocesses, require their inputs to be materialized on disk, and interprocess locks still require real ownership. Unsafe paths, lost ownership, and non-capacity errors are not permission to overwrite another process's data. Use the `artifact` tool to read spilled output, including memory-only artifacts; a shell cannot see the memory overlay. diff --git a/src/config_files.rs b/src/config_files.rs new file mode 100644 index 00000000..30dda98a --- /dev/null +++ b/src/config_files.rs @@ -0,0 +1,34 @@ +//! Symlink-compatible reads for user-selected configuration and context files. + +use std::{io, path::Path}; + +// User-selected configuration files may be symlinks. Resolve only at this read +// boundary: managed storage continues to reject final symlinks. Resolve each +// target through the facade so even a memory-only target remains readable. +pub fn read_in(filesystem: &crate::resilient_fs::Fs, path: &Path) -> std::io::Result> { + let mut path = path.to_path_buf(); + for _ in 0..40 { + if !filesystem.symlink_metadata(&path)?.file_type().is_symlink() { + return filesystem.read(path); + } + let target = filesystem.read_link(&path)?; + path = if target.is_absolute() { + target + } else { + path.parent().unwrap_or_else(|| Path::new(".")).join(target) + }; + } + Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "too many symbolic links in configuration path", + )) +} + +pub fn read(path: &Path) -> io::Result> { + read_in(crate::resilient_fs::global(), path) +} + +pub fn read_to_string(path: &Path) -> io::Result { + String::from_utf8(read(path)?) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error)) +} diff --git a/src/events.rs b/src/events.rs index 40674fe0..bf59995f 100644 --- a/src/events.rs +++ b/src/events.rs @@ -36,6 +36,8 @@ pub const EVENTS_ENV: &str = "KIT_RUNTIME_EVENTS"; #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(tag = "event", rename_all = "snake_case")] pub enum RuntimeEvent { + /// Process-wide durability state, independent of the active ACP session. + StorageStatus { pending: bool, exhausted: bool }, /// A persisted ACP session was opened by the child runtime. SessionStarted { session_id: String }, /// A nested tool call started running. @@ -115,7 +117,8 @@ impl RuntimeEvent { pub fn parent_call(&self) -> Option<&str> { let call = match self { Self::ChildStarted { call, .. } | Self::ChildFinished { call, .. } => call, - Self::SessionStarted { .. } + Self::StorageStatus { .. } + | Self::SessionStarted { .. } | Self::CompactionStarted { .. } | Self::CompactionFinished { .. } | Self::SubagentStateChanged { .. } @@ -228,6 +231,19 @@ mod tests { summarize_output, write_event, }; + #[test] + fn storage_status_events_round_trip_without_session_affinity() { + for (pending, exhausted) in [(true, false), (false, false), (true, true)] { + let event = RuntimeEvent::StorageStatus { pending, exhausted }; + let mut wire = Vec::new(); + write_event(&mut wire, &event); + let wire = String::from_utf8(wire).unwrap(); + let parsed = parse(wire.trim_end()).unwrap(); + assert_eq!(parsed, event); + assert_eq!(parsed.parent_call(), None); + } + } + #[test] fn reads_back_an_emitted_event_line() { let event = RuntimeEvent::ChildStarted { diff --git a/src/lib.rs b/src/lib.rs index 116d8639..b800e72e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,8 @@ mod acp_child; mod artifacts; pub mod compaction; mod compose_output; +#[doc(hidden)] +pub mod config_files; mod credentials; pub mod docs; pub mod events; diff --git a/src/main.rs b/src/main.rs index 25f17a06..4d06877b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -274,7 +274,7 @@ impl Config { env::current_dir()?.join(path) }; let config_dir = absolute_parent(&config_path)?; - let contents = match fs::read_to_string(path) { + let contents = match kit::config_files::read_to_string(path) { Ok(contents) => contents, Err(error) if error.kind() == io::ErrorKind::NotFound => { return Ok(Self { @@ -1272,6 +1272,28 @@ mod tests { supervise_serve_with_trigger, validate_auth_storage, }; + #[cfg(unix)] + #[tokio::test] + async fn config_symlink_loads_and_initializes_plugins() { + let directory = tempfile::tempdir().unwrap(); + let target = directory.path().join("settings.toml"); + let link = directory.path().join("config.toml"); + fs::write(&target, "model = \"gpt-5.6-sol\"\n[plugins]\n").unwrap(); + std::os::unix::fs::symlink("settings.toml", &link).unwrap(); + let config = Config::load(&link).unwrap(); + assert_eq!(config.config_path.as_deref(), Some(link.as_path())); + assert!( + config + .plugin_runtime(directory.path()) + .await + .unwrap() + .is_some() + ); + fs::write(&target, "invalid TOML [").unwrap(); + assert!(Config::load(&link).is_err()); + assert!(config.plugin_runtime(directory.path()).await.is_err()); + } + #[test] fn subagent_names_are_rejected_as_unknown_configuration() { let directory = tempfile::tempdir().unwrap(); @@ -2103,6 +2125,65 @@ future_option = true ); } + #[tokio::test] + async fn stdio_eof_stops_a2a_and_completes_supervision() { + const CHILD: &str = "KIT_TEST_STDIO_EOF_CHILD"; + if std::env::var_os(CHILD).is_none() { + for version in ["1", "2"] { + let mut command = tokio::process::Command::new(std::env::current_exe().unwrap()); + command + .args([ + "--exact", + "tests::stdio_eof_stops_a2a_and_completes_supervision", + "--nocapture", + ]) + .env(CHILD, version) + .stdin(std::process::Stdio::null()) + .kill_on_drop(true); + let output = tokio::time::timeout(Duration::from_secs(5), command.output()) + .await + .expect("ACP EOF must stop serve even with an A2A listener") + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + } + return; + } + let root = tempfile::tempdir().unwrap(); + let runtime = kit::Runtime::new(root.path(), "gpt-5.4").unwrap(); + let sessions = kit::protocols::acp::SessionRegistry::new(); + let http = kit::protocols::http::start_with_registry( + Arc::clone(&runtime), + "127.0.0.1:0".into(), + true, + false, + None, + sessions.clone(), + ) + .await + .unwrap(); + let address = http.address(); + supervise_serve_with_trigger( + runtime, + sessions, + false, + if std::env::var(CHILD).unwrap() == "1" { + super::AcpProtocolVersion::V1 + } else { + super::AcpProtocolVersion::V2 + }, + http, + std::future::pending(), + ) + .await + .unwrap(); + // The final recovery in main can now run; no listener keeps serve alive. + let _rebound = tokio::net::TcpListener::bind(address).await.unwrap(); + } + #[tokio::test] async fn injected_shutdown_stops_no_stdio_supervisor() { let root = tempfile::tempdir().unwrap(); diff --git a/src/plugins.rs b/src/plugins.rs index 5d34e3ce..2c92579c 100644 --- a/src/plugins.rs +++ b/src/plugins.rs @@ -343,7 +343,7 @@ impl PluginRuntime { } pub(crate) async fn stage(&self) -> Result { - let contents = match fs::read_to_string(&self.inner.config_path) { + let contents = match crate::config_files::read_to_string(&self.inner.config_path) { Ok(contents) => contents, Err(error) if error.kind() == io::ErrorKind::NotFound => String::new(), Err(error) => { @@ -5125,11 +5125,19 @@ mod tests { object_store_limit: Some(&objects), }); invalidator.join().unwrap(); - assert_eq!( - result, - Err(GitFailure::ObjectStoreInspection( - io::ErrorKind::InvalidData - )) + // Inspection can observe either the replacement file or the race + // between metadata and read_dir (including the remove/create gap). + // Every case must stop the live Git process rather than bypass limits. + assert!( + matches!( + result, + Err(GitFailure::ObjectStoreInspection( + io::ErrorKind::InvalidData + | io::ErrorKind::NotADirectory + | io::ErrorKind::NotFound + )) + ), + "{result:?}" ); assert!(started.elapsed() < Duration::from_secs(4)); } diff --git a/src/protocols/acp.rs b/src/protocols/acp.rs index 35a7c33c..6e4ea8ac 100644 --- a/src/protocols/acp.rs +++ b/src/protocols/acp.rs @@ -2154,8 +2154,69 @@ async fn drive_until_pause( } } +/// The SDK can retain its outgoing task after input EOF. Observe EOF ourselves +/// so the serve supervisor can stop HTTP and perform final storage recovery. +async fn connect_stdio(component: impl ConnectTo) -> Result<(), AcpRuntimeError> { + use std::io::{BufRead as _, Write as _}; + // A dedicated OS thread does not hold Tokio runtime teardown hostage if a + // termination signal arrives while stdin is idle. + let (send, receive) = tokio::sync::mpsc::channel(16); + std::thread::Builder::new() + .name("kit-acp-stdin".into()) + .spawn(move || { + for line in std::io::stdin().lock().lines() { + if send.blocking_send(line).is_err() { + break; + } + } + }) + .map_err(|error| AcpRuntimeError::Sdk(error.to_string()))?; + let eof = tokio_util::sync::CancellationToken::new(); + let incoming = + futures_util::stream::unfold((receive, eof.clone()), async |(mut receive, eof)| { + match receive.recv().await { + Some(line) => Some((line, (receive, eof))), + None => { + eof.cancel(); + None + } + } + }); + let (send, mut receive) = tokio::sync::mpsc::channel::<( + String, + tokio::sync::oneshot::Sender>, + )>(1); + std::thread::Builder::new() + .name("kit-acp-stdout".into()) + .spawn(move || { + while let Some((line, reply)) = receive.blocking_recv() { + let mut stdout = std::io::stdout().lock(); + let result = writeln!(stdout, "{line}").and_then(|()| stdout.flush()); + let failed = result.is_err(); + let _ = reply.send(result); + if failed { + break; + } + } + }) + .map_err(|error| AcpRuntimeError::Sdk(error.to_string()))?; + let outgoing = futures_util::sink::unfold(send, async |send, line: String| { + let (reply, result) = tokio::sync::oneshot::channel(); + send.send((line, reply)) + .await + .map_err(std::io::Error::other)?; + result.await.map_err(std::io::Error::other)??; + Ok::<_, std::io::Error>(send) + }); + let transport = agent_client_protocol::Lines::new(Box::pin(outgoing), Box::pin(incoming)); + tokio::select! { + result = component.connect_to(transport) => result.map_err(|error| AcpRuntimeError::Sdk(error.to_string())), + _ = eof.cancelled() => Ok(()), + } +} + pub async fn serve(runtime: Arc) -> Result<(), AcpRuntimeError> { - serve_transport(runtime, agent_client_protocol::Stdio::new()).await + serve_with_registry(runtime, SessionRegistry::new()).await } pub async fn serve_with_registry( @@ -2165,13 +2226,14 @@ pub async fn serve_with_registry( let component = component(runtime, registry.clone())?; let shutdown = crate::resilient_fs::shutdown_token(); let result = tokio::select! { - result = component.connect_to(agent_client_protocol::Stdio::new()) => result.map_err(|error| AcpRuntimeError::Sdk(error.to_string())), + result = connect_stdio(component) => result, _ = shutdown.cancelled() => Ok(()), }; registry.shutdown().await; result } +#[cfg(test)] async fn serve_transport( runtime: Arc, transport: impl ConnectTo + 'static, diff --git a/src/protocols/acp/v2.rs b/src/protocols/acp/v2.rs index 5548a467..5d40c4cd 100644 --- a/src/protocols/acp/v2.rs +++ b/src/protocols/acp/v2.rs @@ -1838,30 +1838,17 @@ fn replay_tool_output_content(output: &ToolOutput) -> Option) -> Result<(), AcpRuntimeError> { - serve_transport(runtime, agent_client_protocol::Stdio::new()).await + let registry = SessionRegistry::new(); + let result = serve_with_registry(runtime, registry.clone()).await; + registry.shutdown().await; + result } pub async fn serve_with_registry( runtime: Arc, registry: SessionRegistry, ) -> Result<(), AcpRuntimeError> { - v2_router(runtime, registry)? - .connect_to(agent_client_protocol::Stdio::new()) - .await - .map_err(|error| AcpRuntimeError::Sdk(error.to_string())) -} - -async fn serve_transport( - runtime: Arc, - transport: impl ConnectTo + 'static, -) -> Result<(), AcpRuntimeError> { - let registry = SessionRegistry::new(); - let result = v2_router(runtime, registry.clone())? - .connect_to(transport) - .await - .map_err(|error| AcpRuntimeError::Sdk(error.to_string())); - registry.shutdown().await; - result + super::connect_stdio(v2_router(runtime, registry)?).await } pub(crate) fn http_router(runtime: Arc, registry: SessionRegistry) -> axum::Router { diff --git a/src/resilient_fs/backend.rs b/src/resilient_fs/backend.rs index 4bcff97f..838fd9c0 100644 --- a/src/resilient_fs/backend.rs +++ b/src/resilient_fs/backend.rs @@ -448,6 +448,8 @@ mod native { } // SAFETY: successful fstatat initialized the structure. let info = unsafe { info.assume_init() }; + // dev_t is signed on macOS and u64 on Linux. + #[allow(clippy::unnecessary_cast)] Ok(FileIdentity { volume: info.st_dev as u64, file: info.st_ino, diff --git a/src/resilient_fs/mod.rs b/src/resilient_fs/mod.rs index c9761cc5..88542716 100644 --- a/src/resilient_fs/mod.rs +++ b/src/resilient_fs/mod.rs @@ -504,7 +504,6 @@ impl Fs { ) -> io::Result { let path = self.norm(path.as_ref())?; let scope = self.norm(scope.as_ref())?; - self.require_disk(&path)?; let mut state = lock(&self.service.state); state .leases @@ -540,6 +539,18 @@ impl Fs { return Ok(Lease { inner: owner }); } } + // Retained authority precedes recovery: parent sync touches the lock. + let report = self.recover_locked(&mut state); + self.rebase(&mut state); + Self::prune(&mut state); + if state.pending.iter().any(|p| p.action.touches(&path)) { + return Err(report.blocked.unwrap_or_else(|| { + error( + io::ErrorKind::WouldBlock, + "bounded recovery has pending work", + ) + })); + } state.leases.try_reserve(1).map_err(|_| allocation_oom())?; let native = self.service.backend.acquire_lease(&LeaseRequest { path: path.clone(), @@ -768,11 +779,37 @@ impl Fs { ..Default::default() }, )?; - let object = Arc::new(Mutex::new(Object::native(native, path.to_path_buf())?)); + let opened = Object::native(native, path.to_path_buf())?; + // Share logical identity only while the named inode still matches. + // Explicit and external replacements must remain distinct. + for object in s.objects.iter().filter_map(|w| w.upgrade()) { + let held = lock(&object); + if held.path.as_deref() == Some(path) + && same_disk_identity(held.meta.disk_identity, opened.meta.disk_identity) + { + drop(held); + return Ok(object); + } + } + let object = Arc::new(Mutex::new(opened)); s.objects.try_reserve(1).map_err(|_| allocation_oom())?; s.objects.push(Arc::downgrade(&object)); Ok(object) } + fn live_object(&self, s: &State, path: &Path) -> io::Result> { + if let Some(entry) = s.entries.iter().find(|e| e.path == path) { + return Ok(entry.object.clone()); + } + let identity = match self.service.backend.identity(path, false) { + Ok(identity) => identity, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e), + }; + Ok(s.objects.iter().filter_map(|w| w.upgrade()).find(|o| { + let o = lock(o); + o.path.as_deref() == Some(path) && same_disk_identity(o.meta.disk_identity, identity) + })) + } fn entry(s: &mut State, path: PathBuf, object: Option) { if let Some(o) = &object { lock(o).path = Some(path.clone()); @@ -797,8 +834,70 @@ impl Fs { ) -> io::Result { #[cfg(unix)] { - let _ = (s, path); - Ok(permissions(private, dir)) + if private || dir { + return Ok(permissions(private, dir)); + } + // Let the kernel apply umask without changing process-global state. + // Probe an empty exclusive file; never broaden its mode. + let mut parent = path + .parent() + .ok_or_else(|| error(io::ErrorKind::InvalidInput, "no parent"))?; + while s.entries.iter().any(|e| { + e.path == parent && e.object.as_ref().is_some_and(|o| lock(o).meta.is_dir()) + }) && matches!(self.service.backend.metadata(parent, false), + Err(e) if e.kind() == io::ErrorKind::NotFound) + { + parent = parent + .parent() + .ok_or_else(|| error(io::ErrorKind::InvalidInput, "no disk ancestor"))?; + } + for _ in 0..8 { + let mut random = [0u8; 16]; + getrandom::fill(&mut random).map_err(io::Error::other)?; + let probe = parent.join(format!( + ".kit-mode-{:032x}.tmp", + u128::from_ne_bytes(random) + )); + match self.service.backend.open( + &probe, + &DiskOpenOptions { + write: true, + create_new: true, + ..Default::default() + }, + ) { + Ok(file) => { + let result = file.metadata().map(|m| m.permissions()); + // Never unlink another actor's replacement, including a + // symlink. Keep the descriptor alive through cleanup. + if !same_disk_identity( + file.identity()?, + self.service.backend.identity(&probe, false)?, + ) { + return Err(error( + io::ErrorKind::PermissionDenied, + "permission probe identity changed", + )); + } + match self.service.backend.remove_file(&probe) { + Ok(()) => {} + // An empty probe may remain, but no user bytes are + // exposed and capacity failure must permit fallback. + Err(e) if capacity(&e) => {} + Err(e) => return Err(e), + } + return result; + } + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue, + // Only capacity failure allows fallback; use a restrictive mode. + Err(e) if capacity(&e) => return Ok(permissions(true, false)), + Err(e) => return Err(e), + } + } + Err(error( + io::ErrorKind::AlreadyExists, + "permission probe collisions", + )) } #[cfg(not(unix))] { @@ -1002,8 +1101,9 @@ impl Fs { len, patches: Arc::new(Vec::new()), }; - object.meta.disk_identity = disk_identity; - object.meta.disk = Some(meta); + let identity = object.meta.identity; + object.meta = Metadata::disk(meta, disk_identity); + object.meta.identity = identity; object.dirty = false; } } else if let Ok(native) = self.service.backend.open( @@ -1017,8 +1117,9 @@ impl Fs { && let Ok(image) = Image::native(native) { object.image = image; - object.meta.disk_identity = disk_identity; - object.meta.disk = Some(meta); + let identity = object.meta.identity; + object.meta = Metadata::disk(meta, disk_identity); + object.meta.identity = identity; object.dirty = false; } } @@ -1318,10 +1419,7 @@ impl Fs { }; let data = Arc::new(bytes(contents)?); let object = if !new_object { - s.entries - .iter() - .find(|e| e.path == path) - .and_then(|e| e.object.clone()) + self.live_object(&s, &path)? } else { None }; @@ -1974,11 +2072,7 @@ impl OpenOptions { permissions: perms.clone(), modified: SystemTime::now(), }; - let obj = s - .entries - .iter() - .find(|e| e.path == p) - .and_then(|e| e.object.clone()); + let obj = fs.live_object(&s, &p)?; let a = Fs::put_action(&mut s, &p, Image::memory(data.clone()), perms); let (accepted, result) = fs.submit(&mut s, a); if !accepted { @@ -2039,7 +2133,7 @@ impl File { } pub fn metadata(&self) -> io::Result { let object = lock(&self.object); - if object.image.patches.is_empty() + if !object.dirty && let Source::Native(file) = &object.image.source { let file = lock(file); @@ -2167,15 +2261,36 @@ impl File { pub fn set_permissions(&self, p: Permissions) -> io::Result<()> { let mut s = lock(&self.fs.service.state); self.fs.recover_before(&mut s)?; - let path = lock(&self.object).path.clone(); - let (accepted, result) = if let Some(path) = path { - self.fs.authority(&path)?; - self.fs.secure_path(&s, &path, false)?; - Fs::prepare(&mut s, 0)?; - self.fs.enqueue( + let path = { + let mut object = lock(&self.object); + if let Some(path) = &object.path + && !s.pending.iter().any(|p| p.action.touches(path)) + && let Source::Native(native) = &object.image.source + { + let held = lock(native).identity()?; + match self.fs.service.backend.identity(path, false) { + Ok(named) if same_disk_identity(held, named) => {} + Ok(None) => { + return Err(error( + io::ErrorKind::PermissionDenied, + "native identity unavailable", + )); + } + Ok(_) => object.path = None, + Err(e) if e.kind() == io::ErrorKind::NotFound => object.path = None, + Err(e) => return Err(e), + } + } + object.path.clone() + }; + let (accepted, result) = if let Some(path) = &path { + self.fs.authority(path)?; + self.fs.secure_path(&s, path, false)?; + Fs::prepare(&mut s, 1)?; + self.fs.submit( &mut s, Action::Chmod { - path, + path: path.clone(), permissions: p.clone(), }, ) @@ -2189,6 +2304,12 @@ impl File { let mut object = lock(&self.object); object.meta.permissions = p; object.meta.disk = None; + object.dirty = true; + drop(object); + if let Some(path) = path { + Fs::entry(&mut s, path, Some(self.object.clone())); + } + self.fs.rebase(&mut s); } result } @@ -2202,10 +2323,11 @@ impl Read for File { )); } let mut cursor = lock(&self.cursor); - let image = lock(&self.object).image.clone(); - let n = if image.patches.is_empty() - && let Source::Native(file) = &image.source - { + let (image, dirty) = { + let object = lock(&self.object); + (object.image.clone(), object.dirty) + }; + let n = if !dirty && let Source::Native(file) = &image.source { let mut file = lock(file); file.seek(SeekFrom::Start(*cursor))?; file.read(buf)? diff --git a/src/resilient_fs/tests.rs b/src/resilient_fs/tests.rs index 58c83cb9..bbae0b49 100644 --- a/src/resilient_fs/tests.rs +++ b/src/resilient_fs/tests.rs @@ -11,9 +11,13 @@ enum Point { Rename, Mkdir, DirectorySync, + Chmod, + Remove, + ProbeCollision, + ProbeSwap, } #[derive(Default)] -struct Faults(Mutex>); +struct Faults(Mutex>, AtomicUsize); impl Faults { fn arm(&self, point: Point, code: i32) { *self.0.lock().unwrap() = Some((point, code)); @@ -92,8 +96,20 @@ impl Backend for Injected { if o.write || o.append { self.faults.check(Point::Open)?; } + let probe = p + .file_name() + .is_some_and(|n| n.to_string_lossy().starts_with(".kit-mode-")); + if probe { + self.faults.1.fetch_add(1, Ordering::Relaxed); + self.faults.check(Point::ProbeCollision)?; + } + let disk = self.disk.open(p, o)?; + if probe && self.faults.check(Point::ProbeSwap).is_err() { + native::rename(p, p.with_extension("held")).unwrap(); + native::write(p, b"replacement").unwrap(); + } Ok(Box::new(InjectedFile { - disk: self.disk.open(p, o)?, + disk, faults: self.faults.clone(), wrote_prefix: false, })) @@ -115,6 +131,7 @@ impl Backend for Injected { self.disk.create_dir(p, private) } fn remove_file(&self, p: &Path) -> io::Result<()> { + self.faults.check(Point::Remove)?; self.disk.remove_file(p) } fn remove_dir(&self, p: &Path) -> io::Result<()> { @@ -125,6 +142,7 @@ impl Backend for Injected { self.disk.rename(a, b) } fn set_permissions(&self, p: &Path, mode: Permissions) -> io::Result<()> { + self.faults.check(Point::Chmod)?; self.disk.set_permissions(p, mode) } fn sync_directory(&self, p: &Path) -> io::Result<()> { @@ -1035,3 +1053,236 @@ fn allocator_failure_invokes_exit_hook_before_returning_an_error() { .unwrap(); assert_eq!(output.status.code(), Some(73), "{output:?}"); } + +#[test] +fn independent_append_handles_share_service_mutations() { + let t = Fixture::new(); + native::write(t.path("value"), b"start").unwrap(); + let mut a = OpenOptions::new() + .append(true) + .open_in(&t.fs, t.path("value")) + .unwrap(); + let mut b = OpenOptions::new() + .append(true) + .open_in(&t.fs, t.path("value")) + .unwrap(); + a.write_all(b"A").unwrap(); + b.write_all(b"B").unwrap(); + assert_eq!(native::read(t.path("value")).unwrap(), b"startAB"); + t.fs.write(t.path("value"), b"ordinary").unwrap(); + b.write_all(b"B").unwrap(); + assert_eq!(native::read(t.path("value")).unwrap(), b"ordinaryB"); + let _truncated = OpenOptions::new() + .write(true) + .truncate(true) + .open_in(&t.fs, t.path("value")) + .unwrap(); + a.write_all(b"after truncate").unwrap(); + assert_eq!(native::read(t.path("value")).unwrap(), b"after truncate"); + t.fs.replace(t.path("value"), b"replacement").unwrap(); + b.write_all(b"old inode").unwrap(); + assert_eq!(native::read(t.path("value")).unwrap(), b"replacement"); +} + +#[test] +fn handle_chmod_obeys_operation_budget() { + let t = Fixture::budget(1024, 1); + native::write(t.path("value"), b"start").unwrap(); + let mut file = OpenOptions::new() + .append(true) + .open_in(&t.fs, t.path("value")) + .unwrap(); + let before = file.metadata().unwrap().permissions(); + t.faults.arm(Point::Open, libc::ENOSPC); + file.write_all(b"pending").unwrap(); + let mut changed = before.clone(); + changed.set_readonly(true); + for _ in 0..3 { + assert_eq!( + file.set_permissions(changed.clone()).unwrap_err().kind(), + io::ErrorKind::OutOfMemory + ); + assert_eq!(t.fs.status().pending_operations, 1); + assert_eq!(file.metadata().unwrap().permissions(), before); + } + t.settle(); +} + +#[test] +fn retained_lease_handoff_with_pending_parent_sync() { + let t = Fixture::new(); + let path = t.path("session.lock"); + let lease = + t.fs.acquire_lease(&path, &t.root, LeaseMode::CreateNew) + .unwrap(); + let guarded = t.fs.guarded(&lease).unwrap(); + t.faults.arm(Point::DirectorySync, libc::ENOSPC); + guarded.write(t.path("value"), b"published").unwrap(); + guarded.sync_directory(&t.root).unwrap(); + assert!(t.fs.status().pending_operations > 0); + assert_eq!( + t.fs.acquire_lease(&path, &t.root, LeaseMode::ExistingOrNew) + .err() + .unwrap() + .kind(), + io::ErrorKind::WouldBlock + ); + drop(guarded); + drop(lease); + let resumed = + t.fs.acquire_lease(&path, &t.root, LeaseMode::ExistingOrNew) + .unwrap(); + resumed.check().unwrap(); + t.settle(); +} + +#[cfg(unix)] +#[test] +fn ordinary_creation_honors_restrictive_umask() { + use std::os::unix::fs::PermissionsExt; + const CHILD: &str = "KIT_FS_UMASK_CHILD"; + if std::env::var_os(CHILD).is_none() { + // Only the isolated child changes umask, before running its sole test. + use std::os::unix::process::CommandExt; + let mut command = std::process::Command::new(std::env::current_exe().unwrap()); + let test_name = std::thread::current().name().unwrap().to_owned(); + command.args(["--exact", &test_name, "--nocapture"]); + command.env(CHILD, "1"); + unsafe { + command.pre_exec(|| { + libc::umask(0o077); + Ok(()) + }); + } + assert!(command.status().unwrap().success()); + return; + } + let t = Fixture::new(); + t.fs.write(t.path("write"), b"secret").unwrap(); + let _file = OpenOptions::new() + .write(true) + .create(true) + .open_in(&t.fs, t.path("open")) + .unwrap(); + for name in ["write", "open"] { + assert_eq!( + native::metadata(t.path(name)).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + native::set_permissions(t.path("write"), Permissions::from_mode(0o640)).unwrap(); + t.fs.write(t.path("write"), b"preserved").unwrap(); + assert_eq!( + native::metadata(t.path("write")) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o640 + ); + t.faults.arm(Point::Open, libc::ENOSPC); + t.fs.write(t.path("fallback"), b"secret").unwrap(); + t.settle(); + assert_eq!( + native::metadata(t.path("fallback")) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); +} + +#[cfg(unix)] +#[test] +fn stale_handle_chmod_does_not_change_replacement() { + use std::os::unix::fs::PermissionsExt; + for external in [false, true] { + let t = Fixture::new(); + let path = t.path("value"); + native::write(&path, b"old").unwrap(); + let file = OpenOptions::new().read(true).open_in(&t.fs, &path).unwrap(); + if external { + native::write(t.path("new"), b"replacement").unwrap(); + native::rename(t.path("new"), &path).unwrap(); + } else { + t.fs.replace(&path, b"replacement").unwrap(); + } + native::set_permissions(&path, Permissions::from_mode(0o640)).unwrap(); + file.set_permissions(Permissions::from_mode(0o600)).unwrap(); + assert_eq!( + native::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o640 + ); + assert_eq!(file.metadata().unwrap().permissions().mode() & 0o777, 0o600); + } +} + +#[cfg(unix)] +#[test] +fn pending_handle_metadata_and_zero_patch_reads_are_logical() { + use std::os::unix::fs::PermissionsExt; + let t = Fixture::new(); + let path = t.path("value"); + native::write(&path, b"baseline").unwrap(); + let mut file = OpenOptions::new() + .read(true) + .write(true) + .open_in(&t.fs, &path) + .unwrap(); + t.faults.arm(Point::Chmod, libc::ENOSPC); + file.set_permissions(Permissions::from_mode(0o600)).unwrap(); + assert_eq!(file.metadata().unwrap().permissions().mode() & 0o777, 0o600); + assert_eq!( + t.fs.metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + t.settle(); + t.faults.arm(Point::Open, libc::ENOSPC); + file.set_len(0).unwrap(); + assert_eq!(file.metadata().unwrap().len(), 0); + assert_eq!(file.seek(SeekFrom::End(0)).unwrap(), 0); + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes).unwrap(); + assert!(bytes.is_empty()); + file.set_len(5).unwrap(); + assert_eq!(file.metadata().unwrap().len(), 5); + file.read_to_end(&mut bytes).unwrap(); + assert_eq!(bytes, [0; 5]); + t.settle(); + assert_eq!(native::read(&path).unwrap(), [0; 5]); +} + +#[cfg(unix)] +#[test] +fn permission_probe_retries_collisions_and_preserves_replacements() { + let t = Fixture::new(); + t.faults.arm(Point::ProbeCollision, libc::EEXIST); + assert_eq!( + t.fs.write(t.path("value"), b"data").unwrap_err().kind(), + io::ErrorKind::AlreadyExists + ); + assert_eq!(t.fs.status().pending_operations, 0); + assert_eq!(t.faults.1.load(Ordering::Relaxed), 8); + t.faults.arm(Point::ProbeSwap, libc::EIO); + assert_eq!( + t.fs.write(t.path("value"), b"data").unwrap_err().kind(), + io::ErrorKind::PermissionDenied + ); + let replacement = native::read_dir(&t.root) + .unwrap() + .map(|e| e.unwrap().path()) + .find(|p| p.extension().is_some_and(|e| e == "tmp")) + .unwrap(); + assert_eq!(native::read(replacement).unwrap(), b"replacement"); +} + +#[cfg(unix)] +#[test] +fn capacity_during_permission_probe_cleanup_does_not_reject_write() { + let t = Fixture::new(); + t.faults.arm(Point::Remove, libc::ENOSPC); + t.fs.write(t.path("value"), b"data").unwrap(); + assert_eq!(native::read(t.path("value")).unwrap(), b"data"); + t.settle(); +} diff --git a/src/runtime.rs b/src/runtime.rs index 428929f9..6134f640 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -2295,7 +2295,7 @@ async fn load_initial_transcript(root: &Path, system_prompt: String) -> Result>(); for directory in ancestors.into_iter().rev() { let path = directory.join("AGENTS.md"); - let body = match crate::resilient_fs::read_to_string(&path) { + let body = match crate::config_files::read_to_string(&path) { Ok(body) => body, Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, Err(error) => { diff --git a/src/runtime/tests.rs b/src/runtime/tests.rs index 0ba08ab8..37dc1498 100644 --- a/src/runtime/tests.rs +++ b/src/runtime/tests.rs @@ -646,6 +646,37 @@ fn nonmatching_failed_load_does_not_touch_configured_or_generated_queues() { assert_eq!(next.id(), "selected"); } +#[cfg(unix)] +#[tokio::test] +async fn agents_md_symlink_loads_and_rereads_target() { + let root = tempfile::tempdir().unwrap(); + let target = root.path().join("instructions.md"); + let link = root.path().join("AGENTS.md"); + std::os::unix::fs::symlink("instructions.md", &link).unwrap(); + for body in ["first guidance", "updated guidance"] { + std::fs::write(&target, body).unwrap(); + let transcript = load_initial_transcript(root.path(), "system".into()) + .await + .unwrap(); + let context = transcript.last().unwrap(); + assert_eq!(context.kind, ItemKind::Context); + let Part::Text(text) = &context.parts[0] else { + panic!("expected context text") + }; + assert!(text.text.contains(body)); + assert_eq!( + context.metadata["agentkit.context.path"], + serde_json::json!(link.display().to_string()) + ); + } + std::fs::remove_file(&target).unwrap(); + assert!( + load_initial_transcript(root.path(), "system".into()) + .await + .is_ok() + ); +} + #[tokio::test] async fn loads_all_agents_md_files_outermost_first() { let parent = tempfile::tempdir().unwrap(); diff --git a/src/storage_runtime.rs b/src/storage_runtime.rs index 64c6441e..59c8ad70 100644 --- a/src/storage_runtime.rs +++ b/src/storage_runtime.rs @@ -70,8 +70,10 @@ pub fn start_recovery_worker() { let fs = crate::resilient_fs::global(); let mut delay = 1; let mut warned = false; + let mut emitted_status = None; loop { let status = fs.status(); + publish_status(status.pending_operations > 0, status.exhausted, &mut emitted_status); if status.exhausted { let _ = std::io::stderr().write_all( b"kit: internal storage memory budget exhausted; cancelling work and shutting down. Unpersisted data cannot survive process exit.\n", @@ -85,8 +87,10 @@ pub fn start_recovery_worker() { ); warned = true; } - let report = fs.recover(); - if report.remaining_operations == 0 { + let _ = fs.recover(); + let recovered = fs.status(); + publish_status(recovered.pending_operations > 0, recovered.exhausted, &mut emitted_status); + if recovered.pending_operations == 0 { let _ = std::io::stderr().write_all( b"kit: internal storage recovered; pending changes are persisted.\n", ); @@ -96,6 +100,12 @@ pub fn start_recovery_worker() { delay = (delay * 2).min(30); } } else { + if warned { + let _ = std::io::stderr().write_all( + b"kit: internal storage recovered; pending changes are persisted.\n", + ); + warned = false; + } delay = 1; } if shutdown_token().is_cancelled() { @@ -116,6 +126,13 @@ pub fn start_recovery_worker() { }); } +fn publish_status(pending: bool, exhausted: bool, previous: &mut Option<(bool, bool)>) { + if *previous != Some((pending, exhausted)) { + crate::events::emit(&crate::events::RuntimeEvent::StorageStatus { pending, exhausted }); + *previous = Some((pending, exhausted)); + } +} + #[cfg(test)] mod tests { #[test] diff --git a/src/tools/mcp.rs b/src/tools/mcp.rs index 6c63d099..156a860d 100644 --- a/src/tools/mcp.rs +++ b/src/tools/mcp.rs @@ -946,7 +946,7 @@ fn prepare_plugins( async fn read_source(source: &ConfigSource) -> Result>, String> { let path = source.path.clone(); - match tokio::task::spawn_blocking(move || crate::resilient_fs::read(path)) + match tokio::task::spawn_blocking(move || crate::config_files::read(&path)) .await .map_err(|error| format!("MCP config read task failed: {error}"))? { @@ -2848,6 +2848,98 @@ mod tests { }; use crate::plugins::{PluginRuntime, ResolvedPluginMcp, ResolvedPlugins}; + #[cfg(unix)] + mod config_capacity { + use crate as kit; + include!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/support/capacity.rs" + )); + } + + #[cfg(unix)] + #[test] + fn config_symlink_reads_memory_only_target() { + use config_capacity::{Capacity, CapacityDisk}; + use std::sync::atomic::AtomicBool; + let directory = tempfile::tempdir().unwrap(); + let capacity = Arc::new(Capacity { + exhausted: AtomicBool::new(true), + exhaust_on_write: AtomicBool::new(false), + repaired: directory.path().join("repaired"), + }); + let filesystem = crate::resilient_fs::Fs::new(Arc::new(CapacityDisk(capacity))); + let target = directory.path().join("config.json"); + let link = directory.path().join(".mcp.json"); + std::os::unix::fs::symlink("config.json", &link).unwrap(); + filesystem.write(&target, b"pending").unwrap(); + assert!(!target.exists()); + assert_eq!( + crate::config_files::read_in(&filesystem, &link).unwrap(), + b"pending" + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn config_sources_follow_symlinks_and_reload_targets() { + use std::os::unix::fs::symlink; + let directory = tempfile::tempdir().unwrap(); + let target = directory.path().join("config.json"); + let link = directory.path().join(".mcp.json"); + let chained = directory.path().join("explicit.json"); + std::fs::write(&target, b"first").unwrap(); + symlink("config.json", &link).unwrap(); + symlink(&link, &chained).unwrap(); + for source in [ + ConfigSource::required(chained), + ConfigSource::optional_project(link.clone(), directory.path().to_path_buf()), + ] { + assert_eq!( + super::read_source(&source).await.unwrap(), + Some(b"first".to_vec()) + ); + crate::resilient_fs::write(&target, b"second").unwrap(); + assert_eq!( + super::read_source(&source).await.unwrap(), + Some(b"second".to_vec()) + ); + std::fs::write(&target, b"first").unwrap(); + } + // Security-sensitive managed reads still reject final symlinks. + assert_eq!( + crate::resilient_fs::read(&link).unwrap_err().kind(), + std::io::ErrorKind::PermissionDenied + ); + std::fs::remove_file(&target).unwrap(); + assert!( + super::read_source(&ConfigSource::optional_project( + link.clone(), + directory.path().to_path_buf() + )) + .await + .unwrap() + .is_none() + ); + assert!( + super::read_source(&ConfigSource::required(link)) + .await + .is_err() + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn config_source_rejects_symlink_cycles() { + let directory = tempfile::tempdir().unwrap(); + let link = directory.path().join("loop.json"); + std::os::unix::fs::symlink("loop.json", &link).unwrap(); + let error = super::read_source(&ConfigSource::required(link)) + .await + .unwrap_err(); + assert!(error.contains("too many symbolic links"), "{error}"); + } + fn spec(name: &str, description: &str) -> ToolSpec { ToolSpec::new(ToolName::new(name), description, json!({"type": "object"})) } diff --git a/src/tui/app.rs b/src/tui/app.rs index d15ee5bb..9dd14c9e 100644 --- a/src/tui/app.rs +++ b/src/tui/app.rs @@ -647,6 +647,9 @@ pub struct App { pub usage: Option, pub logs: Vec, pub show_logs: bool, + /// Shared child storage outlives individual sessions. + pub storage_pending: bool, + pub storage_exhausted: bool, pub show_thoughts: bool, agents_visible: bool, agents: HashMap, @@ -937,6 +940,8 @@ impl App { usage: None, logs: Vec::new(), show_logs: false, + storage_pending: false, + storage_exhausted: false, show_thoughts: false, agents_visible: false, agents: HashMap::new(), @@ -2021,6 +2026,11 @@ impl App { } fn apply_runtime(&mut self, event: RuntimeEvent) { + if let RuntimeEvent::StorageStatus { pending, exhausted } = event { + self.storage_pending = pending; + self.storage_exhausted = exhausted; + return; + } if let RuntimeEvent::SessionStarted { session_id } = event { self.runtime_session_id = Some(session_id); return; @@ -2074,7 +2084,8 @@ impl App { millis, .. } => call.finish_child(&child_call, ok, summary, millis), - RuntimeEvent::SessionStarted { .. } + RuntimeEvent::StorageStatus { .. } + | RuntimeEvent::SessionStarted { .. } | RuntimeEvent::CompactionStarted { .. } | RuntimeEvent::CompactionFinished { .. } | RuntimeEvent::SubagentStateChanged { .. } @@ -4644,6 +4655,31 @@ mod tests { assert_eq!(app.editor.text(), "/new"); } + #[test] + fn storage_status_survives_session_switch_and_clears_after_recovery() { + let mut app = app(); + app.start_session("old".into()); + // Storage events must bypass the session routing guard. + app.apply(Update::Runtime(RuntimeEvent::StorageStatus { + pending: true, + exhausted: false, + })); + assert!(app.storage_pending); + assert!(!app.show_logs); + app.start_session("new".into()); + assert!(app.storage_pending); + app.apply(Update::Runtime(RuntimeEvent::StorageStatus { + pending: false, + exhausted: false, + })); + assert!(!app.storage_pending); + app.apply(Update::Runtime(RuntimeEvent::StorageStatus { + pending: false, + exhausted: true, + })); + assert!(app.storage_exhausted); + } + #[test] fn switching_sessions_restores_the_initial_view_but_retains_diagnostics() { let mut app = app(); diff --git a/src/tui/mod.rs b/src/tui/mod.rs index 712c5491..0f47fc3b 100644 --- a/src/tui/mod.rs +++ b/src/tui/mod.rs @@ -816,16 +816,26 @@ pub async fn run_with_reasoning_effort_and_openrouter_key( let (exit_tx, mut exit_rx) = oneshot::channel(); let (shutdown_tx, shutdown_rx) = oneshot::channel(); let watcher = tokio::spawn(async move { - tokio::select! { + let status = tokio::select! { status = child.wait() => { - let _ = exit_tx.send(status); + let notification = status.as_ref().copied().map_err(|error| { + std::io::Error::new(error.kind(), error.to_string()) + }); + let _ = exit_tx.send(notification); + status } _ = shutdown_rx => { - // Unlike dropping a `kill_on_drop` child, `kill().await` waits - // until the process is gone and its OS file locks are released. - let _ = child.kill().await; + // ACP EOF makes serve stop A2A, drain sessions, and run + // final storage recovery before exiting. + wait_for_storage_exit(&mut child, Duration::from_secs(10)).await } + }?; + if !status.success() { + return Err(std::io::Error::other(format!( + "agent exited with {status}; final storage recovery may have failed; unpersisted data may have been lost" + ))); } + Ok::<_, std::io::Error>(()) }); let notifications = updates_tx.clone(); @@ -1657,22 +1667,44 @@ pub async fn run_with_reasoning_effort_and_openrouter_key( .map_err(explain) }; - // The A2A listener keeps the child alive after ACP closes. Stop it only - // after CloseSession has unwound the lock owner, then wait for OS locks to - // be released rather than relying on `kill_on_drop`. + // The client future has dropped its transport. Allow the storage-owning + // process to complete graceful shutdown, including its final recovery. let _ = shutdown_tx.send(()); - let _ = watcher.await; + let shutdown_result = watcher.await; // If the server failed before acknowledging CloseSession, reclaim only a // lock that is now provably stale; a live owner's OS lock is never stolen. if let Ok(active) = active_persisted_id.lock() { let _ = crate::session::remove_stale_lock(&cleanup_root, &active.id); } + // Keep startup/protocol diagnostics as well as final persistence failure. + if let Err(shutdown) = shutdown_result? { + return Err(match result { + Err(error) => std::io::Error::other(format!("{error}\n{shutdown}")), + Ok(()) => shutdown, + } + .into()); + } result?; Ok(()) } } +async fn wait_for_storage_exit( + child: &mut tokio::process::Child, + timeout: Duration, +) -> std::io::Result { + match tokio::time::timeout(timeout, child.wait()).await { + Ok(status) => status, + Err(_) => { + let message = "kit: agent graceful shutdown timed out; forcing termination. Final storage recovery may not have completed; unpersisted data may be lost."; + eprintln!("{message}"); + child.kill().await?; + Err(std::io::Error::new(std::io::ErrorKind::TimedOut, message)) + } + } +} + fn config_save_message(setting: &str) -> String { if crate::resilient_fs::global().status().pending_operations > 0 { format!("updated {setting} in memory; disk persistence is pending (not durable)") @@ -3749,6 +3781,50 @@ mod signal_tests { assert!(matches!(result, Err(RequestInterrupt::Stopped))); } + #[cfg(unix)] + #[tokio::test] + async fn storage_exit_waits_for_final_flush_and_preserves_failure_status() { + use tokio::io::AsyncReadExt; + for code in [0, 1] { + let mut child = tokio::process::Command::new("sh") + .arg("-c") + .arg(format!( + "cat >/dev/null; sleep 0.05; printf recovered; exit {code}" + )) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .kill_on_drop(true) + .spawn() + .unwrap(); + let mut stdout = child.stdout.take().unwrap(); + // Like dropping the ACP transport: EOF initiates child shutdown. + drop(child.stdin.take()); + let status = super::wait_for_storage_exit(&mut child, Duration::from_secs(2)) + .await + .unwrap(); + assert_eq!(status.code(), Some(code)); + let mut flushed = String::new(); + stdout.read_to_string(&mut flushed).await.unwrap(); + assert_eq!(flushed, "recovered"); + } + } + + #[cfg(unix)] + #[tokio::test] + async fn storage_exit_timeout_reports_possible_loss_and_reaps_child() { + let mut child = tokio::process::Command::new("sh") + .args(["-c", "exec sleep 60"]) + .kill_on_drop(true) + .spawn() + .unwrap(); + let error = super::wait_for_storage_exit(&mut child, Duration::from_millis(10)) + .await + .unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::TimedOut); + assert!(error.to_string().contains("unpersisted data may be lost")); + assert!(child.try_wait().unwrap().is_some()); + } + #[tokio::test] async fn a_stuck_close_is_bounded() { let closed = bounded_graceful_close( diff --git a/src/tui/ui.rs b/src/tui/ui.rs index 4cf3471b..e996b371 100644 --- a/src/tui/ui.rs +++ b/src/tui/ui.rs @@ -127,6 +127,25 @@ pub fn draw(frame: &mut Frame<'_>, app: &mut App, images: &mut ImageRuntime) { } else if app.effort_dialog.is_some() { draw_effort_dialog(frame, app); } + // Durability stays visible on the start screen and over session pickers. + // Pending data belongs to the process, not the currently selected session. + if app.storage_pending || app.storage_exhausted { + let area = frame.area(); + let warning = if app.storage_exhausted { + " Storage exhausted: shutting down; unpersisted data is at risk" + } else { + " Memory-only storage: awaiting disk recovery; data at risk on exit" + }; + frame.render_widget( + Paragraph::new(warning).style(Style::default().fg(theme::warn_color())), + Rect::new( + area.x, + area.bottom().saturating_sub(1), + area.width, + u16::from(area.height > 0), + ), + ); + } } #[derive(Clone, Copy)] @@ -2830,6 +2849,36 @@ mod tests { .join("\n") } + #[test] + fn storage_warning_is_visible_without_logs_and_clears_on_recovery() { + let mut app = App::new( + PathBuf::from("/tmp"), + "openai".into(), + "gpt".into(), + "127.0.0.1:7331".into(), + ); + app.apply(Update::Runtime(RuntimeEvent::StorageStatus { + pending: true, + exhausted: false, + })); + assert!(!app.show_logs); + assert!(render(&mut app, 80, 24).contains("Memory-only storage")); + app.blocks.push(Block::Agent("response".into())); + assert!(render(&mut app, 80, 24).contains("Memory-only storage")); + app.start_session("next".into()); + assert!(render(&mut app, 80, 24).contains("Memory-only storage")); + app.apply(Update::Runtime(RuntimeEvent::StorageStatus { + pending: false, + exhausted: false, + })); + assert!(!render(&mut app, 80, 24).contains("Memory-only storage")); + app.apply(Update::Runtime(RuntimeEvent::StorageStatus { + pending: false, + exhausted: true, + })); + assert!(render(&mut app, 80, 24).contains("Storage exhausted")); + } + #[test] fn command_popup_renders_in_start_and_compact_layouts() { let mut app = App::new( diff --git a/tests/resilient_exit.rs b/tests/resilient_exit.rs index 953d4b17..ab372c8a 100644 --- a/tests/resilient_exit.rs +++ b/tests/resilient_exit.rs @@ -19,6 +19,7 @@ fn pending_replacement() -> (tempfile::TempDir, PathBuf, Arc, Fs) { fs::write(&path, b"old generation\n").unwrap(); let backend = Arc::new(Capacity { exhausted: AtomicBool::new(true), + exhaust_on_write: AtomicBool::new(false), repaired: directory.path().join("capacity-repaired"), }); let filesystem = Fs::new(Arc::new(CapacityDisk(backend.clone()))); diff --git a/tests/resilient_session.rs b/tests/resilient_session.rs index 69743fcf..e701b77d 100644 --- a/tests/resilient_session.rs +++ b/tests/resilient_session.rs @@ -47,6 +47,7 @@ fn session_survives_outage_close_reopen_and_tool_driven_recovery() { fs::create_dir(&root).unwrap(); let capacity = Arc::new(Capacity { exhausted: AtomicBool::new(false), + exhaust_on_write: AtomicBool::new(false), repaired: home.join("repaired"), }); assert!( @@ -174,3 +175,72 @@ fn session_survives_outage_close_reopen_and_tool_driven_recovery() { assert_eq!(resilient_fs::global().recover().remaining_operations, 0); assert_eq!(fs::read_to_string(transcript).unwrap(), durable); } + +#[test] +fn legacy_migration_reopens_with_retained_parent_sync_during_outage() { + const CHILD: &str = "KIT_RESILIENT_MIGRATION_CHILD"; + if std::env::var_os(CHILD).is_none() { + let home = tempfile::tempdir().unwrap(); + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "legacy_migration_reopens_with_retained_parent_sync_during_outage", + "--nocapture", + ]) + .env(CHILD, "1") + .env("HOME", home.path()) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return; + } + let home = PathBuf::from(std::env::var_os("HOME").unwrap()); + let root = home.join("project"); + let legacy = root.join(".kit/sessions"); + fs::create_dir_all(&legacy).unwrap(); + let capacity = Arc::new(Capacity { + exhausted: AtomicBool::new(false), + exhaust_on_write: AtomicBool::new(false), + repaired: home.join("repaired"), + }); + resilient_fs::initialize_global(Fs::new(Arc::new(CapacityDisk(capacity.clone())))).unwrap(); + // Materialize the workspace's storage directory before the simulated outage. + drop( + kit::session::open( + &root, + "bootstrap", + false, + false, + vec![Item::text(ItemKind::System, "system")], + ) + .unwrap(), + ); + let record = json!({ + "schema_version": 2, "session_id": "legacy", "generation": 1, + "item": Item::text(ItemKind::System, "legacy history"), + }); + fs::write(legacy.join("legacy.jsonl"), format!("{record}\n")).unwrap(); + // Acquire real scoped and legacy leases before the first data write fails. + capacity.exhaust_on_write.store(true, Ordering::SeqCst); + let opened = kit::session::open(&root, "legacy", true, false, vec![]).unwrap(); + let transcript = opened.transcript.clone(); + assert!(resilient_fs::global().status().pending_operations > 0); + drop(opened); + let reopened = kit::session::open(&root, "legacy", true, false, vec![]).unwrap(); + assert_eq!(reopened.transcript, transcript); + assert!(kit::session::open(&root, "legacy", true, true, vec![]).is_err()); + capacity.exhausted.store(false, Ordering::SeqCst); + assert_eq!(resilient_fs::global().recover().remaining_operations, 0); + drop(reopened); + assert_eq!( + kit::session::open(&root, "legacy", true, false, vec![]) + .unwrap() + .transcript, + transcript + ); +} diff --git a/tests/resilient_shutdown.rs b/tests/resilient_shutdown.rs index cf09559b..a1e7306a 100644 --- a/tests/resilient_shutdown.rs +++ b/tests/resilient_shutdown.rs @@ -15,6 +15,7 @@ async fn exhausted_storage_requests_orderly_process_shutdown() { let directory = tempfile::tempdir().unwrap(); let capacity = Arc::new(Capacity { exhausted: AtomicBool::new(true), + exhaust_on_write: AtomicBool::new(false), repaired: directory.path().join("repair"), }); fs::initialize_global(Fs::with_budget(Arc::new(CapacityDisk(capacity)), 0, 0)).unwrap(); diff --git a/tests/support/capacity.rs b/tests/support/capacity.rs index 8ecce63c..50b62079 100644 --- a/tests/support/capacity.rs +++ b/tests/support/capacity.rs @@ -12,6 +12,7 @@ use std::{ }; pub struct Capacity { pub exhausted: AtomicBool, + pub exhaust_on_write: AtomicBool, pub repaired: PathBuf, } impl Capacity { @@ -40,6 +41,9 @@ impl Seek for CapacityFile { } impl Write for CapacityFile { fn write(&mut self, bytes: &[u8]) -> io::Result { + if self.capacity.exhaust_on_write.swap(false, Ordering::SeqCst) { + self.capacity.exhausted.store(true, Ordering::SeqCst); + } self.capacity.check()?; self.inner.write(bytes) } From 42346fc52017569b5345cf4b64363e5d3e6bc197 Mon Sep 17 00:00:00 2001 From: daniel Date: Sat, 5 Sep 2026 10:26:38 +0100 Subject: [PATCH 4/4] fix(storage): isolate displaced handles during pending replacement --- src/config_files.rs | 79 ++++++++++++++----- src/resilient_fs/mod.rs | 10 +++ src/resilient_fs/tests.rs | 156 ++++++++++++++++++++++++++++++++++++++ src/tools/mcp.rs | 49 ++++++++++++ 4 files changed, 276 insertions(+), 18 deletions(-) diff --git a/src/config_files.rs b/src/config_files.rs index 30dda98a..07499ed1 100644 --- a/src/config_files.rs +++ b/src/config_files.rs @@ -2,26 +2,69 @@ use std::{io, path::Path}; -// User-selected configuration files may be symlinks. Resolve only at this read -// boundary: managed storage continues to reject final symlinks. Resolve each -// target through the facade so even a memory-only target remains readable. -pub fn read_in(filesystem: &crate::resilient_fs::Fs, path: &Path) -> std::io::Result> { - let mut path = path.to_path_buf(); - for _ in 0..40 { - if !filesystem.symlink_metadata(&path)?.file_type().is_symlink() { - return filesystem.read(path); +// Resolve components in filesystem order. In particular, `..` applies to the +// resolved directory, not to the lexical parent of a directory symlink. Each +// lookup uses the overlay and never requires the final target to exist on disk. +fn resolve_in( + filesystem: &crate::resilient_fs::Fs, + path: &Path, + links: &mut usize, +) -> io::Result { + use std::path::{Component, PathBuf}; + + let absolute = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir()?.join(path) + }; + let mut resolved = PathBuf::new(); + for component in absolute.components() { + match component { + Component::Prefix(_) | Component::RootDir => resolved.push(component.as_os_str()), + Component::CurDir => {} + Component::ParentDir => { + if !filesystem.metadata(&resolved)?.is_dir() { + return Err(io::ErrorKind::NotADirectory.into()); + } + // Popping a root has no effect, as with native path traversal. + resolved.pop(); + } + Component::Normal(name) => { + if !filesystem.metadata(&resolved)?.is_dir() { + return Err(io::ErrorKind::NotADirectory.into()); + } + resolved.push(name); + if filesystem + .symlink_metadata(&resolved)? + .file_type() + .is_symlink() + { + if *links == 40 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "too many symbolic links in configuration path", + )); + } + *links += 1; + let target = filesystem.read_link(&resolved)?; + let target = if target.is_absolute() { + target + } else { + resolved + .parent() + .unwrap_or_else(|| Path::new(".")) + .join(target) + }; + resolved = resolve_in(filesystem, &target, links)?; + } + } } - let target = filesystem.read_link(&path)?; - path = if target.is_absolute() { - target - } else { - path.parent().unwrap_or_else(|| Path::new(".")).join(target) - }; } - Err(std::io::Error::new( - std::io::ErrorKind::InvalidInput, - "too many symbolic links in configuration path", - )) + Ok(resolved) +} + +pub fn read_in(filesystem: &crate::resilient_fs::Fs, path: &Path) -> io::Result> { + filesystem.read(resolve_in(filesystem, path, &mut 0)?) } pub fn read(path: &Path) -> io::Result> { diff --git a/src/resilient_fs/mod.rs b/src/resilient_fs/mod.rs index 88542716..428119fe 100644 --- a/src/resilient_fs/mod.rs +++ b/src/resilient_fs/mod.rs @@ -1441,6 +1441,16 @@ impl Fs { if !accepted { return result; } + if new_object { + // Acceptance replaces the logical name even when publication is queued. + // Old handles must not publish writes or chmod through that name. + for object in s.objects.iter().filter_map(|w| w.upgrade()) { + let mut object = lock(&object); + if object.path.as_ref() == Some(&path) { + object.path = None; + } + } + } let object = if let Some(o) = object { *lock(&o) = Object::memory(data.clone(), meta); o diff --git a/src/resilient_fs/tests.rs b/src/resilient_fs/tests.rs index bbae0b49..a1dfd663 100644 --- a/src/resilient_fs/tests.rs +++ b/src/resilient_fs/tests.rs @@ -1218,6 +1218,162 @@ fn stale_handle_chmod_does_not_change_replacement() { } } +#[cfg(unix)] +#[test] +fn blocked_replacement_detaches_stale_handle_mutations() { + use std::os::unix::fs::PermissionsExt; + for replacement in [ + "replace", + "private", + "unlink", + "rename", + "replace-rename", + "rename-replace", + ] { + for mutation in ["chmod", "write", "set_len"] { + for dirty in [false, true] { + let t = Fixture::new(); + let path = t.path("value"); + let moved = t.path("moved"); + native::write(&path, b"baseline").unwrap(); + native::set_permissions(&path, Permissions::from_mode(0o640)).unwrap(); + native::write(t.path("source"), b"replacement").unwrap(); + native::set_permissions(t.path("source"), Permissions::from_mode(0o640)).unwrap(); + let mut held = OpenOptions::new() + .read(true) + .write(true) + .open_in(&t.fs, &path) + .unwrap(); + t.faults.arm(Point::Rename, libc::ENOSPC); + if dirty { + // Also cover a displaced object with an existing queued image. + t.fs.write(&path, b"baseline").unwrap(); + } + let named = match replacement { + "replace" => { + t.fs.replace(&path, b"replacement").unwrap(); + &path + } + "private" => { + t.fs.replace_private(&path, b"replacement").unwrap(); + &path + } + "unlink" => { + t.faults.arm(Point::Remove, libc::ENOSPC); + t.fs.remove_file(&path).unwrap(); + t.fs.write(&path, b"replacement").unwrap(); + t.fs.set_permissions(&path, Permissions::from_mode(0o640)) + .unwrap(); + &path + } + "rename" => { + t.fs.rename(t.path("source"), &path).unwrap(); + &path + } + "replace-rename" => { + t.fs.replace(&path, b"replacement").unwrap(); + t.fs.rename(&path, &moved).unwrap(); + &moved + } + "rename-replace" => { + t.fs.rename(&path, &moved).unwrap(); + t.fs.replace(&moved, b"replacement").unwrap(); + &moved + } + _ => unreachable!(), + }; + let mode = if replacement == "private" { + 0o600 + } else { + 0o640 + }; + let pending = t.fs.status().pending_operations; + assert!(pending > 0); + match mutation { + "chmod" => held.set_permissions(Permissions::from_mode(0o400)).unwrap(), + "write" => held.write_all(b"stale").unwrap(), + "set_len" => held.set_len(3).unwrap(), + _ => unreachable!(), + } + assert_eq!(t.fs.status().pending_operations, pending); + assert_eq!(native::read(&path).unwrap(), b"baseline"); + assert_eq!( + native::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o640 + ); + assert_eq!(t.fs.read(named).unwrap(), b"replacement"); + assert_eq!( + t.fs.metadata(named).unwrap().permissions().mode() & 0o777, + mode + ); + held.rewind().unwrap(); + let mut old = Vec::new(); + held.read_to_end(&mut old).unwrap(); + assert_eq!( + old, + match mutation { + "write" => b"staleine".as_slice(), + "set_len" => b"bas".as_slice(), + _ => b"baseline".as_slice(), + } + ); + assert_eq!( + held.metadata().unwrap().permissions().mode() & 0o777, + if mutation == "chmod" { 0o400 } else { 0o640 } + ); + t.settle(); + assert_eq!(native::read(named).unwrap(), b"replacement"); + assert_eq!( + native::metadata(named).unwrap().permissions().mode() & 0o777, + mode + ); + assert_eq!(t.fs.read(named).unwrap(), b"replacement"); + if named == &moved { + assert!(!t.fs.try_exists(&path).unwrap()); + assert!(!path.exists()); + } + } + } + } +} + +#[cfg(unix)] +#[test] +fn blocked_ordinary_write_and_truncate_keep_shared_handle_identity() { + use std::os::unix::fs::PermissionsExt; + let t = Fixture::new(); + let path = t.path("value"); + native::write(&path, b"baseline").unwrap(); + native::set_permissions(&path, Permissions::from_mode(0o640)).unwrap(); + let mut held = OpenOptions::new() + .read(true) + .write(true) + .open_in(&t.fs, &path) + .unwrap(); + t.faults.arm(Point::Rename, libc::ENOSPC); + t.fs.write(&path, b"ordinary").unwrap(); + let mut bytes = Vec::new(); + held.read_to_end(&mut bytes).unwrap(); + assert_eq!(bytes, b"ordinary"); + let truncated = t.fs.create(&path).unwrap(); + assert_eq!(held.metadata().unwrap().len(), 0); + held.rewind().unwrap(); + held.write_all(b"shared").unwrap(); + assert_eq!(truncated.metadata().unwrap().len(), 6); + held.set_permissions(Permissions::from_mode(0o600)).unwrap(); + assert_eq!(t.fs.read(&path).unwrap(), b"shared"); + assert_eq!( + t.fs.metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + t.settle(); + assert_eq!(native::read(&path).unwrap(), b"shared"); + assert_eq!( + native::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600 + ); +} + #[cfg(unix)] #[test] fn pending_handle_metadata_and_zero_patch_reads_are_logical() { diff --git a/src/tools/mcp.rs b/src/tools/mcp.rs index 156a860d..1d8f5569 100644 --- a/src/tools/mcp.rs +++ b/src/tools/mcp.rs @@ -2869,10 +2869,59 @@ mod tests { repaired: directory.path().join("repaired"), }); let filesystem = crate::resilient_fs::Fs::new(Arc::new(CapacityDisk(capacity))); + let project = directory.path().join("project"); + let elsewhere = directory.path().join("elsewhere"); + std::fs::create_dir(&project).unwrap(); + std::fs::create_dir_all(elsewhere.join("nested")).unwrap(); let target = directory.path().join("config.json"); let link = directory.path().join(".mcp.json"); std::os::unix::fs::symlink("config.json", &link).unwrap(); filesystem.write(&target, b"pending").unwrap(); + let parent_link = project.join(".mcp.json"); + std::os::unix::fs::symlink("../config.json", &parent_link).unwrap(); + assert_eq!( + crate::config_files::read_in(&filesystem, &parent_link).unwrap(), + b"pending" + ); + + // `..` must follow directory symlinks before selecting the parent. + std::os::unix::fs::symlink(elsewhere.join("nested"), project.join("directory-link")) + .unwrap(); + let indirect = project.join("indirect.json"); + std::os::unix::fs::symlink("directory-link/../config.json", &indirect).unwrap(); + filesystem + .write(elsewhere.join("config.json"), b"resolved parent") + .unwrap(); + filesystem + .write(project.join("config.json"), b"wrong lexical parent") + .unwrap(); + assert_eq!( + crate::config_files::read_in(&filesystem, &indirect).unwrap(), + b"resolved parent" + ); + + // Traversal also works through an accepted, memory-only directory. + filesystem + .create_dir(directory.path().join("virtual")) + .unwrap(); + let virtual_link = project.join("virtual.json"); + std::os::unix::fs::symlink("../virtual/../config.json", &virtual_link).unwrap(); + assert_eq!( + crate::config_files::read_in(&filesystem, &virtual_link).unwrap(), + b"pending" + ); + assert_eq!( + crate::config_files::read_in(&filesystem, &project.join("missing/../config.json")) + .unwrap_err() + .kind(), + std::io::ErrorKind::NotFound + ); + assert_eq!( + crate::config_files::read_in(&filesystem, &project.join("config.json/../config.json")) + .unwrap_err() + .kind(), + std::io::ErrorKind::NotADirectory + ); assert!(!target.exists()); assert_eq!( crate::config_files::read_in(&filesystem, &link).unwrap(),