Recover terminal model failures and preserve verification evidence

.github/workflows/ci.yml

@@ -16,10 +16,12 @@ jobs:
   quality:
     name: format · lint · deterministic tests
     runs-on: ubuntu-latest
+    env:
+      HI_PIPE_WRAP: ${{ github.workspace }}/target/pipe-wrap/x86_64-unknown-linux-gnu/pipe-wrap
     steps:
       - uses: actions/checkout@v5
-      - name: Install Linux build dependencies
-        run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libasound2-dev
+      - name: Install Linux build and test dependencies
+        run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libasound2-dev ripgrep
       - uses: dtolnay/rust-toolchain@1.96.0
         with:
           components: rustfmt, clippy
@@ -33,24 +35,41 @@ jobs:
       - name: Clippy (0.2 core only)
         run: >-
           cargo clippy
-          -p hi-ai -p hi-tools -p hi-lsp -p hi-agent -p hi-workspace
+          -p hi-ai -p hi-tools -p hi-lsp -p hi-agent -p hi-workspace -p hi-sqlite-journal
           -p hi-control -p hi-pipefs -p hi-shell
           -p hi-tui -p hi -p hi-eval -p hi-smoke
           --no-default-features
           --all-targets -- -A clippy::large-enum-variant -D warnings
 
+      - name: Build Linux sandbox helper
+        run: tools/build-pipe-wrap.sh
+
+      - name: Enable unprivileged user namespaces
+        run: |
+          sudo sysctl -w kernel.unprivileged_userns_clone=1 2>/dev/null || true
+          sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 2>/dev/null || true
+
+      - name: Verify Linux sandbox helper enforcement
+        run: '"${HI_PIPE_WRAP}" --unshare-all --ro-bind / / -- true'
+
+      - name: Verify native ripgrep inside the sandbox
+        run: |
+          command -v rg
+          rg --version
+          "${HI_PIPE_WRAP}" --unshare-all --ro-bind / / -- rg --version
+
       - name: Core tests
         run: >-
-          cargo nextest run
-          -p hi-ai -p hi-tools -p hi-lsp -p hi-agent -p hi-workspace
+          cargo nextest run --no-fail-fast
+          -p hi-ai -p hi-tools -p hi-lsp -p hi-agent -p hi-workspace -p hi-sqlite-journal
           -p hi-control -p hi-pipefs -p hi-shell
           -p hi-tui -p hi -p hi-eval -p hi-smoke
           --no-default-features
 
       - name: Core doctests
         run: >-
           cargo test --doc
-          -p hi-ai -p hi-tools -p hi-lsp -p hi-agent -p hi-workspace
+          -p hi-ai -p hi-tools -p hi-lsp -p hi-agent -p hi-workspace -p hi-sqlite-journal
           -p hi-control -p hi-pipefs -p hi-shell
           -p hi-tui -p hi -p hi-eval -p hi-smoke
           --no-default-features
@@ -165,5 +184,22 @@ jobs:
           if-no-files-found: error
           retention-days: 30
 
+  sqlite-macos:
+    name: SQLite permissions and live locks (macOS)
+    runs-on: macos-14
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v5
+      - uses: dtolnay/rust-toolchain@1.96.0
+        with:
+          components: clippy
+      - uses: Swatinem/rust-cache@v2
+        with:
+          key: sqlite-macos
+      - name: Lint portable permission handling
+        run: cargo clippy -p hi-sqlite-journal --all-targets -- -D warnings
+      - name: Verify permissions and live SQLite locks
+        run: cargo test -p hi-sqlite-journal --lib
+
   # PR CI is deliberately provider-free. Network/model-backed evaluation runs
   # only in the scheduled workflow and cannot make pull-request checks flaky.

crates/hi-agent/src/agent/background_task.rs

@@ -247,12 +247,8 @@ impl crate::Agent {
                             .await
                         }
                         None => {
-                            let (provider, child_config, capability_registry) =
-                                read_run.expect("read-only background run was prepared");
                             run_bg_readonly(
-                                provider,
-                                child_config,
-                                capability_registry,
+                                read_run.expect("read-only background run was prepared"),
                                 kind,
                                 prompt_for_factory,
                                 child_teardown,
@@ -649,16 +645,19 @@ fn readonly_child_prompt(kind: BgTaskKind, prompt: &str) -> String {
 
 /// Run a background read-only subagent (`explore` / `plan`) to completion.
 async fn run_bg_readonly(
-    provider: std::sync::Arc<dyn hi_ai::Provider>,
-    config: AgentConfig,
-    capability_registry: hi_ai::ProviderCapabilityRegistry,
+    run: (
+        std::sync::Arc<dyn hi_ai::Provider>,
+        AgentConfig,
+        hi_ai::ProviderCapabilityRegistry,
+    ),
     kind: BgTaskKind,
     prompt: String,
     teardown: hi_tools::BackgroundTaskTeardown,
     mailbox: crate::agent::subagent_mailbox::SubagentMailbox,
     task_id: String,
     ui: &mut dyn Ui,
 ) -> hi_tools::BackgroundTaskOutcome {
+    let (provider, config, capability_registry) = run;
     let kind_label = kind.as_str();
     let child_prompt = readonly_child_prompt(kind, &prompt);
 

crates/hi-agent/src/agent/turn/fast_feedback_observations.rs

@@ -6,6 +6,7 @@ use crate::recovery::{ValidationObservation, ValidationResult};
 use crate::workspace_runtime::WorkspaceRuntime;
 
 pub(super) async fn input_revision(runtime: &WorkspaceRuntime) -> Option<String> {
+    runtime.ensure_ledger_scan_complete_async().await.ok()?;
     runtime.reconcile_ledger_async().await.ok()?;
     Some(runtime.ledger().workspace_revision())
 }
@@ -390,9 +391,17 @@ mod tests {
         let workspace = IsolatedWorkspace::new("fast-feedback-revision");
         let agent = crate::Agent::new(Arc::new(Canned(Mutex::new(Vec::new()))), workspace.config())
             .unwrap();
-        std::fs::write(agent.runtime.root().join("value.rs"), "one").unwrap();
+        let path = agent.runtime.root().join("value.rs");
+        std::fs::write(&path, "one").unwrap();
+        let modified = std::fs::metadata(&path).unwrap().modified().unwrap();
         let input = input_revision(&agent.runtime).await;
-        std::fs::write(agent.runtime.root().join("value.rs"), "two").unwrap();
+        std::fs::write(&path, "two").unwrap();
+        std::fs::File::options()
+            .write(true)
+            .open(&path)
+            .unwrap()
+            .set_times(std::fs::FileTimes::new().set_modified(modified))
+            .unwrap();
         let mut observations = Vec::new();
         let result = package_outcome(
             &agent.runtime,

crates/hi-agent/src/agent/turn/finalize.rs

@@ -48,12 +48,28 @@ impl crate::Agent {
     /// model did not supply an accepted final answer.
     /// This path makes no completion claim that verification cannot support.
     pub(super) fn emit_deterministic_closeout(&mut self, ui: &mut dyn Ui) {
-        if self.messages.as_slice().last().is_some_and(|message| {
-            message.role == hi_ai::Role::Assistant
+        let unresolved_validation = self
+            .task_recovery
+            .unresolved_validation_summary(&self.runtime.ledger().workspace_revision());
+        let needs_evidence_closeout = unresolved_validation.is_some()
+            || self.task_recovery.exhausted
+            || self.report.verify.failed()
+            || (!self.workspace.last_changed_files.is_empty() && !self.report.verify.passed())
+            || self
+                .report
+                .last_turn_outcome
+                .as_ref()
+                .is_some_and(|outcome| outcome.status != crate::TurnStatus::Completed);
+        // Keep an accepted answer, while still recording failed checks and
+        // unverified retained changes that its prose cannot override.
+        if !needs_evidence_closeout
+            && self.messages.as_slice().last().is_some_and(|message| {
+                message.role == hi_ai::Role::Assistant
                     && message.content.iter().any(|content| {
                         matches!(content, Content::Text(text) if text_is_user_visible_answer(text))
                     })
-        }) {
+            })
+        {
             return;
         }
         let closeout = if self.task_recovery.exhausted
@@ -87,10 +103,7 @@ impl crate::Agent {
         } else {
             "The turn is closed. No accepted final answer was produced; available results and diagnostics are shown above."
         };
-        let closeout = match self
-            .task_recovery
-            .unresolved_validation_summary(&self.runtime.ledger().workspace_revision())
-        {
+        let closeout = match unresolved_validation {
             Some(summary) => format!("{closeout}\n{summary}"),
             None => closeout.to_owned(),
         };

crates/hi-agent/src/agent/turn/loop_.rs

@@ -861,34 +861,35 @@ impl crate::Agent {
                         turn.flags.clear_one_shot_forces();
                         self.set_turn_phase(TurnPhase::Tools);
                         let tool_started = std::time::Instant::now();
-                        let batch_result = self
-                            .execute_tool_batch(
-                                &calls,
-                                &mut completion_content,
-                                &tool_specs,
-                                &tool_envelope,
-                                turn.read_only_intent,
-                                turn.max_parallel_tools,
-                                &turn.task_contract,
-                                &mut turn.implementation_tracker,
-                                &mut turn.evidence,
-                                &mut turn.progress_tracker,
-                                &mut turn.tool_timeline,
-                                &mut turn.sched_tool_calls,
-                                &mut turn.sched_max_concurrent,
-                                &mut turn.sched_serial_runs,
-                                &turn.speculation_registry,
-                                &mut turn.program_fallback_next,
-                                &mut turn.program_fallback_used,
-                                &mut turn.plan_updated_goal,
-                                &mut turn.proposed_goal,
-                                &mut turn.turn_snapshot,
-                                &mut turn.turn_checkpoint_allowed,
-                                &mut turn.turn_checkpoint_created,
-                                &mut turn.fast_feedback,
-                                ui,
-                            )
-                            .await;
+                        // A batch retains several tool futures; boxing this
+                        // boundary keeps their state out of the loop future.
+                        let batch_result = Box::pin(self.execute_tool_batch(
+                            &calls,
+                            &mut completion_content,
+                            &tool_specs,
+                            &tool_envelope,
+                            turn.read_only_intent,
+                            turn.max_parallel_tools,
+                            &turn.task_contract,
+                            &mut turn.implementation_tracker,
+                            &mut turn.evidence,
+                            &mut turn.progress_tracker,
+                            &mut turn.tool_timeline,
+                            &mut turn.sched_tool_calls,
+                            &mut turn.sched_max_concurrent,
+                            &mut turn.sched_serial_runs,
+                            &turn.speculation_registry,
+                            &mut turn.program_fallback_next,
+                            &mut turn.program_fallback_used,
+                            &mut turn.plan_updated_goal,
+                            &mut turn.proposed_goal,
+                            &mut turn.turn_snapshot,
+                            &mut turn.turn_checkpoint_allowed,
+                            &mut turn.turn_checkpoint_created,
+                            &mut turn.fast_feedback,
+                            ui,
+                        ))
+                        .await;
                         turn.phase_latencies.tool_batch_ms = turn
                             .phase_latencies
                             .tool_batch_ms
@@ -1707,44 +1708,7 @@ impl crate::Agent {
         // The final cancellation check commits the body's outcome. Entry owns
         // the awaitable diagnostic append after this future returns, so a
         // cancellation during publication cannot append a conflicting receipt.
-        // Automatic post-mortem intake: bad outcomes become findings-ledger
-        // records so `hi metrics` surfaces failure patterns without anyone
-        // spelunking raw transcripts. Best-effort by design.
-        if self.config.memory.learning && crate::learning::outcome_warrants_finding(&outcome) {
-            let ts = std::time::SystemTime::now()
-                .duration_since(std::time::UNIX_EPOCH)
-                .map(|d| d.as_secs())
-                .unwrap_or(0);
-            let state_root = self.runtime.state_root().to_path_buf();
-            let finding = crate::learning::Finding {
-                ts,
-                session_id: self.session.as_deref().and_then(crate::SessionSink::id),
-                turn: Some(self.turn_count),
-                status: outcome.status,
-                stop_reason: outcome.stop_reason,
-                verification: outcome.verification,
-                review: outcome.review,
-                review_unavailable_reason: self
-                    .report
-                    .last_turn_telemetry
-                    .review_unavailable_reason
-                    .clone(),
-                last_no_progress_reason: self
-                    .report
-                    .last_turn_telemetry
-                    .last_no_progress_reason
-                    .clone(),
-                changed_files: outcome.changed_files.len(),
-                model: outcome.effective_route.model.clone(),
-                hint_active: self.task.active_hint_shape.clone(),
-                failure_shape: crate::learning::tool_failure_shape(
-                    &self.report.last_turn_telemetry.tool_timeline,
-                ),
-            };
-            tokio::task::spawn_blocking(move || {
-                crate::learning::append_finding(&state_root, &finding);
-            });
-        }
+        self.record_terminal_finding(&outcome);
         self.workspace.clear_active_baselines();
         Ok(outcome)
         }.await;

crates/hi-agent/src/agent/turn/mod.rs

@@ -44,6 +44,7 @@ mod speculation;
 mod state;
 mod steer;
 mod suggest;
+mod terminal_finding;
 mod terminal_receipt;
 mod terminal_verification;
 mod terminal_workspace;

crates/hi-agent/src/agent/turn/model_retry.rs

@@ -21,8 +21,6 @@ use super::speculation::SpeculationRegistry;
 
 mod compat_telemetry;
 use compat_telemetry::record_compat_fallback;
-#[cfg(test)]
-use compat_telemetry::{COMPAT_FALLBACK_LIMIT, COMPAT_FALLBACK_PREFIX};
 pub(super) const COMPLETED_PLAN_EMPTY_RECAP_FALLBACK: &str = "The plan is complete and the successful tool results were retained. The provider did not return a final recap.";
 /// After keep-working has already spent its chance, a further invalid-tool
 /// storm must not fail the turn as `no_progress` with no recap. One ChatOnly
@@ -464,7 +462,7 @@ impl crate::Agent {
                 // Grok-build starts a new generation after a format reminder.
                 // Sharing the 4-send ledger turns those retries into
                 // AttemptsExhausted and kills the turn.
-                retry_state.execution = retry_state.execution.fresh_operation();
+                retry_state.start_protocol_recovery(&err);
                 let protocol_retries = retry_state.protocol_retries;
                 if request_no_progress_final_answer {
                     // The live no-progress flag remains sticky in the caller,
@@ -508,7 +506,7 @@ impl crate::Agent {
                 self.emit_usage(ui);
                 retry_state.protocol_text_fallbacks += 1;
                 retry_state.record_recovery_attempt();
-                retry_state.execution = retry_state.execution.fresh_operation();
+                retry_state.start_protocol_recovery(&err);
                 *text_tool_fallback_next = true;
                 *force_tools_next = false;
                 ui.status(
@@ -546,7 +544,7 @@ impl crate::Agent {
                     // A different next action may succeed as a plain-text call
                     // even if structured JSON already burned the first fallback.
                     retry_state.protocol_text_fallbacks = 0;
-                    retry_state.execution = retry_state.execution.fresh_operation();
+                    retry_state.start_protocol_recovery(&err);
                     return Ok(ProviderStreamResult::Continue);
                 }
 
@@ -561,7 +559,7 @@ impl crate::Agent {
                     *force_tools_next = false;
                     *text_tool_fallback_next = false;
                     retry_state.protocol_retries = 0;
-                    retry_state.execution = retry_state.execution.fresh_operation();
+                    retry_state.start_protocol_recovery(&err);
                     *continue_total_nudges = continue_total_nudges.saturating_add(1);
                     self.messages
                         .push_nudge_or_fold(NudgeKind::Continue, PROTOCOL_EXHAUSTION_WRAP_UP_NUDGE);
@@ -822,30 +820,3 @@ impl crate::Agent {
         }
     }
 }
-
-#[cfg(test)]
-mod diagnostic_tests {
-    use super::*;
-
-    #[test]
-    fn compatibility_fallbacks_are_deduplicated_and_bounded() {
-        let mut fallbacks = Vec::new();
-        for index in 0..100 {
-            record_compat_fallback(&mut fallbacks, format!("fallback-{index}"));
-        }
-        record_compat_fallback(&mut fallbacks, "fallback-0".into());
-
-        assert_eq!(fallbacks.len(), COMPAT_FALLBACK_LIMIT);
-        assert_eq!(fallbacks.first().map(String::as_str), Some("fallback-0"));
-        assert_eq!(
-            fallbacks.get(COMPAT_FALLBACK_PREFIX).map(String::as_str),
-            Some("fallback-99")
-        );
-        assert!(
-            fallbacks
-                .last()
-                .is_some_and(|marker| marker.contains("37 additional compatibility events omitted")),
-            "exact dropped count is surfaced in the bounded diagnostic: {fallbacks:?}"
-        );
-    }
-}