Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 14 additions & 7 deletions cmd/entire/cli/strategy/manual_commit_condensation.go
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ func prepareTranscriptForStorage(
return externalized, assets, int64(len(sanitized))
}

// prepareTaskTranscriptForStorage runs the sanitize -> externalize -> redact
// prepareTaskTranscriptForStorage runs the sanitize -> externalize -> cap -> redact
// chain for a single subagent transcript, mirroring prepareTranscriptForStorage's
// first two steps and then completing with the same redaction the session
// transcript gets (see redactSessionTranscript). It is a SEPARATE entry point
Expand All @@ -228,10 +228,11 @@ func prepareTranscriptForStorage(
//
// It also carries checkpoint.prepareSubagentTranscript's size guard, for the
// same reason: agent-<agent-id>.jsonl is neither chunked nor capped, and
// redaction runs at roughly 220ms/MB. The cap is measured against the SANITIZED
// bytes, not the raw ones — sanitizing strips the bulk (Codex encrypted_content
// runs to ~20% of a rollout's bytes), so measuring raw would drop a transcript
// oversized only by payloads about to be discarded.
// redaction runs at roughly 220ms/MB. The cap is measured against the
// EXTERNALIZED bytes (after sanitization and image extraction). This ensures
// that a transcript which is only large because of inline images can still be
// stored if those images are externalized into separate assets, while still
// protecting the metadata branch from oversized text blobs.
func prepareTaskTranscriptForStorage(
ctx, logCtx context.Context,
ag agent.Agent,
Expand All @@ -240,16 +241,22 @@ func prepareTaskTranscriptForStorage(
raw []byte,
) (redacted redact.RedactedBytes, assets []cpkg.TranscriptAsset, tooLarge bool, err error) {
sanitized := agent.SanitizeTranscriptForStorage(ag, raw)
if len(sanitized) > agent.MaxChunkSize {
externalized, assets := externalizeSessionImages(ctx, logCtx, state, sanitized)

// cap measures the size of the transcript that will be stored in git (after
// sanitization and image externalization, but before redaction). Oversized
// transcripts are dropped to keep the metadata branch pushable.
if len(externalized) > agent.MaxChunkSize {
Comment on lines +246 to +249
logging.Warn(logCtx, "subagent transcript exceeds the blob size cap; storing task without it",
slog.String("session_id", state.SessionID),
slog.String("path", path),
slog.Int("raw_bytes", len(raw)),
slog.Int("sanitized_bytes", len(sanitized)),
slog.Int("externalized_bytes", len(externalized)),
slog.Int("cap", agent.MaxChunkSize))
return redact.RedactedBytes{}, nil, true, nil
}
externalized, assets := externalizeSessionImages(ctx, logCtx, state, sanitized)

redacted, _, err = redactSessionTranscript(logCtx, externalized)
if err != nil {
return redact.RedactedBytes{}, nil, false, err
Expand Down
98 changes: 98 additions & 0 deletions cmd/entire/cli/strategy/manual_commit_condensation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package strategy

import (
"context"
"encoding/base64"
"encoding/json"
"os"
"os/exec"
Expand Down Expand Up @@ -1188,3 +1189,100 @@ func TestCheckpointStepCount(t *testing.T) {
})
}
}

// TestCondenseSession_TaskRecordImageExternalizationBoundary verifies the fix
// for #2063: a task transcript whose RAW size (including base64 images) exceeds
// the cap, but whose EXTERNALIZED size is within the cap, must be stored
// successfully.
func TestCondenseSession_TaskRecordImageExternalizationBoundary(t *testing.T) {
sessionID := "2026-08-23-task-image-boundary"
repo, state := setupCondensableSessionWithTranscript(t, sessionID)
// Enable image externalization for the test via env.
t.Setenv("ENTIRE_EXTERNALIZE_IMAGES", "1")
state.AgentType = agent.AgentTypeClaudeCode
dir := t.TempDir()
agentTranscriptPath := filepath.Join(dir, "agent-transcript.jsonl")

// Helper to write a transcript with a base64 image of specified decoded size.
writeImageTranscript := func(path string, decodedSize int) {
f, err := os.Create(path)
require.NoError(t, err)
defer f.Close()

_, err = f.WriteString(`{"role":"assistant","content":[{"type":"image","source":{"type":"base64","media_type":"image/png","data":"`)
require.NoError(t, err)

// Stream base64 to avoid huge memory allocations.
// 3 bytes -> 4 base64 chars.
buf := make([]byte, 3072) // multiple of 3
for i := 0; i < decodedSize; i += len(buf) {
n := len(buf)
if i+n > decodedSize {
n = decodedSize - i
}
for j := 0; j < n; j++ {
buf[j] = byte((i + j) % 256)
}
_, err = f.WriteString(base64.StdEncoding.EncodeToString(buf[:n]))
require.NoError(t, err)
}

_, err = f.WriteString(`"}}],"message":{"id":"msg_123"}}` + "\n")
require.NoError(t, err)
}

// Use small test boundaries to avoid huge disk writes.
// We use the imageextract package's internal variable.
// Since we are in the strategy package, we need to export it or use the real cap.
// Given the constraints, we will stick to the real 40MiB size for the primary test
// but add a comment explaining why we aren't using a smaller mock.
//
// Note: Case 1 already uses 40MiB which is > 32MiB (approx base64 for 50MiB)
// but < 50MiB. Actually, 40MiB decoded -> 53.3MiB base64, which is > 50MiB cap.
// The goal is to prove that after externalization (placeholder only), it's < 50MiB.

// Case 1: 40 MiB decoded -> ~53.3 MiB base64.
// This exceeds agent.MaxChunkSize (50 MiB) in raw form.
// After externalization, the transcript will only contain a placeholder.
writeImageTranscript(agentTranscriptPath, 40*1024*1024)

state.TaskRecords = []session.TaskRecord{
{
ToolUseID: "toolu_boundary",
AgentID: "agent-boundary",
DeclaredTranscriptPath: agentTranscriptPath,
CompletedAt: time.Now(),
},
}
require.NoError(t, SaveSessionState(context.Background(), state))
checkpointID := id.MustCheckpointID("aabbccdd2063")
result, err := (&ManualCommitStrategy{}).CondenseSession(context.Background(), repo, checkpointID, state, nil)
require.NoError(t, err)
require.False(t, result.Skipped)

// Verify the transcript was stored.
jsonl, ok := checkpointTaskFile(t, repo, checkpointID, "tasks/toolu_boundary/agent-agent-boundary.jsonl")
require.True(t, ok, "transcript must be stored because externalized size is under cap")
require.Contains(t, jsonl, "entire-asset:assets/", "transcript must contain the asset placeholder")
require.Less(t, len(jsonl), 1024, "transcript must be shrunken")

// Case 2: Image that is NOT externalized and hits the 50MiB cap.
// Since we can't easily mock agent.MaxChunkSize (it's a const),
// we use a 51MiB image. Because it's > 50MiB DECODED, it stays inline
// and then prepareTaskTranscriptForStorage hits the MaxChunkSize cap.
writeImageTranscript(agentTranscriptPath, 51*1024*1024)

state.TaskRecords[0].ToolUseID = "toolu_oversize"
checkpointID2 := id.MustCheckpointID("aabbccdd2064")
_, err = (&ManualCommitStrategy{}).CondenseSession(context.Background(), repo, checkpointID2, state, nil)
require.NoError(t, err)

taskJSON, ok := checkpointTaskFile(t, repo, checkpointID2, "tasks/toolu_oversize/task.json")
require.True(t, ok)
var meta struct {
TranscriptUnavailableReason string `json:"transcript_unavailable_reason"`
}
require.NoError(t, json.Unmarshal([]byte(taskJSON), &meta))
require.Equal(t, taskTranscriptReasonTooLarge, meta.TranscriptUnavailableReason,
"transcript with >50MiB decoded image must hit the cap")
}