diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index f1151c4..de342bb 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -45,6 +45,14 @@ container's current directory, and saves combined stdout/stderr to `/logs/agent/nanopycodeagent.txt`. It uses the CLI's 50-turn default; override that with `--agent-kwarg max_turns=20`. +For the per-reply generation limit, pass `--agent-kwarg max_tokens=32768`. +This becomes `--max-tokens 32768` in the container and overrides the forwarded +`ANTHROPIC_MAX_TOKENS` environment variable. When omitted, the adapter sends no +token flag, so the installed agent's environment/settings/default applies +(32768 in the version introducing this option). Older releases require omitting +the new option. The effective budget is recorded in the startup log and in +`agent.extra.max_tokens` in the ATIF trajectory. + The adapter also asks the agent to write an ATIF-v1.7 trajectory directly to `/logs/agent/trajectory.json`. Harbor collects that file as the trial's native ATIF output and backfills prompt, completion, cache-token, and cost totals into diff --git a/benchmarks/harbor/README.zh-CN.md b/benchmarks/harbor/README.zh-CN.md index 1af3f2f..15620c4 100644 --- a/benchmarks/harbor/README.zh-CN.md +++ b/benchmarks/harbor/README.zh-CN.md @@ -41,6 +41,12 @@ adapter 通过 stdin 发送任务指令，在 task 容器的当前目录中运 的 stdout/stderr 保存到 `/logs/agent/nanopycodeagent.txt`。它默认沿用 CLI 的 50 轮限制；可以通过 `--agent-kwarg max_turns=20` 覆盖此设置。 +每次回复的生成上限可通过 `--agent-kwarg max_tokens=32768` 指定，它会转换为容器内的 +`--max-tokens 32768`，优先于透传的 `ANTHROPIC_MAX_TOKENS` 环境变量。未指定时， +adapter 不添加该 flag，沿用已安装 agent 的环境变量／配置文件／默认值（引入该参数的 +版本默认为 32768）。安装旧版本时需省略新参数。生效预算会记录到启动日志和 ATIF +trajectory 的 `agent.extra.max_tokens`。 + adapter 还会要求 agent 将 ATIF-v1.7 trajectory 直接写入 `/logs/agent/trajectory.json`。Harbor 会把该文件作为 trial 的原生 ATIF 输出采集， 并将 prompt、completion、cache token 和 cost 汇总回填到 agent result。trajectory diff --git a/benchmarks/harbor/results/tb21-generation-budget-65536-20260912.json b/benchmarks/harbor/results/tb21-generation-budget-65536-20260912.json new file mode 100644 index 0000000..f4f150a --- /dev/null +++ b/benchmarks/harbor/results/tb21-generation-budget-65536-20260912.json @@ -0,0 +1,1449 @@ +{ + "schema_version": 1, + "generated_at": "2026-09-12T17:19:18.073195+00:00", + "purpose": "Generation-budget diagnostics: 24 attempts over 19 distinct tasks, including separate infrastructure and recorder retries. This is not a single leaderboard run or a controlled budget-only comparison.", + "agent_ref": "7a610cbf87ee3ac27f8cdb8a4329076f210dee8d", + "baseline_agent_ref": "2b8309794cb9e00cb4d3b08baf1e4733153105e0", + "baseline_result": "benchmarks/harbor/results/tb21-flash0731-pilot20-20260906.json", + "wheel_sha256": "91996b1d4c70af8062318a38b3af9235c85d93c9685e10ac1323f3bef3625425", + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "dataset": "terminal-bench/terminal-bench-2-1", + "dataset_ref": "sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a", + "configuration": { + "max_tokens": 65536, + "project_default_max_tokens": 32768, + "max_turns": 50, + "agent_execution_limit_seconds": 3600, + "finalization_grace_seconds": 120, + "harbor_outer_limit_seconds": 3780, + "agent_setup_limit_seconds": 1080, + "verifier_limits": "Native task deadlines", + "maximum_simultaneous_trials": 2, + "automatic_harbor_retries": 0 + }, + "observations": { + "remaining18_initial": { + "attempts": 18, + "passed": 5, + "scored_zero": 8, + "unscored": 5 + }, + "remaining18_baseline_passed": 8, + "supplemental_attempts_on_remaining18": 5, + "supplemental_new_passes": 0, + "any_max_tokens_stop": false, + "any_agent_deadline": false, + "tasks_using_more_than_32768_in_a_reply": [ + "schemelike-metacircular-eval", + "regex-chess", + "dna-assembly" + ], + "passing_tasks_using_more_than_32768_in_a_reply": [], + "tasks_with_missing_or_malformed_tool_inputs_raising_KeyError": [ + "circuit-fibsqrt", + "qemu-alpine-ssh", + "llm-inference-batching-scheduler", + "dna-assembly" + ], + "native_trajectories": 20, + "reconstructed_trajectories": 1, + "no_trajectory_before_agent_setup_completed": 3 + }, + "billing_known_lower_bound_usd": 0.280588638, + "billing_total_usd": null, + "billing_limit": "Tensor-parallelism billing lookup returned 404 on three checks. Separate receipts price the interrupted Scheme and scheduler requests without repairing missing native usage. The failed DNA TLS bootstrap received no response headers. Unknown usage and cost are not imputed as zero.", + "comparison_limits": [ + "The 8192 baseline used another source revision and native task deadlines; dependencies, sampling, cache, and backend were not fixed.", + "All 18 initial task refs and cached image digests match the baseline. Instrumentation and installation changes are recorded per group.", + "The initial scheduler failed inside periodic stack dumping; its subsequent control avoided SIGSEGV but failed on malformed tool input.", + "The first cached DNA bootstrap omitted CA certificates; a public HTTPS probe confirmed the omission and a further trial restored original system dependencies.", + "The QEMU setup retry recorded reward zero after verifier dependency installation failed; pytest did not run.", + "regex-log was not rerun at 65536. Its two prior passes used 32768 and are excluded from this file.", + "Full local HTTP bodies and Journals are not uploaded; only diagnostic metadata is published." + ], + "experiments": [ + { + "kind": "compressor", + "job": "tb21-compressor-65536-3600s-trace-20260912-7a610cb", + "purpose": "One compressor trial at 65536; the maximum reply was below 32768.", + "local_record": "jobs/tb21-compressor-65536-3600s-trace-20260912-7a610cb-record", + "counts": { + "attempts": 1, + "passed": 1, + "scored_zero": 0, + "unscored": 0 + }, + "billing_known_cost_usd": 0.018819072, + "billing_cost_complete": true, + "workflow_sha256": { + "run.py": "ffd0714f620c499a65da5c07fde9fdd4beb09f078616cf0de7f73a6092353379", + "diagnostic_adapter.py": "6b00af9113e2010f0f149509ba94a48c525aa408725c2742d38698d52940f95f", + "diagnostic_runner.py": "9a83b3c0915d8daae53e5654096c9e312d3ab11d594cb36e851be982dc852654", + "status.py": "c1ae99b78f05efdcb37fff325ab0309d1b3d763741bb6372b674e373b3c78ebe", + "checkpoint_artifacts.py": "17785880a6ed1941fb49d92df3388ec76e36eba2def4e3989e3e9efa6494d874", + "analyze.py": "ed5be7c419a88ff96278f916aa0fdfd7ea77d5ecdf37890361e7ae0035692819", + "monitor.py": "af316ea66ab85681408d5cf5dc76fcae4531cee970952b3cf06999bf3763b9c6", + "checkpoint_when_needed.py": "c86a508b0802b788f510953f5d0c66730960c2922673ddda4acc8acee33cc477" + }, + "trials": [ + { + "task": "terminal-bench/write-compressor", + "task_ref": "sha256:d9ddd9a8e925e2c566b37b2492cbf995afecefe58874e4043ef78d7f3c892c7e", + "trial": "write-compressor__hzfqDFs", + "reward": 1.0, + "baseline_reward": 0.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 0.976, + "agent_setup": 278.203, + "agent_execution": 1976.369, + "verifier": 169.073 + }, + "terminal_outcome": "completed", + "terminal_error_type": null, + "exception_type": null, + "failure_category": "passed", + "model_calls_started": 32, + "model_calls_completed": 32, + "tool_calls": 32, + "max_reply_output_tokens": 32073, + "replies_exceeding_32768": 0, + "last_stop_reason": "end_turn", + "usage_complete": true, + "cost_complete": true, + "final_metrics": { + "total_steps": 33, + "total_prompt_tokens": 1528746, + "total_completion_tokens": 59563, + "total_cached_tokens": 1471744, + "total_cost_usd": 0.018819072 + }, + "billing_known_cost_usd": 0.018819072, + "billing_cost_complete": true, + "billing_total_cost_usd": 0.018819072, + "supplemental_cost_usd": 0.0, + "atif_valid": true, + "native_journal_projection_matches": true, + "journal_truncated_entries": 0, + "verifier_timed_out": false, + "runtime": { + "python": "3.13.9 (main, Nov 19 2025, 22:47:49) [Clang 21.1.4 ]", + "packages": { + "nanopycodeagent": "0.8.1.dev16+g7a610cbf8.d20260909", + "anthropic": "1.5.0", + "httpx": "0.28.1", + "httpx2": "2.12.0", + "pydantic": "2.13.5" + }, + "sdk_http_module": "httpx2" + }, + "trajectory_source": "native", + "worker_exit_code": 0, + "local_trajectory": "jobs/tb21-compressor-65536-3600s-trace-20260912-7a610cb/write-compressor__hzfqDFs/agent/trajectory.json", + "finding": "All three tests passed; output is 2263 bytes. Peak reply 32073 does not establish that doubling 32768 was necessary; this execution exceeded the original 900-second deadline." + } + ] + }, + { + "kind": "initial18", + "job": "tb21-remaining18-65536-3600s-trace-20260912-7a610cb", + "purpose": "One initial attempt for each of the other 18 pilot tasks.", + "local_record": "jobs/tb21-remaining18-65536-3600s-trace-20260912-7a610cb-record", + "counts": { + "attempts": 18, + "passed": 5, + "scored_zero": 8, + "unscored": 5 + }, + "billing_known_cost_usd": 0.242647727, + "billing_cost_complete": false, + "workflow_sha256": { + "diagnostic_adapter.py": "6b00af9113e2010f0f149509ba94a48c525aa408725c2742d38698d52940f95f", + "diagnostic_runner.py": "7619be9e15e514490fec10ac10c35cc9cc27c8f6dab0f7593462c25480761160", + "status.py": "c1ae99b78f05efdcb37fff325ab0309d1b3d763741bb6372b674e373b3c78ebe", + "run.py": "d168f5148c8c9d4a7e4bf62cca133e4d123df87c799c563ffc0f71e7922b56b6", + "monitor.py": "fad87a820be21b8187740d2a5a2f1b8e81550c34a4c8caa1e106335389215610" + }, + "trials": [ + { + "task": "terminal-bench/torch-tensor-parallelism", + "task_ref": "sha256:f32ce74a5aeb6638480247ab799fe46127bbee631acdd0921b0f394ec49b3684", + "trial": "torch-tensor-parallelism__EEWJenL", + "reward": null, + "baseline_reward": 0.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 1.079, + "agent_setup": 1038.105, + "agent_execution": 427.656, + "verifier": 900.074 + }, + "terminal_outcome": null, + "terminal_error_type": "RemoteProtocolError", + "exception_type": "NonZeroAgentExitCodeError", + "failure_category": "RemoteProtocolError", + "model_calls_started": 1, + "model_calls_completed": 0, + "tool_calls": 0, + "max_reply_output_tokens": null, + "replies_exceeding_32768": 0, + "last_stop_reason": null, + "usage_complete": false, + "cost_complete": false, + "final_metrics": { + "total_steps": 2, + "extra": { + "usage_complete": false + } + }, + "billing_known_cost_usd": 0.0, + "billing_cost_complete": false, + "billing_total_cost_usd": null, + "supplemental_cost_usd": 0.0, + "atif_valid": true, + "native_journal_projection_matches": true, + "journal_truncated_entries": 0, + "verifier_timed_out": true, + "runtime": { + "python": "3.13.9 (main, Nov 19 2025, 22:47:49) [Clang 21.1.4 ]", + "packages": { + "nanopycodeagent": "0.8.1.dev16+g7a610cbf8.d20260909", + "anthropic": "1.5.0", + "httpx": "0.28.1", + "httpx2": "2.12.0", + "pydantic": "2.13.5" + }, + "sdk_http_module": "httpx2" + }, + "trajectory_source": "native", + "worker_exit_code": 1, + "local_trajectory": "jobs/tb21-remaining18-65536-3600s-trace-20260912-7a610cb/torch-tensor-parallelism__EEWJenL/agent/trajectory.json", + "finding": "The first response stream ended in RemoteProtocolError before any completed reply or tool call. Verifier dependency installation separately timed out after 900 seconds; no official score. Billing returned 404 on three checks, leaving usage and cost unknown." + }, + { + "task": "terminal-bench/schemelike-metacircular-eval", + "task_ref": "sha256:58130c2166c3115276dc8592f358e326ff2d81ea852e3d88636c82fd1dff57e6", + "trial": "schemelike-metacircular-eval__Xuw2yU4", + "reward": 0.0, + "baseline_reward": 0.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 1.074, + "agent_setup": 877.31, + "agent_execution": 1803.98, + "verifier": 26.426 + }, + "terminal_outcome": null, + "terminal_error_type": "RemoteProtocolError", + "exception_type": "NonZeroAgentExitCodeError", + "failure_category": "RemoteProtocolError", + "model_calls_started": 19, + "model_calls_completed": 18, + "tool_calls": 22, + "max_reply_output_tokens": 38496, + "replies_exceeding_32768": 1, + "last_stop_reason": "tool_use", + "usage_complete": false, + "cost_complete": true, + "final_metrics": { + "total_steps": 20, + "extra": { + "usage_complete": false + }, + "total_cost_usd": 0.021944872 + }, + "billing_known_cost_usd": 0.023049472, + "billing_cost_complete": true, + "billing_total_cost_usd": 0.023049472, + "supplemental_cost_usd": 0.0011046, + "atif_valid": true, + "native_journal_projection_matches": true, + "journal_truncated_entries": 0, + "verifier_timed_out": false, + "runtime": { + "python": "3.13.7 (main, Aug 15 2025, 22:13:55) [GCC 12.2.0]", + "packages": { + "nanopycodeagent": "0.8.1.dev16+g7a610cbf8.d20260909", + "anthropic": "1.5.0", + "httpx": "0.28.1", + "httpx2": "2.12.0", + "pydantic": "2.13.5" + }, + "sdk_http_module": "httpx2" + }, + "trajectory_source": "native", + "worker_exit_code": 1, + "local_trajectory": "jobs/tb21-remaining18-65536-3600s-trace-20260912-7a610cb/schemelike-metacircular-eval__Xuw2yU4/agent/trajectory.json", + "finding": "Reply 19 ended in RemoteProtocolError after 18 complete replies. The interpreter passed only 4 of 63 subcases. Reply 6 completed normally with 38496 output tokens; the eventual interruption did not report max_tokens." + }, + { + "task": "terminal-bench/kv-store-grpc", + "task_ref": "sha256:973c5d4c111fb61a344457936f1c36400acd2d9e44389e7b319586fe23a7a307", + "trial": "kv-store-grpc__fGX6VEa", + "reward": 0.0, + "baseline_reward": 1.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 1.073, + "agent_setup": 152.627, + "agent_execution": 563.006, + "verifier": 59.378 + }, + "terminal_outcome": "completed", + "terminal_error_type": null, + "exception_type": null, + "failure_category": "verifier_failed", + "model_calls_started": 17, + "model_calls_completed": 17, + "tool_calls": 16, + "max_reply_output_tokens": 2772, + "replies_exceeding_32768": 0, + "last_stop_reason": "end_turn", + "usage_complete": true, + "cost_complete": true, + "final_metrics": { + "total_steps": 18, + "total_prompt_tokens": 120910, + "total_completion_tokens": 11085, + "total_cached_tokens": 112384, + "total_cost_usd": 0.002126912 + }, + "billing_known_cost_usd": 0.002126912, + "billing_cost_complete": true, + "billing_total_cost_usd": 0.002126912, + "supplemental_cost_usd": 0.0, + "atif_valid": true, + "native_journal_projection_matches": true, + "journal_truncated_entries": 0, + "verifier_timed_out": false, + "runtime": { + "python": "3.13.7 (main, Aug 15 2025, 22:13:55) [GCC 12.2.0]", + "packages": { + "nanopycodeagent": "0.8.1.dev16+g7a610cbf8.d20260909", + "anthropic": "1.5.0", + "httpx": "0.28.1", + "httpx2": "2.12.0", + "pydantic": "2.13.5" + }, + "sdk_http_module": "httpx2" + }, + "trajectory_source": "native", + "worker_exit_code": 0, + "local_trajectory": "jobs/tb21-remaining18-65536-3600s-trace-20260912-7a610cb/kv-store-grpc__fGX6VEa/agent/trajectory.json", + "finding": "SetValRequest lacks the required value field. Five of seven tests passed, including service availability; the agent completed normally." + }, + { + "task": "terminal-bench/pypi-server", + "task_ref": "sha256:1a1e0542f58e2d3362fec17a9bbb98667717d9a4a3e9a4c8413d3150a4fa0ff1", + "trial": "pypi-server__cbM9sLf", + "reward": 1.0, + "baseline_reward": 1.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 0.991, + "agent_setup": 610.047, + "agent_execution": 315.517, + "verifier": 34.26 + }, + "terminal_outcome": "completed", + "terminal_error_type": null, + "exception_type": null, + "failure_category": "passed", + "model_calls_started": 15, + "model_calls_completed": 15, + "tool_calls": 17, + "max_reply_output_tokens": 695, + "replies_exceeding_32768": 0, + "last_stop_reason": "end_turn", + "usage_complete": true, + "cost_complete": true, + "final_metrics": { + "total_steps": 16, + "total_prompt_tokens": 72990, + "total_completion_tokens": 5216, + "total_cached_tokens": 64512, + "total_cost_usd": 0.001272496 + }, + "billing_known_cost_usd": 0.001272496, + "billing_cost_complete": true, + "billing_total_cost_usd": 0.001272496, + "supplemental_cost_usd": 0.0, + "atif_valid": true, + "native_journal_projection_matches": true, + "journal_truncated_entries": 0, + "verifier_timed_out": false, + "runtime": { + "python": "3.13.7 (main, Aug 15 2025, 22:13:55) [GCC 12.2.0]", + "packages": { + "nanopycodeagent": "0.8.1.dev16+g7a610cbf8.d20260909", + "anthropic": "1.5.0", + "httpx": "0.28.1", + "httpx2": "2.12.0", + "pydantic": "2.13.5" + }, + "sdk_http_module": "httpx2" + }, + "trajectory_source": "native", + "worker_exit_code": 0, + "local_trajectory": "jobs/tb21-remaining18-65536-3600s-trace-20260912-7a610cb/pypi-server__cbM9sLf/agent/trajectory.json", + "finding": "The official API test passed after installing the task package from the local PyPI server." + }, + { + "task": "terminal-bench/dna-assembly", + "task_ref": "sha256:e41a8e94d86019949b08d3b5f88a85f6d943ba0fd85d5e1d5ebb95cb8f66223f", + "trial": "dna-assembly__HVY9Uhs", + "reward": null, + "baseline_reward": 0.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 0.965, + "agent_setup": 349.923, + "agent_execution": null, + "verifier": null + }, + "terminal_outcome": null, + "exception_type": "NetworkConnectionError", + "failure_category": "NetworkConnectionError", + "last_stop_reason": null, + "usage_complete": false, + "cost_complete": false, + "final_metrics": null, + "verifier_timed_out": false, + "runtime": null, + "trajectory_source": null, + "worker_exit_code": null, + "local_trajectory": null, + "finding": "Setup failed while downloading uv with a TLS error before model execution; no trajectory or official score." + }, + { + "task": "terminal-bench/torch-pipeline-parallelism", + "task_ref": "sha256:db605337c749a872cea7b5b413429b3915bb4c3efe0f7875f0c46ce81bd8c4fb", + "trial": "torch-pipeline-parallelism__gmVo6Cp", + "reward": null, + "baseline_reward": 0.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 0.999, + "agent_setup": 323.847, + "agent_execution": 1831.106, + "verifier": 900.003 + }, + "terminal_outcome": "max_turns_exhausted", + "terminal_error_type": null, + "exception_type": "VerifierTimeoutError", + "failure_category": "VerifierTimeoutError", + "model_calls_started": 50, + "model_calls_completed": 50, + "tool_calls": 56, + "max_reply_output_tokens": 22329, + "replies_exceeding_32768": 0, + "last_stop_reason": "tool_use", + "usage_complete": true, + "cost_complete": true, + "final_metrics": { + "total_steps": 51, + "total_prompt_tokens": 2711266, + "total_completion_tokens": 65109, + "total_cached_tokens": 2645248, + "total_cost_usd": 0.029011424 + }, + "billing_known_cost_usd": 0.029011424, + "billing_cost_complete": true, + "billing_total_cost_usd": 0.029011424, + "supplemental_cost_usd": 0.0, + "atif_valid": true, + "native_journal_projection_matches": true, + "journal_truncated_entries": 0, + "verifier_timed_out": true, + "runtime": { + "python": "3.13.9 (main, Nov 19 2025, 22:47:49) [Clang 21.1.4 ]", + "packages": { + "nanopycodeagent": "0.8.1.dev16+g7a610cbf8.d20260909", + "anthropic": "1.5.0", + "httpx": "0.28.1", + "httpx2": "2.12.0", + "pydantic": "2.13.5" + }, + "sdk_http_module": "httpx2" + }, + "trajectory_source": "native", + "worker_exit_code": 0, + "local_trajectory": "jobs/tb21-remaining18-65536-3600s-trace-20260912-7a610cb/torch-pipeline-parallelism__gmVo6Cp/agent/trajectory.json", + "finding": "Verifier timed out after 900 seconds while installing Torch/CUDA dependencies; no official test result. Agent independently exhausted 50 rounds without max_tokens truncation." + }, + { + "task": "terminal-bench/qemu-alpine-ssh", + "task_ref": "sha256:60b7050b0e0aa51641208cf65766743d340e59575db2c4d2f8628240846c2a28", + "trial": "qemu-alpine-ssh__SeNa8Uy", + "reward": null, + "baseline_reward": 1.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 1.046, + "agent_setup": 41.51, + "agent_execution": null, + "verifier": null + }, + "terminal_outcome": null, + "exception_type": "NonZeroAgentExitCodeError", + "failure_category": "NonZeroAgentExitCodeError", + "last_stop_reason": null, + "usage_complete": false, + "cost_complete": false, + "final_metrics": null, + "verifier_timed_out": false, + "runtime": null, + "trajectory_source": null, + "worker_exit_code": null, + "local_trajectory": null, + "finding": "APT dependency installation returned HTTP 404 before model execution; no trajectory or official score." + }, + { + "task": "terminal-bench/openssl-selfsigned-cert", + "task_ref": "sha256:d4afa2bd2a9ba1420db8d6cfde42ffdb4873ae2d955c35014e8da94444c83302", + "trial": "openssl-selfsigned-cert__RtV3EKQ", + "reward": 1.0, + "baseline_reward": 1.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 0.943, + "agent_setup": 105.513, + "agent_execution": 185.436, + "verifier": 229.946 + }, + "terminal_outcome": "completed", + "terminal_error_type": null, + "exception_type": null, + "failure_category": "passed", + "model_calls_started": 8, + "model_calls_completed": 8, + "tool_calls": 9, + "max_reply_output_tokens": 2800, + "replies_exceeding_32768": 0, + "last_stop_reason": "end_turn", + "usage_complete": true, + "cost_complete": true, + "final_metrics": { + "total_steps": 9, + "total_prompt_tokens": 29649, + "total_completion_tokens": 5069, + "total_cached_tokens": 25856, + "total_cost_usd": 0.000764088 + }, + "billing_known_cost_usd": 0.000764088, + "billing_cost_complete": true, + "billing_total_cost_usd": 0.000764088, + "supplemental_cost_usd": 0.0, + "atif_valid": true, + "native_journal_projection_matches": true, + "journal_truncated_entries": 0, + "verifier_timed_out": false, + "runtime": { + "python": "3.13.7 (main, Aug 15 2025, 22:13:55) [GCC 12.2.0]", + "packages": { + "nanopycodeagent": "0.8.1.dev16+g7a610cbf8.d20260909", + "anthropic": "1.5.0", + "httpx": "0.28.1", + "httpx2": "2.12.0", + "pydantic": "2.13.5" + }, + "sdk_http_module": "httpx2" + }, + "trajectory_source": "native", + "worker_exit_code": 0, + "local_trajectory": "jobs/tb21-remaining18-65536-3600s-trace-20260912-7a610cb/openssl-selfsigned-cert__RtV3EKQ/agent/trajectory.json", + "finding": "All six official certificate tests passed." + }, + { + "task": "terminal-bench/regex-chess", + "task_ref": "sha256:e763e0ac1c9759081af0a4a82ba51b8cf9ae5485a93de3bbe42d7d344597bd78", + "trial": "regex-chess__Lwdnros", + "reward": 0.0, + "baseline_reward": 0.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 0.992, + "agent_setup": 676.077, + "agent_execution": 2315.577, + "verifier": 41.16 + }, + "terminal_outcome": "max_turns_exhausted", + "terminal_error_type": null, + "exception_type": null, + "failure_category": "max_turns_exhausted", + "model_calls_started": 50, + "model_calls_completed": 50, + "tool_calls": 50, + "max_reply_output_tokens": 60483, + "replies_exceeding_32768": 1, + "last_stop_reason": "tool_use", + "usage_complete": true, + "cost_complete": true, + "final_metrics": { + "total_steps": 51, + "total_prompt_tokens": 7074131, + "total_completion_tokens": 165188, + "total_cached_tokens": 6823680, + "total_cost_usd": 0.081484747 + }, + "billing_known_cost_usd": 0.081484747, + "billing_cost_complete": true, + "billing_total_cost_usd": 0.081484747, + "supplemental_cost_usd": 0.0, + "atif_valid": true, + "native_journal_projection_matches": true, + "journal_truncated_entries": 0, + "verifier_timed_out": false, + "runtime": { + "python": "3.13.7 (main, Aug 15 2025, 22:13:55) [GCC 12.2.0]", + "packages": { + "nanopycodeagent": "0.8.1.dev16+g7a610cbf8.d20260909", + "anthropic": "1.5.0", + "httpx": "0.28.1", + "httpx2": "2.12.0", + "pydantic": "2.13.5" + }, + "sdk_http_module": "httpx2" + }, + "trajectory_source": "native", + "worker_exit_code": 0, + "local_trajectory": "jobs/tb21-remaining18-65536-3600s-trace-20260912-7a610cb/regex-chess__Lwdnros/agent/trajectory.json", + "finding": "Regex length check passed, but all three game tests failed. At the starting position, the produced move set contained an empty string instead of a legal successor FEN. Agent exhausted 50 rounds without generation truncation." + }, + { + "task": "terminal-bench/log-summary-date-ranges", + "task_ref": "sha256:27b074a2f10fff7606e096f3abd8dced418ad8fda0f53d88acbe477f2d9ceaf6", + "trial": "log-summary-date-ranges__U3uovPo", + "reward": 1.0, + "baseline_reward": 1.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 1.065, + "agent_setup": 89.586, + "agent_execution": 47.348, + "verifier": 42.298 + }, + "terminal_outcome": "completed", + "terminal_error_type": null, + "exception_type": null, + "failure_category": "passed", + "model_calls_started": 8, + "model_calls_completed": 8, + "tool_calls": 9, + "max_reply_output_tokens": 1392, + "replies_exceeding_32768": 0, + "last_stop_reason": "end_turn", + "usage_complete": true, + "cost_complete": true, + "final_metrics": { + "total_steps": 9, + "total_prompt_tokens": 42313, + "total_completion_tokens": 3457, + "total_cached_tokens": 31744, + "total_cost_usd": 0.000869217 + }, + "billing_known_cost_usd": 0.000869217, + "billing_cost_complete": true, + "billing_total_cost_usd": 0.000869217, + "supplemental_cost_usd": 0.0, + "atif_valid": true, + "native_journal_projection_matches": true, + "journal_truncated_entries": 0, + "verifier_timed_out": false, + "runtime": { + "python": "3.13.7 (main, Aug 15 2025, 22:13:55) [GCC 12.2.0]", + "packages": { + "nanopycodeagent": "0.8.1.dev16+g7a610cbf8.d20260909", + "anthropic": "1.5.0", + "httpx": "0.28.1", + "httpx2": "2.12.0", + "pydantic": "2.13.5" + }, + "sdk_http_module": "httpx2" + }, + "trajectory_source": "native", + "worker_exit_code": 0, + "local_trajectory": "jobs/tb21-remaining18-65536-3600s-trace-20260912-7a610cb/log-summary-date-ranges__U3uovPo/agent/trajectory.json", + "finding": "Summary file exists and its structure and counts match the expected output." + }, + { + "task": "terminal-bench/model-extraction-relu-logits", + "task_ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17", + "trial": "model-extraction-relu-logits__b7NJtqL", + "reward": null, + "baseline_reward": 0.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 1.022, + "agent_setup": 294.463, + "agent_execution": null, + "verifier": null + }, + "terminal_outcome": null, + "exception_type": "NetworkConnectionError", + "failure_category": "NetworkConnectionError", + "last_stop_reason": null, + "usage_complete": false, + "cost_complete": false, + "final_metrics": null, + "verifier_timed_out": false, + "runtime": null, + "trajectory_source": null, + "worker_exit_code": null, + "local_trajectory": null, + "finding": "Setup failed before model execution: curl 35 SSL_ERROR_SYSCALL while fetching the uv installer from astral.sh:443." + }, + { + "task": "terminal-bench/path-tracing", + "task_ref": "sha256:cf56094c881a488b27e9f204a638a7e78ed7d55e12dc3064108c93357190314c", + "trial": "path-tracing__XRchuVa", + "reward": 0.0, + "baseline_reward": 0.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 0.996, + "agent_setup": 96.189, + "agent_execution": 1378.18, + "verifier": 30.223 + }, + "terminal_outcome": "max_turns_exhausted", + "terminal_error_type": null, + "exception_type": null, + "failure_category": "max_turns_exhausted", + "model_calls_started": 50, + "model_calls_completed": 50, + "tool_calls": 59, + "max_reply_output_tokens": 32227, + "replies_exceeding_32768": 0, + "last_stop_reason": "tool_use", + "usage_complete": true, + "cost_complete": true, + "final_metrics": { + "total_steps": 51, + "total_prompt_tokens": 6616150, + "total_completion_tokens": 140239, + "total_cached_tokens": 6472960, + "total_cost_usd": 0.068679374 + }, + "billing_known_cost_usd": 0.068679374, + "billing_cost_complete": true, + "billing_total_cost_usd": 0.068679374, + "supplemental_cost_usd": 0.0, + "atif_valid": true, + "native_journal_projection_matches": true, + "journal_truncated_entries": 0, + "verifier_timed_out": false, + "runtime": { + "python": "3.13.9 (main, Nov 19 2025, 22:47:49) [Clang 21.1.4 ]", + "packages": { + "nanopycodeagent": "0.8.1.dev16+g7a610cbf8.d20260909", + "anthropic": "1.5.0", + "httpx": "0.28.1", + "httpx2": "2.12.0", + "pydantic": "2.13.5" + }, + "sdk_http_module": "httpx2" + }, + "trajectory_source": "native", + "worker_exit_code": 0, + "local_trajectory": "jobs/tb21-remaining18-65536-3600s-trace-20260912-7a610cb/path-tracing__XRchuVa/agent/trajectory.json", + "finding": "The required image.c was absent, causing file existence, compilation, dependency, output, and image similarity checks to fail. Agent exhausted 50 rounds; maximum reply was 32227 tokens with no truncation." + }, + { + "task": "terminal-bench/caffe-cifar-10", + "task_ref": "sha256:7b0045106d7d5af724efe96b610ba64f7893f5c88528401c573c4d47e384e2bf", + "trial": "caffe-cifar-10__ZNW82kp", + "reward": 0.0, + "baseline_reward": 0.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 1.009, + "agent_setup": 138.782, + "agent_execution": 1804.535, + "verifier": 39.622 + }, + "terminal_outcome": "max_turns_exhausted", + "terminal_error_type": null, + "exception_type": null, + "failure_category": "max_turns_exhausted", + "model_calls_started": 50, + "model_calls_completed": 50, + "tool_calls": 55, + "max_reply_output_tokens": 9316, + "replies_exceeding_32768": 0, + "last_stop_reason": "tool_use", + "usage_complete": true, + "cost_complete": true, + "final_metrics": { + "total_steps": 51, + "total_prompt_tokens": 1470151, + "total_completion_tokens": 28776, + "total_cached_tokens": 1427712, + "total_cost_usd": 0.015421336 + }, + "billing_known_cost_usd": 0.015421336, + "billing_cost_complete": true, + "billing_total_cost_usd": 0.015421336, + "supplemental_cost_usd": 0.0, + "atif_valid": true, + "native_journal_projection_matches": true, + "journal_truncated_entries": 0, + "verifier_timed_out": false, + "runtime": { + "python": "3.13.9 (main, Nov 19 2025, 22:47:49) [Clang 21.1.4 ]", + "packages": { + "nanopycodeagent": "0.8.1.dev16+g7a610cbf8.d20260909", + "anthropic": "1.5.0", + "httpx": "0.28.1", + "httpx2": "2.12.0", + "pydantic": "2.13.5" + }, + "sdk_http_module": "httpx2" + }, + "trajectory_source": "native", + "worker_exit_code": 0, + "local_trajectory": "jobs/tb21-remaining18-65536-3600s-trace-20260912-7a610cb/caffe-cifar-10__ZNW82kp/agent/trajectory.json", + "finding": "Only the prototxt existence check passed. Caffe binary and trained model were missing. The final tool result reported g++ not found and make exit 2; no model round remained to repair the build." + }, + { + "task": "terminal-bench/mteb-leaderboard", + "task_ref": "sha256:484f6d7008a05b5b8640fc6618a384b8c9447cd76f85416c8a595028d29bff9c", + "trial": "mteb-leaderboard__3RsuVW3", + "reward": 0.0, + "baseline_reward": 0.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 1.063, + "agent_setup": 569.825, + "agent_execution": 1020.893, + "verifier": 32.494 + }, + "terminal_outcome": "max_turns_exhausted", + "terminal_error_type": null, + "exception_type": null, + "failure_category": "max_turns_exhausted", + "model_calls_started": 50, + "model_calls_completed": 50, + "tool_calls": 66, + "max_reply_output_tokens": 2747, + "replies_exceeding_32768": 0, + "last_stop_reason": "tool_use", + "usage_complete": true, + "cost_complete": true, + "final_metrics": { + "total_steps": 51, + "total_prompt_tokens": 2077822, + "total_completion_tokens": 26702, + "total_cached_tokens": 1988864, + "total_cost_usd": 0.009200885 + }, + "billing_known_cost_usd": 0.009200885, + "billing_cost_complete": true, + "billing_total_cost_usd": 0.009200885, + "supplemental_cost_usd": 0.0, + "atif_valid": true, + "native_journal_projection_matches": true, + "journal_truncated_entries": 0, + "verifier_timed_out": false, + "runtime": { + "python": "3.13.9 (main, Nov 19 2025, 22:47:49) [Clang 21.1.4 ]", + "packages": { + "nanopycodeagent": "0.8.1.dev16+g7a610cbf8.d20260909", + "anthropic": "1.5.0", + "httpx": "0.28.1", + "httpx2": "2.12.0", + "pydantic": "2.13.5" + }, + "sdk_http_module": "httpx2" + }, + "trajectory_source": "native", + "worker_exit_code": 0, + "local_trajectory": "jobs/tb21-remaining18-65536-3600s-trace-20260912-7a610cb/mteb-leaderboard__3RsuVW3/agent/trajectory.json", + "finding": "The required /app/result.txt was absent after 50 rounds. Maximum reply output was 2747 tokens; no generation truncation." + }, + { + "task": "terminal-bench/llm-inference-batching-scheduler", + "task_ref": "sha256:a3bf47589118daec5124fe3689b4439802c805b2f4ed9d402a48ae73ca2fea67", + "trial": "llm-inference-batching-scheduler__GsQKU8C", + "reward": 0.0, + "baseline_reward": 1.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 1.069, + "agent_setup": 155.134, + "agent_execution": 121.43, + "verifier": 17.387 + }, + "terminal_outcome": null, + "terminal_error_type": "WorkerExitedWithoutTerminal", + "exception_type": "NonZeroAgentExitCodeError", + "failure_category": "recorder_stack_dump_crash", + "model_calls_started": 6, + "model_calls_completed": 5, + "tool_calls": 7, + "max_reply_output_tokens": 11697, + "replies_exceeding_32768": 0, + "last_stop_reason": "tool_use", + "usage_complete": false, + "cost_complete": false, + "final_metrics": { + "total_steps": 7, + "extra": { + "usage_complete": false, + "known_cost_usd": 0.0, + "cost_is_partial": true, + "missing_generation_ids": [ + "gen-1789230328-MxcYwnNZB26ftfj6dnwg", + "gen-1789230330-I1ms4VSVAjhnReMgJuI5", + "gen-1789230332-l0dpTOZ919v1w6WP272g", + "gen-1789230398-WypjWjLrpy3b2DqK2jO4", + "gen-1789230419-6dZmVHEFcE9bBFxtKCU1" + ] + } + }, + "billing_known_cost_usd": 0.003683803, + "billing_cost_complete": true, + "billing_total_cost_usd": 0.003683803, + "supplemental_cost_usd": 0.003683803, + "atif_valid": true, + "native_journal_projection_matches": null, + "journal_truncated_entries": 0, + "verifier_timed_out": false, + "runtime": { + "python": "3.13.7 (main, Aug 15 2025, 22:13:55) [GCC 12.2.0]", + "packages": { + "nanopycodeagent": "0.8.1.dev16+g7a610cbf8.d20260909", + "anthropic": "1.5.0", + "httpx": "0.28.1", + "httpx2": "2.12.0", + "pydantic": "2.13.5" + }, + "sdk_http_module": "httpx2" + }, + "trajectory_source": "reconstructed", + "worker_exit_code": -11, + "local_trajectory": "jobs/tb21-remaining18-65536-3600s-trace-20260912-7a610cb/llm-inference-batching-scheduler__GsQKU8C/agent/trajectory.json", + "finding": "Python worker received SIGSEGV in periodic traceback dumping after 120.794 seconds; the output plans were not produced. No generation or agent deadline was reached." + }, + { + "task": "terminal-bench/pytorch-model-recovery", + "task_ref": "sha256:2e628841cff93290919172398e573e794f34c95d9382b7425adde4364022decc", + "trial": "pytorch-model-recovery__svZHN3t", + "reward": 1.0, + "baseline_reward": 1.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 1.028, + "agent_setup": 118.715, + "agent_execution": 76.719, + "verifier": 367.559 + }, + "terminal_outcome": "completed", + "terminal_error_type": null, + "exception_type": null, + "failure_category": "passed", + "model_calls_started": 6, + "model_calls_completed": 6, + "tool_calls": 5, + "max_reply_output_tokens": 6015, + "replies_exceeding_32768": 0, + "last_stop_reason": "end_turn", + "usage_complete": true, + "cost_complete": true, + "final_metrics": { + "total_steps": 7, + "total_prompt_tokens": 51231, + "total_completion_tokens": 9187, + "total_cached_tokens": 37632, + "total_cost_usd": 0.001677349 + }, + "billing_known_cost_usd": 0.001677349, + "billing_cost_complete": true, + "billing_total_cost_usd": 0.001677349, + "supplemental_cost_usd": 0.0, + "atif_valid": true, + "native_journal_projection_matches": true, + "journal_truncated_entries": 0, + "verifier_timed_out": false, + "runtime": { + "python": "3.13.12 (main, Mar 16 2026, 23:05:54) [GCC 12.2.0]", + "packages": { + "nanopycodeagent": "0.8.1.dev16+g7a610cbf8.d20260909", + "anthropic": "1.5.0", + "httpx": "0.28.1", + "httpx2": "2.12.0", + "pydantic": "2.13.5" + }, + "sdk_http_module": "httpx2" + }, + "trajectory_source": "native", + "worker_exit_code": 0, + "local_trajectory": "jobs/tb21-remaining18-65536-3600s-trace-20260912-7a610cb/pytorch-model-recovery__svZHN3t/agent/trajectory.json", + "finding": "Weights remained unchanged; the recovered model exists, loads the weights, matches the state dict, and meets the loss check." + }, + { + "task": "terminal-bench/circuit-fibsqrt", + "task_ref": "sha256:9bcffe1054bb33249aa578a9a2a74f3c8cca66b0cb7aa1328233f1d31822aae3", + "trial": "circuit-fibsqrt__hKj963g", + "reward": 0.0, + "baseline_reward": 0.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 2.187, + "agent_setup": 114.441, + "agent_execution": 124.08, + "verifier": 16.726 + }, + "terminal_outcome": null, + "terminal_error_type": "KeyError", + "exception_type": "NonZeroAgentExitCodeError", + "failure_category": "KeyError", + "model_calls_started": 8, + "model_calls_completed": 8, + "tool_calls": 8, + "max_reply_output_tokens": 10454, + "replies_exceeding_32768": 0, + "last_stop_reason": "tool_use", + "usage_complete": true, + "cost_complete": true, + "final_metrics": { + "total_steps": 9, + "total_prompt_tokens": 108626, + "total_completion_tokens": 16166, + "total_cached_tokens": 88576, + "total_cost_usd": 0.002826103 + }, + "billing_known_cost_usd": 0.002826103, + "billing_cost_complete": true, + "billing_total_cost_usd": 0.002826103, + "supplemental_cost_usd": 0.0, + "atif_valid": true, + "native_journal_projection_matches": true, + "journal_truncated_entries": 0, + "verifier_timed_out": false, + "runtime": { + "python": "3.13.7 (main, Aug 15 2025, 22:13:55) [GCC 12.2.0]", + "packages": { + "nanopycodeagent": "0.8.1.dev16+g7a610cbf8.d20260909", + "anthropic": "1.5.0", + "httpx": "0.28.1", + "httpx2": "2.12.0", + "pydantic": "2.13.5" + }, + "sdk_http_module": "httpx2" + }, + "trajectory_source": "native", + "worker_exit_code": 1, + "local_trajectory": "jobs/tb21-remaining18-65536-3600s-trace-20260912-7a610cb/circuit-fibsqrt__hKj963g/agent/trajectory.json", + "finding": "Gates file exists and meets size constraints but computes incorrect sqrt/fibonacci outputs. After 8 completed replies, a complete tool_use response with an empty bash input caused KeyError(command) in the native ConsoleRenderer, stopping further repair." + }, + { + "task": "terminal-bench/merge-diff-arc-agi-task", + "task_ref": "sha256:6aab6511a5344ce87698293bb1ce4cc51d9a45f1ad9f0c075d2a83197b36727d", + "trial": "merge-diff-arc-agi-task__sgijZ9d", + "reward": 1.0, + "baseline_reward": 1.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 0.925, + "agent_setup": 197.749, + "agent_execution": 94.481, + "verifier": 18.215 + }, + "terminal_outcome": "completed", + "terminal_error_type": null, + "exception_type": null, + "failure_category": "passed", + "model_calls_started": 14, + "model_calls_completed": 14, + "tool_calls": 16, + "max_reply_output_tokens": 5137, + "replies_exceeding_32768": 0, + "last_stop_reason": "end_turn", + "usage_complete": true, + "cost_complete": true, + "final_metrics": { + "total_steps": 15, + "total_prompt_tokens": 151490, + "total_completion_tokens": 11460, + "total_cached_tokens": 124672, + "total_cost_usd": 0.002580521 + }, + "billing_known_cost_usd": 0.002580521, + "billing_cost_complete": true, + "billing_total_cost_usd": 0.002580521, + "supplemental_cost_usd": 0.0, + "atif_valid": true, + "native_journal_projection_matches": true, + "journal_truncated_entries": 0, + "verifier_timed_out": false, + "runtime": { + "python": "3.13.9 (main, Nov 19 2025, 22:47:49) [Clang 21.1.4 ]", + "packages": { + "nanopycodeagent": "0.8.1.dev16+g7a610cbf8.d20260909", + "anthropic": "1.5.0", + "httpx": "0.28.1", + "httpx2": "2.12.0", + "pydantic": "2.13.5" + }, + "sdk_http_module": "httpx2" + }, + "trajectory_source": "native", + "worker_exit_code": 0, + "local_trajectory": "jobs/tb21-remaining18-65536-3600s-trace-20260912-7a610cb/merge-diff-arc-agi-task__sgijZ9d/agent/trajectory.json", + "finding": "Repository initialization, both fetched bundles, algo.py, map function, and the expected map output all passed." + } + ] + }, + { + "kind": "setup_retry", + "job": "tb21-setup-retry-65536-3600s-trace-20260912-7a610cb", + "purpose": "Retry three tasks that never entered initial model execution. Cached uv and wheel bypassed APT; periodic stack dumps were removed. Omitting CA certificates invalidated the DNA retry.", + "local_record": "jobs/tb21-setup-retry-65536-3600s-trace-20260912-7a610cb-record", + "counts": { + "attempts": 3, + "passed": 0, + "scored_zero": 3, + "unscored": 0 + }, + "billing_known_cost_usd": 0.005875884, + "billing_cost_complete": false, + "workflow_sha256": { + "diagnostic_runner.py": "c423d7ba5322b676c73192dbe52d30110f8a91a6eaada5a85151cc456d046635", + "diagnostic_adapter.py": "6b00af9113e2010f0f149509ba94a48c525aa408725c2742d38698d52940f95f", + "monitor.py": "fad87a820be21b8187740d2a5a2f1b8e81550c34a4c8caa1e106335389215610", + "status.py": "c1ae99b78f05efdcb37fff325ab0309d1b3d763741bb6372b674e373b3c78ebe", + "cached_adapter.py": "f76d86eaa4ebc0692f29b8c561b23da5ca25e12a280b6ed4507c6e899423d5cc", + "run.py": "ff0ba0a512ff8bd6d94e4e2c9e8b18d321461426a5937ad66e2283147c292262" + }, + "trials": [ + { + "task": "terminal-bench/dna-assembly", + "task_ref": "sha256:e41a8e94d86019949b08d3b5f88a85f6d943ba0fd85d5e1d5ebb95cb8f66223f", + "trial": "dna-assembly__R4FTx8r", + "reward": 0.0, + "baseline_reward": 0.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 1.038, + "agent_setup": 19.665, + "agent_execution": 5.615, + "verifier": 51.77 + }, + "terminal_outcome": null, + "terminal_error_type": "APIConnectionError", + "exception_type": "UnknownApiError", + "failure_category": "APIConnectionError", + "model_calls_started": 1, + "model_calls_completed": 0, + "tool_calls": 0, + "max_reply_output_tokens": null, + "replies_exceeding_32768": 0, + "last_stop_reason": null, + "usage_complete": false, + "cost_complete": false, + "final_metrics": { + "total_steps": 2, + "extra": { + "usage_complete": false + } + }, + "billing_known_cost_usd": 0.0, + "billing_cost_complete": false, + "billing_total_cost_usd": null, + "supplemental_cost_usd": 0.0, + "atif_valid": true, + "native_journal_projection_matches": true, + "journal_truncated_entries": 0, + "verifier_timed_out": false, + "runtime": { + "python": "3.14.0 (main, Nov 19 2025, 22:48:15) [Clang 21.1.4 ]", + "packages": { + "nanopycodeagent": "0.8.1.dev16+g7a610cbf8.d20260909", + "anthropic": "1.5.0", + "httpx": "0.28.1", + "httpx2": "2.12.0", + "pydantic": "2.13.5" + }, + "sdk_http_module": "httpx2" + }, + "trajectory_source": "native", + "worker_exit_code": 1, + "local_trajectory": "jobs/tb21-setup-retry-65536-3600s-trace-20260912-7a610cb/dna-assembly__R4FTx8r/agent/trajectory.json", + "finding": "Incomplete cached bootstrap omitted system CA certificates. All three connection attempts failed before response headers; no model reply or tool call. The same-image public HTTPS probe failed certificate verification before CA installation and returned 200 afterward." + }, + { + "task": "terminal-bench/qemu-alpine-ssh", + "task_ref": "sha256:60b7050b0e0aa51641208cf65766743d340e59575db2c4d2f8628240846c2a28", + "trial": "qemu-alpine-ssh__fUdBS9i", + "reward": 0.0, + "baseline_reward": 1.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 1.02, + "agent_setup": 19.381, + "agent_execution": 33.772, + "verifier": 5.796 + }, + "terminal_outcome": null, + "terminal_error_type": "KeyError", + "exception_type": "NonZeroAgentExitCodeError", + "failure_category": "KeyError", + "model_calls_started": 3, + "model_calls_completed": 3, + "tool_calls": 4, + "max_reply_output_tokens": 1606, + "replies_exceeding_32768": 0, + "last_stop_reason": "tool_use", + "usage_complete": true, + "cost_complete": true, + "final_metrics": { + "total_steps": 4, + "total_prompt_tokens": 6587, + "total_completion_tokens": 2975, + "total_cached_tokens": 3328, + "total_cost_usd": 0.000486679 + }, + "billing_known_cost_usd": 0.000486679, + "billing_cost_complete": true, + "billing_total_cost_usd": 0.000486679, + "supplemental_cost_usd": 0.0, + "atif_valid": true, + "native_journal_projection_matches": true, + "journal_truncated_entries": 0, + "verifier_timed_out": false, + "runtime": { + "python": "3.14.0 (main, Nov 19 2025, 22:48:15) [Clang 21.1.4 ]", + "packages": { + "nanopycodeagent": "0.8.1.dev16+g7a610cbf8.d20260909", + "anthropic": "1.5.0", + "httpx": "0.28.1", + "httpx2": "2.12.0", + "pydantic": "2.13.5" + }, + "sdk_http_module": "httpx2" + }, + "trajectory_source": "native", + "worker_exit_code": 1, + "local_trajectory": "jobs/tb21-setup-retry-65536-3600s-trace-20260912-7a610cb/qemu-alpine-ssh__fUdBS9i/agent/trajectory.json", + "finding": "After three complete replies, an empty bash input caused KeyError(command). Separately, verifier bootstrap failed fetching libcurl4 with HTTP 404; pytest did not run even though test.sh recorded reward 0." + }, + { + "task": "terminal-bench/model-extraction-relu-logits", + "task_ref": "sha256:1ae5045ad68b5d34c3398b612066a07c4a08b6dc330d28868ec4021e17c94b17", + "trial": "model-extraction-relu-logits__szFsvGz", + "reward": 0.0, + "baseline_reward": 0.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 1.011, + "agent_setup": 9.404, + "agent_execution": 206.431, + "verifier": 39.091 + }, + "terminal_outcome": "completed", + "terminal_error_type": null, + "exception_type": null, + "failure_category": "verifier_failed", + "model_calls_started": 14, + "model_calls_completed": 14, + "tool_calls": 14, + "max_reply_output_tokens": 15040, + "replies_exceeding_32768": 0, + "last_stop_reason": "end_turn", + "usage_complete": true, + "cost_complete": true, + "final_metrics": { + "total_steps": 15, + "total_prompt_tokens": 390191, + "total_completion_tokens": 26752, + "total_cached_tokens": 345344, + "total_cost_usd": 0.005389205 + }, + "billing_known_cost_usd": 0.005389205, + "billing_cost_complete": true, + "billing_total_cost_usd": 0.005389205, + "supplemental_cost_usd": 0.0, + "atif_valid": true, + "native_journal_projection_matches": true, + "journal_truncated_entries": 0, + "verifier_timed_out": false, + "runtime": { + "python": "3.13.7 (main, Aug 15 2025, 22:13:55) [GCC 12.2.0]", + "packages": { + "nanopycodeagent": "0.8.1.dev16+g7a610cbf8.d20260909", + "anthropic": "1.5.0", + "httpx": "0.28.1", + "httpx2": "2.12.0", + "pydantic": "2.13.5" + }, + "sdk_http_module": "httpx2" + }, + "trajectory_source": "native", + "worker_exit_code": 0, + "local_trajectory": "jobs/tb21-setup-retry-65536-3600s-trace-20260912-7a610cb/model-extraction-relu-logits__szFsvGz/agent/trajectory.json", + "finding": "Agent completed normally; the verifier rejected the recovered matrix because row 11 did not match." + } + ] + }, + { + "kind": "scheduler_control", + "job": "tb21-scheduler-recorder-retry-65536-3600s-trace-20260913-7a610cb", + "purpose": "Repeat the scheduler after its recorder stack-dump crash, keeping the original installer and removing only the periodic stack timer.", + "local_record": "jobs/tb21-scheduler-recorder-retry-65536-3600s-trace-20260913-7a610cb-record", + "counts": { + "attempts": 1, + "passed": 0, + "scored_zero": 1, + "unscored": 0 + }, + "billing_known_cost_usd": 0.008719663, + "billing_cost_complete": true, + "workflow_sha256": { + "diagnostic_runner.py": "c423d7ba5322b676c73192dbe52d30110f8a91a6eaada5a85151cc456d046635", + "diagnostic_adapter.py": "6b00af9113e2010f0f149509ba94a48c525aa408725c2742d38698d52940f95f", + "monitor.py": "fad87a820be21b8187740d2a5a2f1b8e81550c34a4c8caa1e106335389215610", + "status.py": "c1ae99b78f05efdcb37fff325ab0309d1b3d763741bb6372b674e373b3c78ebe", + "run.py": "487f002347e3b829348f8724d6b91ca231357597f9c1810b192c33462fbca218" + }, + "trials": [ + { + "task": "terminal-bench/llm-inference-batching-scheduler", + "task_ref": "sha256:a3bf47589118daec5124fe3689b4439802c805b2f4ed9d402a48ae73ca2fea67", + "trial": "llm-inference-batching-scheduler__BzoMMdJ", + "reward": 0.0, + "baseline_reward": 1.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 1.028, + "agent_setup": 37.429, + "agent_execution": 341.522, + "verifier": 17.336 + }, + "terminal_outcome": null, + "terminal_error_type": "KeyError", + "exception_type": "NonZeroAgentExitCodeError", + "failure_category": "KeyError", + "model_calls_started": 15, + "model_calls_completed": 15, + "tool_calls": 17, + "max_reply_output_tokens": 11062, + "replies_exceeding_32768": 0, + "last_stop_reason": "tool_use", + "usage_complete": true, + "cost_complete": true, + "final_metrics": { + "total_steps": 16, + "total_prompt_tokens": 490667, + "total_completion_tokens": 47732, + "total_cached_tokens": 427264, + "total_cost_usd": 0.008719663 + }, + "billing_known_cost_usd": 0.008719663, + "billing_cost_complete": true, + "billing_total_cost_usd": 0.008719663, + "supplemental_cost_usd": 0.0, + "atif_valid": true, + "native_journal_projection_matches": true, + "journal_truncated_entries": 0, + "verifier_timed_out": false, + "runtime": { + "python": "3.13.7 (main, Aug 15 2025, 22:13:55) [GCC 12.2.0]", + "packages": { + "nanopycodeagent": "0.8.1.dev16+g7a610cbf8.d20260909", + "anthropic": "1.5.0", + "httpx": "0.28.1", + "httpx2": "2.12.0", + "pydantic": "2.13.5" + }, + "sdk_http_module": "httpx2" + }, + "trajectory_source": "native", + "worker_exit_code": 1, + "local_trajectory": "jobs/tb21-scheduler-recorder-retry-65536-3600s-trace-20260913-7a610cb/llm-inference-batching-scheduler__BzoMMdJ/agent/trajectory.json", + "finding": "No SIGSEGV after removing periodic stack dumps. After 15 complete replies, empty write arguments raised KeyError; required output plans were absent." + } + ] + }, + { + "kind": "dna_corrected", + "job": "tb21-dna-bootstrap-retry-65536-3600s-trace-20260913-7a610cb", + "purpose": "Restore all original system dependencies, then use the cached uv and wheel without periodic stack dumps. A same-image public HTTPS probe confirmed the preceding missing-CA failure.", + "local_record": "jobs/tb21-dna-bootstrap-retry-65536-3600s-trace-20260913-7a610cb-record", + "counts": { + "attempts": 1, + "passed": 0, + "scored_zero": 1, + "unscored": 0 + }, + "billing_known_cost_usd": 0.004526292, + "billing_cost_complete": true, + "workflow_sha256": { + "diagnostic_runner.py": "c423d7ba5322b676c73192dbe52d30110f8a91a6eaada5a85151cc456d046635", + "diagnostic_adapter.py": "6b00af9113e2010f0f149509ba94a48c525aa408725c2742d38698d52940f95f", + "monitor.py": "fad87a820be21b8187740d2a5a2f1b8e81550c34a4c8caa1e106335389215610", + "status.py": "c1ae99b78f05efdcb37fff325ab0309d1b3d763741bb6372b674e373b3c78ebe", + "cached_adapter.py": "c5f3c4cb6946bd0ed400bce0a9658bb4f949cfc343ea0cf9431255676daf0551", + "run.py": "d919ff5f42adb23ac178cd47f158325fb5ac3eb7f4b5d67602d21073df574f05" + }, + "trials": [ + { + "task": "terminal-bench/dna-assembly", + "task_ref": "sha256:e41a8e94d86019949b08d3b5f88a85f6d943ba0fd85d5e1d5ebb95cb8f66223f", + "trial": "dna-assembly__sP5PGwn", + "reward": 0.0, + "baseline_reward": 0.0, + "same_task_ref": true, + "phase_seconds": { + "environment_setup": 0.927, + "agent_setup": 177.132, + "agent_execution": 239.843, + "verifier": 18.639 + }, + "terminal_outcome": null, + "terminal_error_type": "KeyError", + "exception_type": "NonZeroAgentExitCodeError", + "failure_category": "KeyError", + "model_calls_started": 3, + "model_calls_completed": 3, + "tool_calls": 3, + "max_reply_output_tokens": 35730, + "replies_exceeding_32768": 1, + "last_stop_reason": "tool_use", + "usage_complete": true, + "cost_complete": true, + "final_metrics": { + "total_steps": 4, + "total_prompt_tokens": 12103, + "total_completion_tokens": 35966, + "total_cached_tokens": 5888, + "total_cost_usd": 0.004526292 + }, + "billing_known_cost_usd": 0.004526292, + "billing_cost_complete": true, + "billing_total_cost_usd": 0.004526292, + "supplemental_cost_usd": 0.0, + "atif_valid": true, + "native_journal_projection_matches": true, + "journal_truncated_entries": 0, + "verifier_timed_out": false, + "runtime": { + "python": "3.13.9 (main, Nov 19 2025, 22:47:49) [Clang 21.1.4 ]", + "packages": { + "nanopycodeagent": "0.8.1.dev16+g7a610cbf8.d20260909", + "anthropic": "1.5.0", + "httpx": "0.28.1", + "httpx2": "2.12.0", + "pydantic": "2.13.5" + }, + "sdk_http_module": "httpx2" + }, + "trajectory_source": "native", + "worker_exit_code": 1, + "local_trajectory": "jobs/tb21-dna-bootstrap-retry-65536-3600s-trace-20260913-7a610cb/dna-assembly__sP5PGwn/agent/trajectory.json", + "finding": "After restoring system dependencies, three model replies completed. The largest reply used 35730 output tokens, including 35625 reported thinking tokens, and ended in tool_use. Missing required tool input raised KeyError; primers.fasta was absent." + } + ] + } + ] +} diff --git a/benchmarks/harbor/src/harbor_adapter/adapter.py b/benchmarks/harbor/src/harbor_adapter/adapter.py index b4c9e0e..805810f 100644 --- a/benchmarks/harbor/src/harbor_adapter/adapter.py +++ b/benchmarks/harbor/src/harbor_adapter/adapter.py @@ -37,7 +37,8 @@ class NanoPyCodeAgent(BaseInstalledAgent): cli="--max-turns", type="int", default=_DEFAULT_MAX_TURNS, - ) + ), + CliFlag("max_tokens", cli="--max-tokens", type="int"), ] def __init__(self, *args, git_ref: str | None = None, **kwargs): @@ -54,6 +55,9 @@ def __init__(self, *args, git_ref: str | None = None, **kwargs): raise ValueError("version and git_ref are mutually exclusive") self._git_ref = git_ref super().__init__(*args, **kwargs) + max_tokens = self._resolved_flags.get("max_tokens") + if max_tokens is not None and max_tokens < 1: + raise ValueError("max_tokens must be a positive integer") @staticmethod @override @@ -112,6 +116,9 @@ def _runtime_env(self) -> dict[str, str]: model = self.model_name.split("/", 1)[-1] if model: env["ANTHROPIC_MODEL"] = model + max_tokens = self._get_env("ANTHROPIC_MAX_TOKENS") + if max_tokens is not None: + env["ANTHROPIC_MAX_TOKENS"] = max_tokens return env @override diff --git a/benchmarks/harbor/tests/test_adapter.py b/benchmarks/harbor/tests/test_adapter.py index fafe681..a4c5a4a 100644 --- a/benchmarks/harbor/tests/test_adapter.py +++ b/benchmarks/harbor/tests/test_adapter.py @@ -114,6 +114,7 @@ def test_run_pipes_the_instruction_and_forwards_anthropic_configuration(tmp_path "ANTHROPIC_MODEL": "deepseek/deepseek-v4-flash-0731", }, max_turns=20, + max_tokens=65536, ) environment = RecordingEnvironment() @@ -124,6 +125,7 @@ def test_run_pipes_the_instruction_and_forwards_anthropic_configuration(tmp_path assert instruction not in command assert 'printf "%s" "$harbor_nanopycodeagent_instruction_' in command assert "nanoPyCodeAgent --max-turns 20" in command + assert "--max-tokens 65536" in command assert "--trajectory /logs/agent/trajectory.json" in command assert command.endswith("2>&1 | tee /logs/agent/nanopycodeagent.txt") @@ -160,6 +162,21 @@ def test_run_normalizes_harbor_provider_configuration_for_the_anthropic_sdk( assert run_env["ANTHROPIC_BASE_URL"] == "https://openrouter.example/api" assert run_env["ANTHROPIC_MODEL"] == "deepseek/deepseek-v4-flash-0731" assert "OPENROUTER_API_KEY" not in run_env + assert "--max-tokens" not in environment.calls[-1]["command"] + + +def test_run_forwards_the_environment_budget_without_forcing_a_cli_default(tmp_path): + adapter = make_adapter(tmp_path, extra_env={"ANTHROPIC_MAX_TOKENS": "16384"}) + environment = RecordingEnvironment() + asyncio.run(adapter.run("fix it", environment, SimpleNamespace())) + assert environment.calls[-1]["env"]["ANTHROPIC_MAX_TOKENS"] == "16384" + assert "--max-tokens" not in environment.calls[-1]["command"] + + +@pytest.mark.parametrize("value", [0, -1, True, "bad", 3.5]) +def test_adapter_rejects_invalid_generation_budgets(tmp_path, value): + with pytest.raises(ValueError, match="max_tokens"): + make_adapter(tmp_path, max_tokens=value) def test_adapter_declares_atif_support_and_populates_complete_context(tmp_path): diff --git a/docs/changelogs/0.8.x.md b/docs/changelogs/0.8.x.md index 021513e..51b034b 100644 --- a/docs/changelogs/0.8.x.md +++ b/docs/changelogs/0.8.x.md @@ -4,6 +4,12 @@ All notable changes in the **0.8.x** release series are documented here. ## [Unreleased] +### Added +- Configure per-reply generation budgets through `--max-tokens` or + `ANTHROPIC_MAX_TOKENS`, including the user settings file and Harbor adapter. + Record the effective limit in startup output, Event Journals, and ATIF + trajectories while retaining compatibility with older journals. + ### Fixed - Stop explicitly when a model response reaches `max_tokens`, reporting `response_truncated` in Event Journals and ATIF trajectories instead of task @@ -12,6 +18,7 @@ All notable changes in the **0.8.x** release series are documented here. New journals use schema v2, with v1 replay still supported. ### Changed +- Increase the default per-reply generation limit from 8192 to 32768 tokens. - Moved detailed CLI and configuration guidance out of the bilingual READMEs into dedicated English and Chinese user references, keeping the READMEs concise while documenting task input, options, exit statuses, trajectories, diff --git a/docs/dev_notes/en/0.8.x.md b/docs/dev_notes/en/0.8.x.md index 68f4185..716673a 100644 --- a/docs/dev_notes/en/0.8.x.md +++ b/docs/dev_notes/en/0.8.x.md @@ -438,11 +438,11 @@ This pilot costs about 0.12 USD. Multiplying mechanically by 89/20 gives about 0 ### Fixing problems found in batch Terminal-Bench runs -The following notes explain the problems, implementations, and proposed fixes. The first truncation-handling fix is implemented; the remaining sections describe problems and proposals, not completed code fixes. +The following notes explain the problems, implementations, and proposed fixes. Truncation handling and configurable model generation budgets are implemented. The remaining sections describe problems and proposed work whose code has not yet been implemented. #### Truncation handling -Each model reply currently has an 8192-token generation limit. If the model exhausts this budget before finishing, the API returns `stop_reason="max_tokens"`, indicating a forced stop at the generation limit. The model may still be analyzing a code change before producing a tool call or complete solution. Some pilot replies spent most of their budget on thinking; all 11 affected failed tasks ended with `max_tokens`. +In the original pilot, each model reply had an 8192-token generation limit. If the model exhausts this budget before finishing, the API returns `stop_reason="max_tokens"`, indicating a forced stop at the generation limit. The model may still be analyzing a code change before producing a tool call or complete solution. Some pilot replies spent most of their budget on thinking; all 11 affected failed tasks ended with `max_tokens`. Previously, the agent interpreted “no more tools requested” as the model finishing its work. However, generation-budget exhaustion can also stop a reply before it produces a tool call. As a result, unfinished tasks were recorded as `completed`. Evaluating a run requires distinguishing a normal model finish, exhaustion of the task's turn budget, and truncation of an individual reply: each explains a different stopping reason. Even a normal model finish leaves correctness for the verifier to establish. @@ -528,17 +528,211 @@ This rerun verifies recognition of truncation followed by stopping. It neither i Raw trial logs, trajectories, and verifier output are under `jobs/tb21-truncation-rerun2-20260909-f778861/`. The corresponding `jobs/tb21-truncation-rerun2-20260909-f778861-record/` contains `manifest.json`, `summary.json`, `results.en.md`, the wheel, and run, temporary-installation, and summary scripts under `workflow/`. These directories are Git-ignored and have not been uploaded. This document preserves versions, methods, and results without treating local artifacts as version-controlled files. Future reruns must use new job names and preserve this original 0/2 result. -#### New requirement: increase the model generation budget (pending) +#### Increasing the model generation budget -Added on 2026-09-09. Here, budget primarily means the **token generation limit for one model reply**. The limit is currently fixed at 8192 and cannot be adjusted through environment variables or CLI arguments. This requirement is to raise the default to a value selected through experiments and provide configuration so normal use and Harbor experiments can adjust and record the effective budget. The new default, option names, and configuration precedence remain to be designed. Also evaluate how the budget is divided between thinking and final output. +**Per-reply generation budgets are now configurable, with the default raised from 8192 to 32768 for evaluation.** This budget is the **generated-token limit for one model reply**, not cumulative task usage or the context window. **In fully traced experiments, `regex-log` passed twice at 32768; `write-compressor` truncated twice at 32768, then passed a 09-12 rerun explicitly configured at 65536, producing a 2263-byte file.** However, the successful run's largest reply used only **32073 output tokens**, never consuming output space above 32768, so this observation alone cannot attribute success to doubling the budget. Agent execution took **1976.4 seconds**, with no target file at the 900-second checkpoint, establishing that this execution did require a longer deadline. Earlier installation failures, service interruptions, and timeouts remain recorded below. -**Handling `stop_reason="max_tokens"` must remain even with a higher limit.** Any finite budget can be exhausted. Raising it gives the model more generation room; existing truncation handling records the stopping reason accurately. Both rerun tasks exhaust 8192 tokens, providing a comparison point for larger budgets without guaranteeing they will pass. +**The other 18 tasks passed 5/18 on their initial attempts at 65536, versus 8/18 in the same group's 8192 baseline; five additional attempts across four tasks added no passes.** These experiments did not terminate at `max_tokens` or the 3600-second agent deadline. They also exposed the 50-turn limit, missing or malformed tool arguments, and environment failures. Their failures cannot all be attributed to thinking budgets, nor do these observations establish that increasing the budget improved or reduced the pass rate. -For acceptance, preserve the model, task hashes, and `max_turns=50`, disable automatic continuation, and change only the per-response budget. Verify that the configured value reaches the API request and record the effective value. Rerun these two tasks under a new job, comparing rewards, truncations, input/output tokens, costs, and durations. If truncation occurs again, retain an accurate outcome, usage, and costs without classifying an unfinished reply as task completion. +**Research conclusion: 32768 and 65536 are suitable experimental candidates, but public sources do not establish a universal default to copy.** TB 2.1 evaluations explicitly use both values. Code agents include defaults around 32k/64k, model-dependent limits, and implementations that omit an explicit limit. Evaluation settings and product defaults are recorded separately below so model capacity is not mistaken for an actual request parameter. -#### New requirement: automatically continue after truncation (pending) +**Settings used by other agents/models on Terminal-Bench.** These first-party evaluation descriptions were checked on 2026-09-09, focusing on this project's **TB 2.1**; TB 2.0 is supplementary. Values are per-generation limits disclosed by the publisher, not cumulative task consumption or independently captured production requests. -Added on 2026-09-09. The current run ends on `max_tokens` even if model turns remain. The new behavior is to initiate further model calls automatically when configuration permits and sufficient budget remains, continuing the original task so headless execution does not depend on a user manually sending “continue.” +| Benchmark | Model and agent | Disclosed per-generation limit | Comparison conditions and sources | +| --- | --- | --- | --- | +| **TB 2.1** | GLM-5.3 and GLM-5.3-Flash / Claude Code **2.1.207** | **65536** (`max_new_tokens`) | Z.ai specifies `temperature=1.0`, `top_p=1`, and a six-hour timeout. [GLM-5.3 notes](https://huggingface.co/zai-org/GLM-5.3/blob/main/README.md#footnotes), [Flash notes](https://huggingface.co/zai-org/GLM-5.3-Flash/blob/main/README.md#footnotes) | +| **TB 2.1** | GLM-5.3-Flash-NVFP4 / Terminus-2 | **32768** (`max_tokens`) | RadixArk's quantized-model evaluation uses `max` effort and **disables task deadlines**. Some trials could not run or finish, so its pass rate is not a comparison under this project's conditions. [Evaluation notes](https://huggingface.co/RadixArk/GLM-5.3-Flash-NVFP4#evaluation) | +| TB 2.0 (supplementary) | Qwen3.6-27B / Harbor + Terminus-2 | **80K** (`max_tokens`, preserving the original unit) | Qwen specifies 256K context, a three-hour timeout, 32 CPUs / 48 GB RAM, and a five-run average. [Model-card footnotes](https://huggingface.co/Qwen/Qwen3.6-27B#evaluation) | +| TB 2.0 (supplementary) | Kimi-K2.6 / Tinker Cookbook's simplified agent | **8192** (`max_tokens`) | Thinking Machines uses 32K context, 200 turns, and no context compaction, and reports many context-limit failures. This establishes that 8192 has been used, not that it suits this project. [Configuration and results](https://github.com/thinking-machines-lab/tinker-cookbook/blob/main/tinker_cookbook/recipes/harbor_rl/README.md#evaluation) | + +DeepSeek-V4 is closer to this project's model. Its technical report describes an internal bash + file-edit harness, at most 500 steps, and 512K **context**, but that section does not disclose per-reply `max_tokens` and evaluates TB 2.0. The 512K value therefore cannot be treated as a generation budget or establish the appropriate setting for `deepseek-v4-flash-0731` on TB 2.1. [Report §5.3.1](https://arxiv.org/html/2606.19348v1#S5.SS3.SSS1) + +**Defaults in other code agents.** Defaults require a version, model, and configuration path. Source links below are pinned to the checked commits; official documentation reflects the 2026-09-09 lookup. + +| Agent / inspected scope | Behavior without an explicit override | Evidence and limits | +| --- | --- | --- | +| Claude Code / Opus 4.6 | **64k** | The **v2.1.77 release notes** explicitly raise this model's default to 64k. The same entry's 128k is an available upper limit, not the default, and this finding cannot be generalized to every model. [Release notes](https://github.com/anthropics/claude-code/releases/tag/v2.1.77) | +| Claude Code / unrecognized model IDs, such as custom gateway names | **32000** | Current official documentation specifies this fallback, overridable with `CLAUDE_CODE_MAX_OUTPUT_TOKENS`. Defaults and ceilings for recognized models vary. [Environment variables](https://code.claude.com/docs/en/env-vars#variables) | +| OpenCode / `830d5eb` | **`min(model output limit, 32000)`**; missing or zero model limits fall back to 32000 | The default request-preparation path uses this calculation; environment variables and plugins can override it. [Calculation](https://github.com/anomalyco/opencode/blob/830d5eb5354874105cc31599635a80c1662609e8/packages/opencode/src/provider/transform.ts#L1468), [request construction](https://github.com/anomalyco/opencode/blob/830d5eb5354874105cc31599635a80c1662609e8/packages/opencode/src/session/llm/request.ts#L117) | +| pi / ordinary model-request path at `acaa253` | **Determined by the model's `maxTokens`**, then tightened to remaining context | `buildBaseOptions` uses `options.maxTokens ?? model.maxTokens` and reserves 4096 tokens in its context estimate. There is no common fixed value for all models. Thinking budgets also undergo provider-specific processing. [Source](https://github.com/earendil-works/pi/blob/acaa253cc8e3f159e6100b6f3874861b1f0bfc99/packages/ai/src/api/simple-options.ts#L11) | +| mini-swe-agent / default configuration and LiteLLM path at `04d809c` | **No explicit uniform `max_tokens`** | The default YAML omits it; the model layer forwards `model_kwargs`. The selected configuration, LiteLLM, and server determine the actual value. Omitting the parameter does not mean unlimited generation. [Default configuration](https://github.com/SWE-agent/mini-swe-agent/blob/04d809ceab9df28f9adaed044884180159172930/src/minisweagent/config/default.yaml), [model call](https://github.com/SWE-agent/mini-swe-agent/blob/04d809ceab9df28f9adaed044884180159172930/src/minisweagent/models/litellm_model.py#L64) | +| Codex CLI / official configuration reference | **No uniform default number confirmed in this research** | The inspected configuration reference does not give a default per-generation limit. API model-page output capacity cannot substitute for the CLI's actual default. [Configuration reference](https://learn.chatgpt.com/docs/config-file/config-reference) | + +**Record 32000 and 32768 separately**, rather than reducing both to “32k.” Preserve original units where sources disclose only `64k` or `80K`. Thinking limits and effort levels such as `high`/`max` also need to remain distinct from total generation limits; a separate thinking parameter has not been designed. + +**Implementation and automated acceptance.** Implementation commit: `7a610cbf8`. Configuration uses `--max-tokens`, `ANTHROPIC_MAX_TOKENS`, and the existing `settings.json` `env` object, with precedence **CLI > environment > settings file > 32768**. Both interactive and headless modes resolve the value once at startup and use it for every reply. The effective value must be a positive integer; invalid values fail before any request with exit code `2`. Settings entries retain the existing string rules, for example `"ANTHROPIC_MAX_TOKENS": "32768"`. + +Harbor forwards `--agent-kwarg max_tokens=N` and `ANTHROPIC_MAX_TOKENS`. Omitting the kwarg does not insert a CLI default, allowing environment configuration and the installed version's default to apply. Startup output, Journal `run.started.max_tokens`, and ATIF `agent.extra.max_tokens` record the effective limit separately from actual usage. Older journals without the field still convert; ATIF does not invent a historical budget. + +```bash +nanoPyCodeAgent --max-tokens 65536 -p "fix the failing tests" +# Harbor: add --agent-kwarg max_tokens=65536 to the run command +``` + +All **242 core tests** and **22 Harbor adapter tests** passed. Coverage includes precedence, budgets on every interactive/headless request, early rejection of invalid values, separation of recorded limits from usage, and real SDK request serialization. **Real-model results at 65536 appear in the 09-12 experiment below; successful parameter forwarding does not establish effectiveness attribution.** + +**Handling `stop_reason="max_tokens"` must remain even with a higher limit.** Any finite budget can be exhausted. Raising it gives the model more generation room; existing truncation handling records the stopping reason accurately. Both preceding cases exhausted 8192, providing a comparison point without guaranteeing success at a higher limit. + +Truncation regressions also pass at higher budgets: `max_tokens` still produces `response_truncated`, preserves usage and costs, and skips tools from that reply. This change adds neither a separate thinking parameter nor automatic continuation. + +**Real-model experiment at 32768.** New job `tb21-budget32768-20260909-7a610cb` compares against the 8192 results in `tb21-truncation-rerun2-20260909-f778861`. It retains the same TB 2.1 dataset, task hashes, OpenRouter `deepseek/deepseek-v4-flash-0731`, `max_turns=50`, native 900-second task timeouts, concurrency 2, one attempt per task, and no retries. A local wheel was built from the implementation commit, with each packaged Python source verified against that commit. The launcher removes `ANTHROPIC_MAX_TOKENS` and omits `--max-tokens` to test the built-in default. + +The first `regex-log` trial exceeded the default 360-second installation timeout before model execution and received no verifier score; that record remains in the original job. New job `tb21-budget32768-regex-setup-retry-20260909-7a610cb` reruns only that task with an installation timeout of 1080 seconds (`--agent-setup-timeout-multiplier 3`), retaining its 900-second task deadline. + +The initial `write-compressor` trial entered model execution and completed tool calls in its first reply. Its second call received the server error `stream closed before completion` after about 421 seconds of agent execution. Harbor records `UnknownApiError`, and the verifier reward is 0. The response did not return `max_tokens`, so this does not establish budget exhaustion. The completed first reply used 1528 input / 103 output tokens and cost $0.00011786; complete usage and cost for the interrupted call are unknown. That known subtotal must not be reported as the task's complete cost. + +To check whether the service failure was transient, one additional attempt uses the same model, budget, and task deadline under `tb21-budget32768-compressor-api-retry-20260909-7a610cb`, also with a 1080-second installation window. Both model runs are retained rather than replacing the initial failure. + +**Final results (executed 2026-09-09, verified and archived 09-10).** Times below are Harbor's agent execution durations, excluding installation and verification. The initial installation failure has no model execution time or score. + +| Task | 8192 baseline | Initial 32768 trial | 32768 retry | +| --- | --- | --- | --- | +| `regex-log` | Reward **0**; `response_truncated`; 187.3 sec | `AgentSetupTimeoutError`; no model execution or score | Reward **0**; `AgentTimeoutError`; 900.9 sec | +| `write-compressor` | Reward **0**; `response_truncated`; 304.4 sec | Reward **0**; `UnknownApiError`; 421.4 sec | Reward **0**; `AgentTimeoutError`; 900.9 sec | + +Both retry startup logs confirm `max tokens 32768`; agent versions and task hashes match the experimental configuration. Verifiers report missing `/app/regex.txt` and `/app/data.comp`, respectively. Including the initial server interruption, **three runs entered model execution and received scores, all 0**; one additional trial failed during installation. No retry result replaces an initial failure. + +**A complete usage and cost comparison was not possible.** Neither timed-out retry exported final ATIF. Harbor records trajectory status `missing` and `null` input/output tokens and costs. Model-call counts, final `stop_reason`, and actual truncation counts also cannot be fully reconstructed from the available records; missing values must not become zeroes. Complete baseline costs at 8192 were $0.00158363 and $0.001742255. At 32768, only the initial `write-compressor` trial's completed first reply has the known subtotal of $0.00011786, so no reliable total cost increase can be calculated. + +**Keep 32768 as the experimental default, without describing the 09-09 experiment as successful acceptance of the higher budget.** Configuration, request forwarding, and truncation regressions are verified. Real-model evaluation remains affected by a server interruption, task timeouts, and missing trajectories after timeout. Without complete final replies, the experiment cannot establish that a higher budget eliminates `max_tokens` truncation or imply a gain across the full benchmark. + +All three jobs used the same wheel, SHA-256 `91996b1d4c70af8062318a38b3af9235c85d93c9685e10ac1323f3bef3625425`. Raw logs, verifier output, and exceptions remain in the corresponding jobs; each `-record/` directory contains its manifest, wheel, and launcher. The consolidated summary is under `jobs/tb21-budget32768-20260909-7a610cb-record/` in `summary.json`, `results.en.md`, and `workflow/summarize.py`. These `jobs/` artifacts are Git-ignored and have not been uploaded. The associated containers have been cleaned up. + +**Extended deadline with a complete trajectory (2026-09-10).** Trial `regex-log__2A9PJgL` in new job `tb21-regex-3600s-trace-20260910-7a610cb` received **reward 1.0**, passing the official verifier's `test_regex_matches_dates`. Harbor records **580.4 seconds** of agent execution, excluding installation and verification. This success did not consume time beyond the original 900-second deadline, so it does not establish that extending the deadline resolved the earlier timeout. + +The run retained the same wheel, task hash, model, built-in `max_tokens=32768`, and 50-turn limit, with one task, one attempt, and no retries. Agent execution was allowed **3600 seconds**, followed by 120 seconds for interruption and export. Harbor's outer allowance was 3780 seconds (`--agent-timeout-multiplier 4.2`) to avoid preempting cleanup. Installation remained at 1080 seconds; the verifier deadline was unchanged. A temporary diagnostic wrapper saved an unshortened Journal, HTTP response bodies, per-chunk arrival times, and stack snapshots every 120 seconds directly to persistent logs, retaining the comparison's agent source, prompt, and tools. This run finished naturally without interruption or trajectory reconstruction. + +| Observation | Result | +| --- | --- | +| Model replies / tool calls | **5 / 6**; the first four replies ended with `tool_use`, the last with `end_turn`; no `max_tokens` truncation | +| Total model duration | **567.3 seconds**, approximately **97.9%** of native run time | +| First / subsequent reply durations | **502.4 seconds** / 35.1, 2.8, 4.2, 22.8 seconds | +| Total local tool duration | **0.023 seconds** | +| Complete usage | **74650** prompt tokens, including **53504** cached tokens; **17264** completion tokens | +| Complete cost | **$0.005338074**, with costs reconciled for all five calls | + +**The evidence points to sustained first-reply thinking generation as this run's main delay, rather than local tool execution.** Thinking began about 2.5 seconds after the first request and continued through 500.8 seconds; tool arguments began at 501.7 seconds. The longest interval between nonempty generation deltas was only **3.4 seconds**, with no long stream gap observed. That reply retained 47007 thinking characters. The provider reported **15123 output tokens**, with **12026** in `output_tokens_details.thinking_tokens`. Character counts and provider token accounting differ; subtracting those token fields does not independently measure visible-text usage. + +Integrity checks passed: **147 Journal entries, none truncated**; native ATIF passed Harbor schema validation and exactly matched reconstruction from the Journal. All five streams contained `message_stop`, and all five requests actually sent 32768. Stream analysis uses monotonic clocks to avoid small system-clock adjustments. The diagnostic wrapper also passed three offline scenarios: normal completion, interruption after a simulated stall, and recovery after forced termination. + +**The exact cause of the historical timeout remains undetermined.** Without that run's complete reply and stream timeline, prolonged generation, provider/network stalls, and a different solution path cannot be distinguished. The new run recorded Anthropic SDK **1.4.0**, httpx **0.28.1**, and httpx2 **2.12.0** in its container; corresponding versions were not retained for the earlier container. Equal wheel bytes therefore do not establish identical runtime conditions. This result demonstrates that this task can finish at 32768, but one success cannot establish a causal benefit from extending the deadline or generalize to the other task or full benchmark. + +Complete ATIF is in `regex-log__2A9PJgL/agent/trajectory.json` under the new job. Adjacent `journals/`, `http/`, `http-events.jsonl`, and `python-stacks.log` retain the raw diagnostic evidence. The corresponding `-record/` directory contains `results.en.md`, `summary.json`, `validation.json`, and `workflow/` for the report, integrity checks, and reproduction scripts. Artifacts remain local under Git-ignored `jobs/` and have not been uploaded. The container has been cleaned up, and earlier failure records remain intact. + +**Repeating the method for `write-compressor` (2026-09-11).** Trial `write-compressor__GTmsjFV` in new job `tb21-compressor-3600s-trace-20260911-7a610cb` received **reward 0**. All three official verifier tests failed because `/app/data.comp` was absent. Harbor records **1449.1 seconds** of agent execution (about 24 minutes), no infrastructure exception, and complete native trajectory export. The final outcome is **`response_truncated`**; the 3600-second deadline was not reached. + +The run retained the preceding diagnostic method: the same model and wheel, built-in `max_tokens=32768`, a 50-turn limit, one task, one attempt, and no retries. Execution, finalization, and Harbor's outer allowances were 3600, 120, and 3780 seconds respectively; installation remained at 1080 seconds. The diagnostic wrapper was byte-identical and continued saving an unshortened Journal, raw response streams, arrival times, and stack snapshots. The task hash matched the preceding `write-compressor` baseline. + +| Model reply | Duration | Output tokens | Provider-reported thinking tokens | Stop reason | +| --- | --- | --- | --- | --- | +| 1 | **5.8 seconds** | 118 | 16 | `tool_use` | +| 2 | **1414.2 seconds** | **32768** | **32767** | **`max_tokens`** | + +**This run directly showed sustained thinking outlasting the original deadline, exhausting the generation budget, and leaving no delivered file.** Reply 1 ran `read` and `bash` in just **0.004 seconds** combined. Reply 2 began returning thinking after about 0.9 seconds and continued through 1414.1 seconds. Its longest nonempty generation-delta gap was **2.2 seconds**, with no long silent interval observed. The complete reply retained **116281 thinking characters**, with no visible text or tool arguments. The current agent ends the run on `max_tokens` without automatic continuation, so no subsequent call could produce the file. + +At **900.4 seconds**, an independent read-only checkpoint near the original deadline confirmed that `data.comp` was absent. Only the first reply and two tool calls had completed; the second reply was still streaming thinking. This establishes that the observed execution could not deliver within the original window. The longer allowance let it continue until the generation budget was exhausted: **extending the deadline alone did not solve this failure**. This timeline explains the present execution, not the missing historical trajectory, and the earlier server interruption must remain distinct from this truncation. + +Complete usage was **3926 prompt tokens**, **0 cached tokens**, and **32886 completion tokens**, costing **$0.00615504**. Cumulative output exceeds 32768 because the budget applies per reply. Costs were reconciled for both calls; the second receipt returned HTTP 404 four times before succeeding on the fifth attempt, after the reply had already been truncated. Harbor installation and verification lasted 143.9 and 226.2 seconds respectively, outside the reported agent execution time. + +**Complete evidence and a truncated model reply are distinct.** All **13 Journal entries are unshortened**, including thinking beyond the default 100000-character logging threshold. Native ATIF passed Harbor schema validation and matched projection from the Journal. Both response streams were fully consumed, contained `message_stop`, and used an actual request budget of 32768. The container installed Anthropic SDK **1.5.0**, versus 1.4.0 in the preceding regex diagnostic. The wheel's dependency range permits newer versions, so using the same method does not fix every dependency or backend condition. + +Complete ATIF is in `write-compressor__GTmsjFV/agent/trajectory.json` under the new job, alongside the Journal, raw HTTP responses and timeline, stack snapshots, and runtime metadata. The corresponding `-record/` contains `results.en.md`, `summary.json`, `validation.json`, `checkpoints/`, and reproduction scripts, including the file check at the original 900-second deadline. Artifacts remain local, Git-ignored, and not uploaded. The container has been removed; all earlier failures and the successful regex record remain intact. + +**One further attempt per task (2026-09-11).** This pair ran `regex-log` and `write-compressor` sequentially, once each, without automatic retry. The second task started only after the first task's verification and container cleanup. Model, wheel, task hashes, the built-in 32768 budget, 50-turn limit, and 3600/120/3780-second allowances remained unchanged from the preceding diagnostics. + +| Task | Preceding complete diagnostic | New repeat | +| --- | --- | --- | +| `regex-log` | Reward **1**; 580.4 seconds; 5 replies | Reward **1**; **631.2 seconds**; 9 replies, ending with `end_turn` | +| `write-compressor` | Reward **0**; 1449.1 seconds; `max_tokens` in reply 2 | Reward **0**; **582.1 seconds**; `max_tokens` in reply 3 | + +Times are Harbor agent execution durations, excluding installation and verification. **Neither new run consumed agent time beyond the original 900-second window.** There was no deadline event, forced termination, or trajectory reconstruction. + +Regex's first reply still took about **517.4 seconds**, using 16445 output tokens, including 16346 provider-reported thinking tokens. It subsequently completed eight tool calls in 0.039 seconds combined. None of its nine replies was truncated, and the official `test_regex_matches_dates` passed again. + +Compressor's second reply completed a tool call after **172.5 seconds and 5463 output tokens**, allowing it to reach reply 3. That reply then took **394.8 seconds** and exhausted the **32768** total generation budget, with **28961** provider-reported thinking tokens. Its captured stream contains **112785 thinking characters** and no visible text or tool arguments; subtracting token fields cannot establish a visible-text count. The longest nonempty generation-delta gap was 1.5 seconds. All three tools took about 0.004 seconds combined. `/app/data.comp` was still absent, so all three official tests failed, with native outcome `response_truncated`. + +| Task | Complete prompt tokens (including cache) | Cached tokens | Completion tokens | Complete cost | +| --- | --- | --- | --- | --- | +| `regex-log` | 161870 | 137216 | 20400 | **$0.00720948** | +| `write-compressor` | 12704 | 1280 | 38335 | **$0.00764908** | + +The two tasks' **256 and 18 Journal entries are all unshortened**. Both native ATIF files passed Harbor schema validation and exactly matched projection from their Journals. All 12 model response streams were fully consumed and contained `message_stop`; every actual request sent 32768, and all generation costs were reconciled. Installation took 833.6/448.1 seconds and verification 114.9/264.6 seconds respectively, recorded separately from agent execution. + +**Both regex diagnostics in this comparison passed, and both compressor diagnostics exhausted the generation budget, but durations and stopping turns differed substantially.** The new compressor failure occurred before 900 seconds, confirming the per-reply cap as this execution's stopping condition. Two observations do not establish that future runs must pass or fail, or that a higher budget would succeed. Both new containers recorded Anthropic SDK 1.5.0 and matching versions of the other recorded packages; the earlier regex run used 1.4.0. Sampling, cache state, and backend conditions were not fixed, so timing differences cannot be attributed directly to one factor. + +The new jobs are `tb21-regex-3600s-trace-repeat2-20260911-7a610cb` and `tb21-compressor-3600s-trace-repeat2-20260911-7a610cb`, with trials `regex-log__NmpJwcQ` and `write-compressor__NqmJQWm`. Each trial's `agent/trajectory.json` and adjacent Journal/HTTP/stack records retain complete evidence. Each `-record/` contains its report and reproduction scripts. The combined comparison is in `results.en.md` and `comparison.json` under `jobs/tb21-3600s-trace-repeat2-20260911-7a610cb-record/`. Raw artifacts remain local, Git-ignored, and not uploaded. Both containers were cleaned up, and all earlier records remain intact. + +**Compressor rerun at 65536 (2026-09-12).** This attempt explicitly passed `--agent-kwarg max_tokens=65536`, retaining the preceding compressor model, wheel, task hash, 50-turn limit, and 3600/120/3780-second allowances. One task ran once without automatic retry; the project default remains 32768. It finished with **reward 1 and all three official tests passing**: `data.comp` is **2263 bytes** and decompresses exactly to the original file. + +| Measurement | Result | +| --- | --- | +| Agent execution | **1976.4 seconds**; installation 278.2 seconds and verifier 169.1 seconds are separate | +| Model replies / tool calls | **32 / 32**; final `end_turn`, outcome `completed` | +| Largest reply | **32073 output tokens**; all actual requests sent 65536, without truncation | +| Complete prompt / cache / completion tokens | **1528746 / 1471744 / 59563**; prompt includes cache | +| Complete cost | **$0.018819072**, with all 32 generation costs reconciled | + +Reply 2 remained the main delay: **1179.7 seconds**, 32073 output tokens including 27900 provider-reported thinking tokens, then a normal `tool_use`. Its complete stream contains 108811 thinking characters and a longest nonempty generation-delta gap of 2.2 seconds. All tools took 27.3 seconds combined. At 900.4 seconds the target file was absent; it measured 2727 bytes at 1299.7 seconds and 2388 bytes at 1449.9 seconds, before the final reduction to 2263. + +**This establishes a successful rerun configured at 65536, but does not establish that 32768 must be insufficient.** Every reply remained below 32768, and sampling, cache state, and backend conditions were not fixed, preventing direct attribution to the doubled cap. The recorded execution did use time beyond the original 900-second deadline. SDK 1.5.0 and the other recorded dependency versions match the preceding compressor diagnostic. There was no deadline event, forced kill, or trajectory reconstruction. + +All 498 Journal entries are unshortened, and native ATIF passes schema validation and exactly matches Journal projection. All 32 model response streams were fully consumed and contain `message_stop`. One cost-receipt GET encountered `ConnectError` before headers, then succeeded on retry without interrupting a model reply. The original analyzer's broad check that every HTTP request received a complete response is consequently false; `validation-notes.json` preserves and explains that distinction. + +Job: `tb21-compressor-65536-3600s-trace-20260912-7a610cb`; trial: `write-compressor__hzfqDFs`. Complete `agent/trajectory.json`, Journal, HTTP streams, and timelines are retained. The corresponding `-record/` holds reports, validation, checkpoints, and reproduction scripts; `task-output/` also preserves the final compressed file, input, decoder source, and hashes. Artifacts remain local, Git-ignored, and not uploaded. The container was cleaned up, and all earlier results remain intact. + +**Extending 65536 to the other 18 tasks (2026-09-12–13).** The initial batch took approximately 2 hours 57 minutes and produced **5 passes, 8 reward-zero results, and 5 unscored results**: 5/18 (27.8%), versus 8/18 for the same tasks in the 8192 baseline. The passing tasks were `pypi-server`, `openssl-selfsigned-cert`, `log-summary-date-ranges`, `pytorch-model-recovery`, and `merge-diff-arc-agi-task`. Five subsequent attempts across four tasks added no passes. Initial and supplemental attempts are reported separately, preserving every failure. + +The experiments retained the same verified wheel from `7a610cb`, DeepSeek V4 Flash 0731 through OpenRouter, and task hashes; cached image digests for all 18 tasks also matched the baseline. Explicit settings were `max_tokens=65536`, 50 turns, concurrency 2, 3600 seconds for agent execution, 120 seconds for finalization, a 3780-second Harbor outer allowance, and 1080 seconds for installation. Verifiers retained native task deadlines. Each task ran once per job, without automatic trial retries. **The project default remains 32768; 65536 is the experimental override.** + +In the table, `—` means no official score was obtained and is not counted as a pass. Reward zero can also result from infrastructure or recorder faults, so its cause must be considered. + +| Task | 8192 baseline | Initial 65536 | Main finding | +| --- | ---: | ---: | --- | +| `torch-tensor-parallelism` | 0 | — | First response stream interrupted; verifier timed out downloading Torch/CUDA dependencies | +| `schemelike-metacircular-eval` | 0 | 0 | Reply 19 interrupted; interpreter passed only 4/63 subcases | +| `kv-store-grpc` | 1 | 0 | `SetValRequest` lacks the required `value` field; 5/7 tests passed, including service availability | +| `pypi-server` | 1 | 1 | Local package installation and API test passed | +| `dna-assembly` | 0 | — | TLS download failure installing uv; no model call | +| `torch-pipeline-parallelism` | 0 | — | Exhausted 50 turns; verifier dependency installation timed out without functional test results | +| `qemu-alpine-ssh` | 1 | — | APT package returned 404 during setup; no model call | +| `openssl-selfsigned-cert` | 1 | 1 | All 6 tests passed | +| `regex-chess` | 0 | 0 | Exhausted 50 turns; invalid successor at the starting position; only the length check passed (1/4) | +| `log-summary-date-ranges` | 1 | 1 | Both tests passed | +| `model-extraction-relu-logits` | 0 | — | TLS failure fetching the uv installer; no model call | +| `path-tracing` | 0 | 0 | Exhausted 50 turns; `image.c` absent; 0/5 tests passed | +| `caffe-cifar-10` | 0 | 0 | Exhausted 50 turns; final build lacked `g++`, with binary and model absent; only the prototxt check passed (1/6) | +| `mteb-leaderboard` | 0 | 0 | Exhausted 50 turns; `result.txt` absent; 0/2 tests passed | +| `llm-inference-batching-scheduler` | 1 | 0 | Recorder received SIGSEGV during a periodic stack dump at approximately 120 seconds; output plans absent | +| `pytorch-model-recovery` | 1 | 1 | All 5 tests passed | +| `circuit-fibsqrt` | 0 | 0 | Incorrect circuit output; empty `bash` arguments triggered native `KeyError('command')`, stopping repair; 2/3 tests passed | +| `merge-diff-arc-agi-task` | 1 | 1 | All 5 tests passed | + +**Supplemental attempts addressed confirmed environment or recorder faults and do not overwrite the initial results.** The three setup failures first retried with cached uv and wheel files to bypass download/APT failures; the scheduler separately reran without periodic stack dumping. The cached installer omitted system CA certificates needed by the DNA image, so DNA received one further attempt with corrected installation. + +| Task | Additional attempts | Supplemental result | +| --- | ---: | --- | +| `dna-assembly` | 2 | First retry failed before response headers because of the omitted CA certificates and cannot evaluate the model. Restoring all original system dependencies allowed 3 replies, but malformed `bash` arguments triggered `KeyError('command')`; `primers.fasta` was absent and 0/1 tests passed | +| `qemu-alpine-ssh` | 1 | After 3 replies, malformed `bash` arguments triggered `KeyError('command')`. Verifier setup independently encountered another APT 404; pytest did not run, so its recorded reward zero is not a functional test failure | +| `model-extraction-relu-logits` | 1 | Completed 14 replies normally, but row 11 of the recovered matrix did not match and the official test failed | +| `llm-inference-batching-scheduler` | 1 | No SIGSEGV; after 15 replies, malformed `write` arguments triggered `KeyError('content')`; plans were absent, with only the input integrity check passing (1/6) | + +Recorder and installation differences are part of the evidence. Before any initial batch trial started, supervisor cleanup was corrected to preserve background task services on normal worker exit and clean the process group only at the deadline. Offline service tests passed, as did the `kv-store-grpc` service availability test. System core metadata, ELF files from the same image, and disassembly located the scheduler crash at `PyCode_Addr2Line+11` during periodic stack dumping. This path matches the [CPython timer implementation](https://github.com/python/cpython/blob/v3.13.7/Modules/faulthandler.c#L537) and [frame dumper](https://github.com/python/cpython/blob/v3.13.7/Python/traceback.c#L842), but the underlying memory-fault mechanism remains unproven. Its control retained the original installer and removed only periodic dumping; the crash did not recur. For DNA, a public HTTPS request from the same image, without credentials, failed certificate verification before installing CA certificates and returned HTTP 200 afterward. The corrected trial restored `curl`, `bash`, `git`, `python3`, and CA certificates before installing cached uv and wheel files. Neither diagnostic fault is evidence of insufficient generation budget. + +**Direct budget conclusion: increasing the cap further cannot serve as a universal fix for these failures.** + +- No run in this 65536 experiment reported `max_tokens` or reached the 3600-second agent deadline. Only Scheme (38496), regex-chess (60483), and corrected DNA (35730) actually exceeded 32768 output tokens in a reply, and all three failed. The passing compressor peaked at 32073; the other passing tasks peaked at no more than 6015. +- Pipeline, regex-chess, path-tracing, Caffe, and MTEB exhausted **50 turns** in the initial batch. This is the total call-count limit, distinct from the per-reply generation budget. +- Circuit, the QEMU retry, the scheduler control, and corrected DNA exposed another agent robustness issue: response streams ended completely with `tool_use` but supplied missing or malformed tool arguments, leaving the SDK with an empty dictionary. The native renderer indexed required fields and raised `KeyError`. Circuit's raw JSON was `{}`, QEMU/scheduler supplied `{`, and DNA supplied an unterminated string; these are not all valid empty JSON objects. Argument validation and error feedback should be addressed separately so the agent can attempt repair; that implementation was not changed in this experiment. + +These are diagnostic observations, **not a controlled comparison changing only the budget**. The 8192 baseline used source `2b83097` and native task deadlines; dependencies, sampling, cache state, and backend conditions were not fixed. The change from 8/18 to 5/18 therefore does not establish that a larger budget is harmful, and the compressor pass cannot be attributed to doubling the budget. The two preceding `regex-log` passes used 32768; it was not rerun at 65536, so these different conditions are not combined into a uniform 20-task pass rate. + +The reconciled billing lower bound for the other 18 tasks and their 5 supplemental attempts is **$0.261769566**; including compressor, all 24 attempts cost at least **$0.280588638**. Tensor-parallelism billing queries returned 404 on three checks, leaving complete total cost and token usage unknown. Interrupted Scheme/scheduler requests were priced separately, but those receipts do not repair missing native usage, and unknown values are not treated as zero. + +The [structured experiment results](../../../benchmarks/harbor/results/tb21-generation-budget-65536-20260912.json) contain configuration, source/wheel hashes, per-task results, accounting limits, and local job/trajectory paths. The 24 attempts cover 19 distinct tasks: **20 native ATIF files, 1 clearly marked post-crash reconstruction, and 3 setup failures without trajectories**. Every present ATIF validates against its schema; every native trajectory matches its full Journal projection, with no truncated Journal strings. Received HTTP streams, including interrupted streams, timelines, runtime versions, validation reports, and diagnostic scripts remain under `jobs/`. The main record directory is `tb21-remaining18-65536-3600s-trace-20260912-7a610cb-record`: `comparison.json` indexes every group, and `results.en.md` records the initial 18 tasks. Raw trajectories remain Git-ignored and unuploaded; all experiment and diagnostic containers have been cleaned up. + +#### Automatically continuing after truncation + +The current run ends on `max_tokens` even if model turns remain. The new behavior is to initiate further model calls automatically when configuration permits and sufficient budget remains, continuing the original task so headless execution does not depend on a user manually sending “continue.” Expected behavior and constraints: @@ -547,7 +741,7 @@ Expected behavior and constraints: - Preserve every `max_tokens` fact in the Journal/ATIF and record subsequent attempts, all usage, and all costs. Do not emit a run terminal event while continuation is still underway. Accurately record normal completion, continuation-limit exhaustion, turn exhaustion, or request failure. Terminal rules and protocol representation for recovery need to be designed together. - Stop at configured limits, cancellation, or unrecoverable errors with explicit diagnostics, avoiding endless continuation. Continued execution still relies on the verifier to establish correctness. -Automated acceptance must cover continuation to completion, repeated truncation reaching its limit, exhausted remaining turns, thinking-only replies, and truncated tool arguments neither executing nor being replayed incorrectly. For real-model acceptance, use these two tasks and new job names to compare continuation disabled/enabled at the same response budget, then evaluate larger budgets together with continuation. Report pass rate, additional model calls, total tokens, cost, and duration separately. Both requirements remain unimplemented; this 0/2 experiment is not their acceptance result. +Automated acceptance must cover continuation to completion, repeated truncation reaching its limit, exhausted remaining turns, thinking-only replies, and truncated tool arguments neither executing nor being replayed incorrectly. For real-model acceptance, use these two tasks and new job names to compare continuation disabled/enabled at the same response budget, then evaluate larger budgets together with continuation. Report pass rate, additional model calls, total tokens, cost, and duration separately. Automatic continuation remains unimplemented; neither the preceding 0/2 result at 8192 nor the higher-budget experiment constitutes acceptance for automatic continuation. #### Crashes caused by missing tool arguments diff --git a/docs/dev_notes/zh-CN/0.8.x.md b/docs/dev_notes/zh-CN/0.8.x.md index 853b2a9..15fc5a7 100644 --- a/docs/dev_notes/zh-CN/0.8.x.md +++ b/docs/dev_notes/zh-CN/0.8.x.md @@ -508,12 +508,12 @@ API 凭据，原件权限已设为 0600，归档副本已脱敏；原件与脱 ### 修复批量运行 Terminal-Bench 时遇到的问题 -以下记录本轮问题的含义、实现与修复方向。“截断处理”已完成第一步修复；其余小节 -仍描述问题与拟议方向，不表示对应代码已修复。 +以下记录本轮问题的含义、实现与修复方向。“截断处理”和“增加模型生成预算”已实现； +其余小节仍描述问题与拟议方向，不表示对应代码已修复。 #### 截断处理 -当前每次模型回复最多生成 8192 tokens。模型尚未输出完就用尽这次预算时，接口 +原始试跑中，每次模型回复最多生成 8192 tokens。模型尚未输出完就用尽这次预算时，接口 返回 `stop_reason="max_tokens"`，表示达到生成长度上限，被迫停止。例如，模型 可能仍在分析如何修改代码，后面的工具调用或完整解答还没有生成，预算就耗尽了。 本轮部分回复的预算主要消耗在 thinking；11 道未通过题的末次回复均为 `max_tokens`。 @@ -642,25 +642,408 @@ validator，启动版本包含 `f778861`，日志都有明确的截断提示， 这些目录被 Git 忽略，未上传；本文保存版本、方法与结果汇总，不将本地产物视为 已经进入版本管理的文件。以后重跑须使用新的 job 名，保留这次 0/2 的原始结果。 -#### 新需求：增加模型生成预算（待实现） +#### 增加模型生成预算 + +**已实现单次生成预算配置，并将默认值从 8192 提高到 32768 进行试用。** +这里的预算指**单次模型回复的生成 token 上限**，不是整题累计 token 或上下文窗口。 +**完整轨迹实验中，`regex-log` 在 32768 下两次通过；`write-compressor` 在 +32768 下两次截断，09-12 显式改用 65536 重跑后通过，最终文件 2263 字节。** +但这次成功运行的最大单轮输出只有 **32073**，未实际用到 32768 以上的输出空间， +不能单凭它证明预算翻倍导致通过。解题耗时 **1976.4 秒**，900 秒检查点仍无目标 +文件，因此这次执行确实需要延长时限。此前的安装失败、服务中断及超时均在下文保留。 + +**其余 18 题在 65536 下首轮通过 5/18（同组 8192 基线为 8/18）；针对 4 题追加 +5 次补跑，没有新增通过。** 这批实验未出现 `max_tokens` 或 3600 秒解题时限终止， +还暴露了 50 轮限制、缺失或不完整工具参数、环境故障；不能把失败统一归因于 thinking +预算，也不能据此认定提高预算改善或降低了通过率。 + +**调研结论：32768、65536 适合作为下一轮实验的候选档位，但公开资料没有给出一个 +可以直接照搬的统一默认值。** TB 2.1 已有明确采用这两个值的评测；code agent 则 +既有约 32k／64k 的默认设置，也有按模型决定或不显式设置的实现。下面分别记录 +评测参数和产品默认行为，避免把模型能力上限当成实际请求参数。 + +**其他 agent／模型跑 Terminal-Bench 时设置多少。** 以下为 2026-09-09 查到的 +第一方评测说明，主看本项目使用的 **TB 2.1**；TB 2.0 仅作补充。数字是发布方 +披露的单次生成上限，不是整道题累计消耗，也不是对其线上请求的独立抓包验证。 + +| Benchmark | 模型与 agent | 披露的单次生成上限 | 影响比较的条件与来源 | +| --- | --- | --- | --- | +| **TB 2.1** | GLM-5.3、GLM-5.3-Flash／Claude Code **2.1.207** | **65536**（`max_new_tokens`） | Z.ai 披露 `temperature=1.0`、`top_p=1`、6 小时 timeout。[GLM-5.3 说明](https://huggingface.co/zai-org/GLM-5.3/blob/main/README.md#footnotes)、[Flash 说明](https://huggingface.co/zai-org/GLM-5.3-Flash/blob/main/README.md#footnotes) | +| **TB 2.1** | GLM-5.3-Flash-NVFP4／Terminus-2 | **32768**（`max_tokens`） | RadixArk 的量化版本评测，effort 为 `max`；**关闭了任务 deadline**，存在无法运行及未完成的 trial，不能视为本项目同条件的通过率对照。[评测说明](https://huggingface.co/RadixArk/GLM-5.3-Flash-NVFP4#evaluation) | +| TB 2.0（补充） | Qwen3.6-27B／Harbor + Terminus-2 | **80K**（`max_tokens`，保留原文单位） | Qwen 披露 256K context、3 小时 timeout、32 CPU／48 GB RAM，取 5 次运行平均。[模型卡脚注](https://huggingface.co/Qwen/Qwen3.6-27B#evaluation) | +| TB 2.0（补充） | Kimi-K2.6／Tinker Cookbook 的简化 agent | **8192**（`max_tokens`） | Thinking Machines 的实验使用 32K context、200 轮、无上下文压缩；说明中记录了大量上下文超限。它证明 8192 也被实际采用过，不代表这一预算适合本项目。[实验配置与结果](https://github.com/thinking-machines-lab/tinker-cookbook/blob/main/tinker_cookbook/recipes/harbor_rl/README.md#evaluation) | + +与本项目更接近的 DeepSeek-V4，技术报告披露了代码 agent 使用内部 bash + file-edit +harness、最多 500 步和 512K **上下文**，但该段没有披露单次 `max_tokens`，评测 +还是 TB 2.0。因此，不能把 512K 当生成预算，也不能用它确认本项目 +`deepseek-v4-flash-0731` 在 TB 2.1 上应设置多少。[报告 §5.3.1](https://arxiv.org/html/2606.19348v1#S5.SS3.SSS1) + +**其他 code agent 的默认值。** 默认值必须同时注明版本、模型和配置路径。 +下面的源码链接固定到本次检查的 commit；官方文档按 2026-09-09 查询结果记录。 + +| Agent／检查范围 | 未主动覆盖时的行为 | 依据与边界 | +| --- | --- | --- | +| Claude Code／Opus 4.6 | **64k** | **v2.1.77 发布说明**明确将该模型的默认值提高到 64k；同条中的 128k 是可提高到的上界，不能当成默认值，也不能将该版本结论推广到所有模型。[发布说明](https://github.com/anthropics/claude-code/releases/tag/v2.1.77) | +| Claude Code／无法识别的模型 ID，例如网关自定义名称 | **32000** | 当前官方文档明确列出这个 fallback；可用 `CLAUDE_CODE_MAX_OUTPUT_TOKENS` 覆盖，已识别模型的默认值和上界因模型而异。[环境变量说明](https://code.claude.com/docs/en/env-vars#variables) | +| OpenCode／`830d5eb` | **`min(模型输出上限, 32000)`**；模型输出上限缺失或为 0 时回退到 32000 | 默认请求准备路径使用这一计算结果；环境变量和插件可覆盖。[计算函数](https://github.com/anomalyco/opencode/blob/830d5eb5354874105cc31599635a80c1662609e8/packages/opencode/src/provider/transform.ts#L1468)、[请求构造](https://github.com/anomalyco/opencode/blob/830d5eb5354874105cc31599635a80c1662609e8/packages/opencode/src/session/llm/request.ts#L117) | +| pi／`acaa253` 的普通模型请求路径 | **按模型的 `maxTokens` 决定**，再按剩余上下文收紧 | 当前 `buildBaseOptions` 采用 `options.maxTokens ?? model.maxTokens`，并为上下文估算留出 4096 tokens；不是所有模型共用一个固定数值。thinking 的预算还会经过对应 provider 的处理。[源码](https://github.com/earendil-works/pi/blob/acaa253cc8e3f159e6100b6f3874861b1f0bfc99/packages/ai/src/api/simple-options.ts#L11) | +| mini-swe-agent／`04d809c` 的默认配置与 LiteLLM 路径 | **不显式设置统一的 `max_tokens`** | 默认 YAML 未指定，模型层透传 `model_kwargs`；实际值由选用配置、LiteLLM 和服务端共同决定，省略参数不等于无限生成。[默认配置](https://github.com/SWE-agent/mini-swe-agent/blob/04d809ceab9df28f9adaed044884180159172930/src/minisweagent/config/default.yaml)、[模型调用](https://github.com/SWE-agent/mini-swe-agent/blob/04d809ceab9df28f9adaed044884180159172930/src/minisweagent/models/litellm_model.py#L64) | +| Codex CLI／官方配置参考 | **本次未确认统一的默认数字** | 所查官方配置参考未给出单次生成上限的默认值；不能拿 API 模型页上的最大输出能力代替 CLI 的实际默认设置。[配置参考](https://learn.chatgpt.com/docs/config-file/config-reference) | + +这些数字也提醒我们：**32000 与 32768 要分别记录**，不能都简写成“32k”后当成同一 +配置；只披露 `64k`、`80K` 的来源则保留原文单位。thinking 上限、`high`／`max` +这类 effort 档位也要与总生成上限分开记录；本次尚未确定独立 thinking 参数的设计。 + +**实现与自动化验收。** 实现提交为 `7a610cbf8`。配置入口采用 `--max-tokens`、 +`ANTHROPIC_MAX_TOKENS` 和现有 `settings.json` 的 `env`，优先级为 +**CLI > 环境变量 > 配置文件 > 32768**；交互和 headless 都生效，启动时解析一次， +每轮请求使用同一值。最终值必须是正整数，非法值在请求前以退出码 `2` 报错。 +配置文件沿用现有 `env` 字符串规则，例如 `"ANTHROPIC_MAX_TOKENS": "32768"`。 + +Harbor 支持 `--agent-kwarg max_tokens=N` 和 `ANTHROPIC_MAX_TOKENS` 透传；省略 +kwarg 时不强制插入 CLI 默认值，以便环境配置及所安装版本的默认值生效。 +启动提示、Journal 的 `run.started.max_tokens` 和 ATIF 的 `agent.extra.max_tokens` +都记录生效上限，实际 usage 另行记录。旧 Journal 缺少此字段时仍可转换，ATIF 不会 +为历史记录补写一个推测的预算。 -2026-09-09 补充。这里的预算首先指**单次模型回复的生成 token 上限**。当前上限 -固定为 8192，不能通过环境变量或 CLI 参数调整。本需求要提高经过实测选择的默认上限， -并提供配置入口，使日常使用与 Harbor 实验可以调整、记录实际预算。新的默认值、 -参数名称与配置优先级仍待设计；同时评估 thinking 与最终输出的预算分配。 +```bash +nanoPyCodeAgent --max-tokens 65536 -p "fix the failing tests" +# Harbor: add --agent-kwarg max_tokens=65536 to the run command +``` + +核心 **242 项测试**、Harbor adapter **22 项测试**通过，覆盖配置优先级、交互与 +headless 每轮请求的预算、非法值提前失败、记录与实际 usage 分离，以及真实 SDK +请求序列化。**65536 的真实模型结果见下文 09-12 实验；参数透传通过不等于效果归因成立。** **即使增大上限，也必须保留 `stop_reason="max_tokens"` 的处理。** 任何有限预算 都有可能耗尽。提高上限可以给模型更多生成空间；现有截断处理负责准确识别停止原因。 -这次两题均用尽 8192 tokens，为评估更高预算提供了对照，但不能保证提高后一定通过。 - -验收时保持模型、题目 hash 和 `max_turns=50` 一致，先关闭自动续写，只改变单次预算。 -检查配置值确实进入 API 请求，并在实验记录中保存生效值；用新 job 重跑上述两题， -比较 reward、截断情况、input／output tokens、费用和耗时。遇到再次截断时，仍须 -准确记录 outcome、保留 usage 与费用，不得将未完成回复当成任务完成。 - -#### 新需求：截断后自动续写（待实现） - -2026-09-09 补充。当前收到 `max_tokens` 就结束本次 run，即使还有剩余模型轮数也 +上次两题均用尽 8192 tokens，为评估更高预算提供了对照，但不能保证提高后一定通过。 + +增大预算后的截断回归也已通过：遇到 `max_tokens` 仍记录 `response_truncated`， +保留 usage 与费用，跳过该回复中的工具调用。本次没有新增独立 thinking 参数或 +自动续写。 + +**32768 的真实模型实验。** 使用新 job +`tb21-budget32768-20260909-7a610cb`，以 +`tb21-truncation-rerun2-20260909-f778861` 的 8192 结果作为对照。保留同一 +TB 2.1 dataset、两题 hash、OpenRouter 的 `deepseek/deepseek-v4-flash-0731`、 +`max_turns=50`、每题 900 秒原生 timeout、并发 2、每题 1 次且不重试。 +从实现提交构建本地 wheel，并逐个核对包内 Python 源文件与该提交一致。 +实验清除 `ANTHROPIC_MAX_TOKENS`，也不传 `--max-tokens`，验证内置默认值。 +`regex-log` 首次在安装阶段超过默认 360 秒，尚未进入模型执行，也没有 verifier +评分；该记录保留在原 job。另建 +`tb21-budget32768-regex-setup-retry-20260909-7a610cb` 仅补跑这题，将安装 timeout +放大为 1080 秒（`--agent-setup-timeout-multiplier 3`），解题 timeout 仍为 900 秒。 +`write-compressor` 首次进入模型执行，第一轮正常完成工具调用；整次执行约 421 秒 +时，第二轮收到服务端 `stream closed before completion`，Harbor 记录 `UnknownApiError`， +verifier reward 为 0。它没有返回 `max_tokens`，不能作为预算耗尽的证据。 +已完成首轮的 usage 为 input 1528／output 103，费用为 $0.00011786；中断调用的 +完整 usage 与费用未知，不能把这一已知小计当成整题总费用。 + +为排除一次性服务故障，再以相同模型、预算和解题时限补跑一次，保存为 +`tb21-budget32768-compressor-api-retry-20260909-7a610cb`；安装窗口同样为 1080 秒。 +两次模型运行均保留，不以补跑覆盖首次失败。 + +**最终结果（2026-09-09 执行，09-10 核对归档）。** 下表的时间是 Harbor 记录的 +agent 执行时间，不含安装与 verifier；首次安装失败没有模型执行时间或评分。 + +| 题目 | 8192 基线 | 32768 首次运行 | 32768 补跑 | +| --- | --- | --- | --- | +| `regex-log` | reward **0**；`response_truncated`；187.3 秒 | `AgentSetupTimeoutError`；未调用模型，未评分 | reward **0**；`AgentTimeoutError`；900.9 秒 | +| `write-compressor` | reward **0**；`response_truncated`；304.4 秒 | reward **0**；`UnknownApiError`；421.4 秒 | reward **0**；`AgentTimeoutError`；900.9 秒 | + +两次补跑的启动日志均确认 `max tokens 32768`，agent 版本和题目 hash 与实验配置 +相符。verifier 分别报告 `/app/regex.txt` 和 `/app/data.comp` 不存在。 +包括首次服务端中断在内，共有 **3 次进入模型执行且获评分的运行，全部为 0**；另有 +1 次安装失败。没有挑选一次补跑结果来替换原始失败。 + +**完整用量与成本比较未能完成。** 两次超时均未导出最终 ATIF，Harbor 将 trajectory +记为 `missing`，input／output tokens 和 cost 都为 `null`。模型调用次数、末次 +`stop_reason` 和实际截断次数也无法从现存记录完整恢复，不能将缺失值写成 0。 +8192 基线的完整费用分别为 $0.00158363、$0.001742255；32768 仅有前述首次 +`write-compressor` 已完成首轮的 $0.00011786 小计，无法计算可靠的总费用增幅。 + +因此，**保留 32768 作为本轮试用默认值，但不将 09-09 的实验写成提高预算后的成功验收**。 +参数配置、请求透传和截断回归已验证；真实模型实验仍受服务端中断、任务 timeout +和超时后 trajectory 缺失影响。没有完整的末次回复，不能断言更高预算消除了 +`max_tokens` 截断，也不能由这两题推导整套 benchmark 的收益。 + +三个 job 的 wheel 完全相同，SHA-256 为 +`91996b1d4c70af8062318a38b3af9235c85d93c9685e10ac1323f3bef3625425`。 +原始日志、verifier 与异常保存在上述各 job；对应的 `-record/` 保存 manifest、 +wheel 和运行脚本。统一汇总位于 +`jobs/tb21-budget32768-20260909-7a610cb-record/summary.json`、`results.en.md` +及 `workflow/summarize.py`。这些 `jobs/` 产物被 Git 忽略，未上传；相关容器已清理。 + +**延长时限并保留完整 trajectory（2026-09-10）。** 新 job +`tb21-regex-3600s-trace-20260910-7a610cb` 的 `regex-log__2A9PJgL` 得到 +**reward 1.0**，官方 verifier 的 `test_regex_matches_dates` 通过。 +Harbor 记录 agent 执行 **580.4 秒**，不含安装和验证；因此，这次成功并未用到 +原 900 秒以外的时间，不能据此认定延长时限解决了此前超时。 + +继续使用上述同一 wheel、题目 hash、模型、内置 `max_tokens=32768` 和 50 轮上限， +单题、单次、不重试。将解题执行窗口改为 **3600 秒**，另留 120 秒用于中断和导出； +Harbor 外层窗口设为 3780 秒（`--agent-timeout-multiplier 4.2`），避免提前杀死 +清理过程。安装仍为 1080 秒，verifier 时限不变。临时诊断包装器将未截短的 Journal、 +HTTP 响应体、逐块到达时间和每 120 秒的线程栈直接写入持久日志目录；agent 源码、 +提示词和工具保持本次对照配置。本次自然完成，未触发中断或轨迹重建。 + +| 观测项 | 结果 | +| --- | --- | +| 模型回复／工具调用 | **5 轮／6 次**；前 4 轮为 `tool_use`，末轮为 `end_turn`，无 `max_tokens` 截断 | +| 模型累计耗时 | **567.3 秒**，占原生 run 时间约 **97.9%** | +| 首轮／后续各轮耗时 | **502.4 秒**／35.1、2.8、4.2、22.8 秒 | +| 本地工具累计耗时 | **0.023 秒** | +| 完整 usage | prompt **74650**（含 cache **53504**），completion **17264** | +| 完整费用 | **$0.005338074**，5 次调用均已完成费用核对 | + +**证据指向本次首轮持续生成 thinking，而非本地工具耗时。** 首轮在请求后约 +2.5 秒开始返回 thinking，持续到第 500.8 秒，第 501.7 秒才开始输出工具参数； +非空生成片段之间最长间隔仅 **3.4 秒**，没有观察到长时间断流。该轮保留了 +47007 个 thinking 字符，服务端报告 output **15123 tokens**，其中 +`output_tokens_details.thinking_tokens` 为 **12026**。字符数与服务端 token +口径不同，不能把两项 token 数的差额直接当成可见文本量。 + +完整性检查通过：**147 条 Journal 记录均未截短**，原生 ATIF 通过 Harbor schema +校验，并与 Journal 重建结果完全一致；5 个响应流都有 `message_stop`，5 次请求均 +实际发送 32768。记录使用单调时钟分析流式耗时，避免系统时钟微调影响。诊断包装器的 +正常结束、模拟卡住后中断、强制终止后恢复三个离线场景也均通过。 + +**仍未确定旧超时的真正原因。** 旧运行缺少完整回复与流式时间线，无法区分当时的 +长时间生成、服务端／网络停滞或不同解题路径。这次记录了实际容器依赖 +Anthropic SDK **1.4.0**、httpx **0.28.1**、httpx2 **2.12.0**；旧容器未保存对应 +版本，因此不能认为同一 wheel 就等于全部运行条件相同。该结果证明这一题在 32768 +下能够完成，但单次成功不能证明延长时限的因果收益，也不能推广到另一题或整套评测。 + +完整 ATIF 位于上述 job 的 `regex-log__2A9PJgL/agent/trajectory.json`；同级 +`journals/`、`http/`、`http-events.jsonl` 和 `python-stacks.log` 保留原始诊断证据。 +对应 `-record/` 下的 `results.en.md`、`summary.json`、`validation.json` 和 +`workflow/` 保存诊断报告、完整性核对和复现脚本。产物保留在本地 `jobs/`，被 Git +忽略且未上传；容器已清理，此前失败记录保留。 + +**同法补跑 `write-compressor`（2026-09-11）。** 新 job +`tb21-compressor-3600s-trace-20260911-7a610cb` 的 `write-compressor__GTmsjFV` +得到 **reward 0**。官方 verifier 的 3 项测试均因 `/app/data.comp` 不存在失败。 +Harbor 记录 agent 执行 **1449.1 秒**（约 24 分钟），无基础设施异常，原生 trajectory +完整导出；最终 outcome 为 **`response_truncated`**，没有触发 3600 秒超时。 + +方法沿用上次诊断：同一模型和 wheel、内置 `max_tokens=32768`、50 轮上限、单题 +单次且不重试；执行／清理／Harbor 外层窗口分别为 3600／120／3780 秒，安装仍为 +1080 秒。诊断包装器与上次逐字节一致，继续保存未截短的 Journal、原始响应流、 +到达时间和线程栈。题目 hash 与此前 `write-compressor` 基线一致。 + +| 模型回复 | 耗时 | output tokens | 服务端报告的 thinking tokens | 停止原因 | +| --- | --- | --- | --- | --- | +| 第 1 轮 | **5.8 秒** | 118 | 16 | `tool_use` | +| 第 2 轮 | **1414.2 秒** | **32768** | **32767** | **`max_tokens`** | + +**这次明确观察到“持续长思考 → 超过原时限 → 用尽生成预算 → 未交付文件”。** +第 1 轮执行了 `read` 和 `bash`，合计仅 **0.004 秒**。第 2 轮约 0.9 秒开始返回 +thinking，持续到第 1414.1 秒；非空生成片段最长间隔 **2.2 秒**，没有长时间断流 +迹象。该轮完整保留 **116281 个 thinking 字符**，没有可见文本或工具参数输出。 +当前 agent 在 `max_tokens` 后结束 run，未自动续写，因此没有后续机会生成文件。 + +在原时限附近的 **第 900.4 秒**，独立只读检查确认 `data.comp` 不存在，仅完成 +首轮回复和两次工具调用，第二轮仍在输出 thinking。这证明本次执行无法在原窗口内 +交付；延长窗口让它继续到生成预算耗尽，**单独延长时限没有解决本次失败**。 +这条时间线解释了本次过程，不能反推旧运行中缺失的轨迹，也不能将此前服务端中断 +与本次截断混为同一个原因。 + +完整 usage 为 prompt **3926**、cache **0**、completion **32886**，费用 +**$0.00615504**。累计输出大于 32768，是因为预算按单次回复计算。两轮费用均已 +核对；第二轮费用查询先返回 4 次 HTTP 404，第 5 次成功，该等待发生在回复截断后。 +Harbor 的安装和 verifier 分别耗时 143.9、226.2 秒，均不计入上述 agent 执行时间。 + +**完整轨迹与模型回复被截断是两回事。** 本次 **13 条 Journal 均未截短**，包括 +超过默认 10 万字符日志阈值的 thinking；原生 ATIF 通过 Harbor schema 校验，并与 +Journal 重建结果一致。两个响应流均完整接收、有 `message_stop`，实际请求均为 +32768。实际容器的 Anthropic SDK 为 **1.5.0**，上次 regex 诊断为 1.4.0；同一 +wheel 的依赖范围允许安装到新版本，不能把相同方法等同于全部依赖和后端条件固定。 + +完整 ATIF 位于该 job 的 `write-compressor__GTmsjFV/agent/trajectory.json`，同级 +目录保留 Journal、HTTP 原始响应与时间线、线程栈和运行信息。对应 `-record/` 下 +保存 `results.en.md`、`summary.json`、`validation.json`、`checkpoints/` 和复现 +脚本，其中包含原 900 秒截止点的文件检查。产物仍在本地、被 Git 忽略且未上传， +容器已清理；此前所有失败及 regex 通过记录保留。 + +**再次各跑一题（2026-09-11）。** 本轮串行运行 `regex-log`、`write-compressor`， +各一次、不自动重试；第一题验证及容器清理完成后才启动第二题。模型、wheel、题目 +hash、32768 内置预算、50 轮上限及 3600／120／3780 秒窗口均沿用前次诊断。 + +| 题目 | 前次完整诊断 | 本次重复实验 | +| --- | --- | --- | +| `regex-log` | reward **1**；580.4 秒；5 轮 | reward **1**；**631.2 秒**；9 轮，末轮 `end_turn` | +| `write-compressor` | reward **0**；1449.1 秒；第 2 轮 `max_tokens` | reward **0**；**582.1 秒**；第 3 轮 `max_tokens` | + +时间均为 Harbor 的 agent 执行时间，不含安装与 verifier。**本次两题都未用到 +原 900 秒以外的解题时间**，没有 deadline、强制终止或轨迹重建。 + +regex 首轮仍耗时约 **517.4 秒**，使用 16445 output tokens，其中服务端报告 +16346 为 thinking tokens；之后完成 8 次工具调用，累计仅 0.039 秒。9 轮回复 +均未截断，官方 `test_regex_matches_dates` 再次通过。 + +compressor 本次第 2 轮用 **172.5 秒、5463 output tokens** 正常完成工具调用， +走到了第 3 轮；第 3 轮再用 **394.8 秒**耗尽 **32768** 总生成预算，服务端报告 +其中 **28961** 为 thinking tokens。捕获流包含 **112785 个 thinking 字符**， +没有可见文本或工具参数；不能把 token 字段的差额推算为可见文本。该轮非空生成 +片段最长间隔 1.5 秒。全部 3 次工具调用合计约 0.004 秒，最终仍没有 `data.comp`， +官方 3 项测试因此失败，outcome 为 `response_truncated`。 + +| 题目 | 完整 prompt tokens（含 cache） | Cache tokens | Completion tokens | 完整费用 | +| --- | --- | --- | --- | --- | +| `regex-log` | 161870 | 137216 | 20400 | **$0.00720948** | +| `write-compressor` | 12704 | 1280 | 38335 | **$0.00764908** | + +两题的 **256／18 条 Journal 均未截短**；原生 ATIF 均通过 Harbor schema 校验， +并与 Journal 重建结果一致。全部 12 个模型响应流均完整接收、有 `message_stop`， +请求实际发送 32768，所有调用费用均已核对。安装分别耗时 833.6／448.1 秒， +verifier 分别耗时 114.9／264.6 秒，均单列而不混入解题时间。 + +**对照中的两次 regex 均通过，两次 compressor 均耗尽生成预算，但耗时和停止轮次 +明显不同。** 本次 compressor 在 900 秒以内就失败，进一步确认这一次的停止条件是 +单次生成上限。两次观察不足以断言以后必过／必败，也没有验证更高预算必定有效。 +本轮两容器均记录 Anthropic SDK 1.5.0，其余已记录包版本一致；前次 regex 为 1.4.0。 +采样、缓存和后端条件没有固定，不能将时间差异直接归因于某一个因素。 + +本轮 job 分别为 `tb21-regex-3600s-trace-repeat2-20260911-7a610cb` 和 +`tb21-compressor-3600s-trace-repeat2-20260911-7a610cb`，trial 分别为 +`regex-log__NmpJwcQ`、`write-compressor__NqmJQWm`；各自 `agent/trajectory.json` +及同级 Journal／HTTP／线程栈保留完整证据。各 `-record/` 保存报告和复现脚本， +统一对照在 `jobs/tb21-3600s-trace-repeat2-20260911-7a610cb-record/` 的 +`results.en.md` 与 `comparison.json`。原始产物仍在本地、被 Git 忽略且未上传， +两容器已清理，此前记录全部保留。 + +**65536 重跑 compressor（2026-09-12）。** 本次显式传入 +`--agent-kwarg max_tokens=65536`，其余沿用前次 compressor 的模型、wheel、题目 +hash、50 轮及 3600／120／3780 秒窗口；单题一次、不自动重试，项目默认值仍为 +32768。最终 **reward 1，官方 3 项测试全部通过**：`data.comp` 为 **2263 字节**， +解压内容与原文件一致。 + +| 项目 | 本次结果 | +| --- | --- | +| 解题时间 | **1976.4 秒**；安装 278.2 秒、verifier 169.1 秒另计 | +| 模型回复／工具调用 | **32 轮／32 次**；末轮 `end_turn`，outcome `completed` | +| 最大单轮输出 | **32073 tokens**；所有请求实际发送 65536，无截断 | +| 完整 prompt／cache／completion tokens | **1528746／1471744／59563**；prompt 含 cache | +| 完整费用 | **$0.018819072**，32 次调用费用全部核对完成 | + +主要耗时仍是第 2 轮：**1179.7 秒**、32073 output tokens，其中服务端报告 +27900 thinking tokens；随后正常返回 `tool_use`。完整流含 108811 个 thinking +字符，非空生成片段最长间隔 2.2 秒。全部工具累计 27.3 秒。900.4 秒检查点尚无 +目标文件，1299.7 秒时为 2727 字节、1449.9 秒时为 2388 字节，最终再缩至 2263。 + +**本次证明“65536 配置下重跑通过”，但没有证明“32768 一定不够”。** 所有单轮 +输出都低于 32768，采样、缓存和后端未固定，不能把通过直接归因于翻倍预算。 +相比之下，本次轨迹确实用到了原 900 秒以外的时间。运行环境记录的 SDK 1.5.0 +及其他依赖版本与前次 compressor 一致；没有 deadline、强杀或轨迹重建。 + +498 条 Journal 均未截短，原生 ATIF 通过 schema 校验并与 Journal 投影一致。 +32 个模型响应流均完整接收并含 `message_stop`；一次费用查询 GET 在响应头前 +`ConnectError`，重试后成功，未中断模型回复。原分析器对“所有 HTTP 请求均收到 +完整响应”的检查因此为 false，`validation-notes.json` 保留并解释该区别。 + +job 为 `tb21-compressor-65536-3600s-trace-20260912-7a610cb`，trial 为 +`write-compressor__hzfqDFs`；完整 `agent/trajectory.json`、Journal、HTTP 流与 +时间线均保留。对应 `-record/` 保存报告、校验、检查点和复现脚本，`task-output/` +另存最终压缩文件及输入／解压器源码和 hash。产物仍在本地、被 Git 忽略且未上传， +容器已清理，先前结果全部保留。 + +**65536 扩展到其余 18 题（2026-09-12～13）。** 首轮耗时约 2 小时 57 分， +结果为 **5 题通过、8 题 reward 0、5 题未评分**，即 5/18（27.8%）；同组 +8192 基线为 8/18。通过的是 `pypi-server`、`openssl-selfsigned-cert`、 +`log-summary-date-ranges`、`pytorch-model-recovery` 和 `merge-diff-arc-agi-task`。 +随后 4 题共补跑 5 次，均未新增通过。首轮与补跑分别记录，保留所有失败。 + +继续使用 `7a610cb` 的同一已校验 wheel、DeepSeek V4 Flash 0731／OpenRouter、 +相同题目 hash；18 题的缓存镜像 digest 也与基线一致。显式设 `max_tokens=65536`、 +最多 50 轮、并发 2；解题 3600 秒、收尾 120 秒、Harbor 外层 3780 秒、安装 +1080 秒，verifier 保留题目原时限。每题每个 job 一次，不自动重试整题。 +**项目默认值仍为 32768，65536 是本次实验覆盖值。** + +表中 `—` 表示未得到官方评分，不计作通过；reward 0 也可能来自基础设施或记录器 +故障，需结合原因判断。 + +| 题目 | 8192 基线 | 65536 首轮 | 主要结果 | +| --- | ---: | ---: | --- | +| `torch-tensor-parallelism` | 0 | — | 首个响应流中断；verifier 下载 Torch／CUDA 依赖超时 | +| `schemelike-metacircular-eval` | 0 | 0 | 第 19 轮响应流中断；解释器仅通过 4/63 子例 | +| `kv-store-grpc` | 1 | 0 | `SetValRequest` 缺少要求的 `value` 字段；通过 5/7 测试，服务存活检查通过 | +| `pypi-server` | 1 | 1 | 本地包安装及 API 测试通过 | +| `dna-assembly` | 0 | — | 安装 uv 时 TLS 下载失败，未调用模型 | +| `torch-pipeline-parallelism` | 0 | — | 耗尽 50 轮；verifier 下载依赖超时，无功能测试结果 | +| `qemu-alpine-ssh` | 1 | — | 安装依赖时 APT 包返回 404，未调用模型 | +| `openssl-selfsigned-cert` | 1 | 1 | 6/6 测试通过 | +| `regex-chess` | 0 | 0 | 耗尽 50 轮；初始局面生成非法后继，仅通过长度检查（1/4） | +| `log-summary-date-ranges` | 1 | 1 | 2/2 测试通过 | +| `model-extraction-relu-logits` | 0 | — | 获取 uv 安装脚本时 TLS 失败，未调用模型 | +| `path-tracing` | 0 | 0 | 耗尽 50 轮，缺少 `image.c`；0/5 测试通过 | +| `caffe-cifar-10` | 0 | 0 | 耗尽 50 轮；末次构建缺少 `g++`，二进制及模型未生成；仅通过 prototxt 检查（1/6） | +| `mteb-leaderboard` | 0 | 0 | 耗尽 50 轮，缺少 `result.txt`；0/2 测试通过 | +| `llm-inference-batching-scheduler` | 1 | 0 | 记录器在约 120 秒定时转储栈时 SIGSEGV，计划文件缺失 | +| `pytorch-model-recovery` | 1 | 1 | 5/5 测试通过 | +| `circuit-fibsqrt` | 0 | 0 | 电路计算错误；空 `bash` 参数触发原生 `KeyError('command')`，中止修复；通过 2/3 测试 | +| `merge-diff-arc-agi-task` | 1 | 1 | 5/5 测试通过 | + +**补跑只用于排除已确认的环境／记录器问题，结果不覆盖首轮。** 3 个安装失败任务 +先用缓存 uv 和 wheel 绕过下载／APT 故障；scheduler 单独移除定时栈转储后再跑。 +缓存安装方案遗漏了 DNA 镜像需要的系统 CA 证书，因此 DNA 又修正安装步骤重跑一次。 + +| 题目 | 追加次数 | 补跑结果 | +| --- | ---: | --- | +| `dna-assembly` | 2 | 第一次因上述 CA 遗漏，在收到响应头前连接失败，不能评价模型；恢复原有全部系统依赖后，完成 3 轮，但不完整 `bash` 参数触发 `KeyError('command')`，缺少 `primers.fasta`，0/1 测试通过 | +| `qemu-alpine-ssh` | 1 | 完成 3 轮后，不完整 `bash` 参数触发 `KeyError('command')`；verifier 又遇 APT 404，pytest 未运行，脚本虽记 reward 0，不能视为功能测试失败 | +| `model-extraction-relu-logits` | 1 | 正常完成 14 轮，但恢复矩阵的第 11 行不匹配，官方测试失败 | +| `llm-inference-batching-scheduler` | 1 | 无 SIGSEGV；完成 15 轮后，不完整 `write` 参数触发 `KeyError('content')`，计划文件缺失，仅通过输入完整性检查（1/6） | + +记录器和安装方法的差异也纳入证据。18 题开跑前已修正 supervisor 的正常退出清理： +保留 agent 启动的后台服务供 verifier 使用，仅在 deadline 时清理进程组；离线服务 +测试通过，`kv-store-grpc` 的服务存活测试也通过。首轮 scheduler 的系统 core 元数据、 +同镜像 ELF 与反汇编定位到定时栈转储中的 `PyCode_Addr2Line+11`；该调用路径与 +[CPython 定时转储实现](https://github.com/python/cpython/blob/v3.13.7/Modules/faulthandler.c#L537) +及[栈帧转储实现](https://github.com/python/cpython/blob/v3.13.7/Python/traceback.c#L842) +一致，但未确定底层内存错误机制。补跑保留原安装方法，只去掉定时转储，未再崩溃。 +DNA 则用同镜像、不带密钥的公开 HTTPS 请求验证：缺 CA 时证书验证失败，补装后 +返回 HTTP 200；修正后的正式补跑恢复 `curl`、`bash`、`git`、`python3` 和 CA +证书，再安装缓存 uv／wheel。这两类诊断错误不能算作模型预算不足。 + +**对生成预算的直接结论：继续加大上限不能统一解决当前失败。** + +- 所有本次 65536 运行均未报告 `max_tokens`，也没有触及 3600 秒解题时限。 + 实际单轮超过 32768 的只有 Scheme（38496）、regex-chess（60483）和修正后 + 的 DNA（35730），三题仍未通过。通过的 compressor 峰值为 32073，其余通过题 + 均不超过 6015。 +- pipeline、regex-chess、path-tracing、Caffe、MTEB 首轮耗尽 **50 轮**；这是 + 总调用轮数限制，与单次生成预算不同。 +- circuit、QEMU、scheduler 补跑和 DNA 修正补跑暴露了另一项 agent 健壮性问题： + 响应流完整结束于 `tool_use`，却给出缺失或不完整的工具参数，SDK 得到空字典； + 原生渲染器直接读取必需字段而抛出 `KeyError`。其中 circuit 的原始 JSON 是 + `{}`，QEMU／scheduler 是 `{`，DNA 是未闭合字符串，不能全部称为合法空 JSON。 + 下一步应单独处理参数校验和错误反馈，使 agent 有机会修复；本次没有修改这部分实现。 + +这些是诊断观察，**不是只改变预算的受控实验**：8192 基线源码为 `2b83097`， +使用原题目时限，依赖、采样、缓存和后端也未固定。因此既不能从 8/18 → 5/18 +认定增大预算有害，也不能把 compressor 的一次通过归功于预算翻倍。`regex-log` +此前两次通过使用的是 32768，本次未在 65536 下重跑；不把这些不同条件拼成统一的 +20 题通过率。 + +其余 18 题及 5 次补跑的已核对费用下界为 **$0.261769566**；加上 compressor， +本次 24 次尝试合计至少 **$0.280588638**。tensor-parallelism 的费用查询三次 +返回 404，完整总费用和 token 用量仍未知；中断的 Scheme／scheduler 调用已单独 +补查账单，不能用账单修补缺失的原生 usage，也不把未知值当作 0。 + +[结构化实验结果](../../../benchmarks/harbor/results/tb21-generation-budget-65536-20260912.json) +收录配置、源码／wheel hash、逐题结果、费用边界及各 job／trajectory 的本地路径。 +24 次尝试覆盖 19 题：**20 份原生 ATIF、1 份明确标记的崩溃后重建 ATIF、3 次 +安装未完成而无轨迹**；所有现存 ATIF 均通过 schema 校验，原生轨迹与完整 Journal +投影一致，无 Journal 字符串截短。实际收到的 HTTP 流（含中断流）、时间线、运行时 +版本、校验报告及诊断脚本均保留在 `jobs/`；主记录目录为 +`tb21-remaining18-65536-3600s-trace-20260912-7a610cb-record`，其中 +`comparison.json` 索引全部分组，`results.en.md` 记录首轮 18 题。 +原始轨迹被 Git 忽略且未上传，所有实验及诊断容器已清理。 + +#### 截断后自动续写 + +当前收到 `max_tokens` 就结束本次 run，即使还有剩余模型轮数也 不会继续。新需求是在配置允许且仍有剩余预算时，由 agent 自动发起后续模型调用， 继续完成原任务，使 headless 运行不必依赖用户手动发送“继续”。 @@ -681,7 +1064,7 @@ validator，启动版本包含 `f778861`，日志都有明确的截断提示， thinking 的回复，以及截断工具参数不被执行或错误重放。真实模型验收使用上述两题 和新的 job 名，在相同单次预算下比较关闭／开启续写的结果，再评估“增加预算＋ 自动续写”的组合；分别报告通过率、追加模型调用数、总 tokens、费用和耗时。 -两个需求都尚未实现，本次 0/2 的实测结果不能当作它们的验收结果。 +自动续写尚未实现；前述 8192 下的 0/2 结果及提高预算后的实验均不属于自动续写验收。 #### 缺失工具参数导致的崩溃 diff --git a/docs/user_docs/en/cli_reference.md b/docs/user_docs/en/cli_reference.md index f9081af..679ea44 100644 --- a/docs/user_docs/en/cli_reference.md +++ b/docs/user_docs/en/cli_reference.md @@ -10,7 +10,7 @@ task. ## Synopsis ```text -nanoPyCodeAgent [-h] [-p TEXT | --prompt-file PATH] [--max-turns N] +nanoPyCodeAgent [-h] [-p TEXT | --prompt-file PATH] [--max-turns N] [--max-tokens N] [--trajectory PATH] [--version] ``` @@ -71,6 +71,7 @@ working directory. | `-p TEXT`, `--prompt TEXT` | — | Run `TEXT` as one headless task. | | `--prompt-file PATH` | — | Read one headless task from a UTF-8 file. The file must be readable and contain a non-empty task. | | `--max-turns N` | `50` | Allow at most `N` model replies in a headless run. `N` must be an integer of at least `1`. | +| `--max-tokens N` | `ANTHROPIC_MAX_TOKENS` or `32768` | Maximum generated tokens per model reply in either mode. `N` must be a positive integer; the CLI value overrides environment and settings-file values. | | `--trajectory PATH` | disabled | Write the headless run as one ATIF-v1.7 JSON document. See [Trajectory output](#trajectory-output). | | `--version` | — | Print `nanoPyCodeAgent VERSION` and exit successfully. | @@ -79,7 +80,14 @@ calls. If reply `N` still requests tools, those tools are not run because no reply remains to consume their results. Reaching the limit prints a diagnostic to stderr but is still a normal headless exit. -Each reply has a separate fixed limit of 8192 generated tokens. If the provider +Each reply has a separate generation limit, defaulting to **32768** tokens. +Use `--max-tokens 65536` to override it for one invocation; use +`ANTHROPIC_MAX_TOKENS` in the environment or settings file for a persistent +default. The provider must accept the requested limit for the chosen model. +The limit applies to the whole generated reply, including thinking where the +provider counts it; it is not a guaranteed amount of visible answer text. + +If the provider returns `stop_reason="max_tokens"`, the agent stops that run, prints a truncation diagnostic to stderr, and skips all tools from that reply. It preserves partial text and records `response_truncated` as the trajectory terminal outcome. @@ -103,7 +111,7 @@ stdout mode. | --- | --- | | `0` | Help or version output completed; an interactive session ended normally; or a headless run started and returned control, even if the model gave up, left work incomplete, or exhausted `--max-turns`. | | `1` | A runtime or infrastructure failure prevented a normal run, including missing API credentials or an Anthropic/HTTP API failure. | -| `2` | Command-line usage was invalid, including conflicting or empty task input, an invalid turn limit, an unreadable prompt file, or an invalid trajectory destination. | +| `2` | Command-line usage or the generation-budget setting was invalid, including conflicting or empty task input, an invalid turn or token limit, an unreadable prompt file, or an invalid trajectory destination. | Exit status `0` does not certify that a headless task succeeded. A script or benchmark must inspect the resulting workspace or run its own verifier. @@ -127,6 +135,11 @@ cost information when available, and terminal state describe a single Agent Run. A caught API failure after the run has started produces a partial trajectory with a failed terminal state. +The startup banner and `run.started.max_tokens` in the Journal record the +effective per-reply limit. ATIF exposes it as `agent.extra.max_tokens`, separately +from actual token usage. Older journals without this field still export; +their trajectories omit the unknown limit. + The path contract is: - `--trajectory` requires a headless task and cannot be used in interactive diff --git a/docs/user_docs/en/configuration.md b/docs/user_docs/en/configuration.md index df6f3c4..b607cfa 100644 --- a/docs/user_docs/en/configuration.md +++ b/docs/user_docs/en/configuration.md @@ -24,6 +24,10 @@ The settings file is loaded before the Anthropic client is created and before the model is selected, so the same precedence applies in interactive and headless modes. +The per-reply generation limit also accepts `--max-tokens N`. For this setting, +the order is **CLI > environment > settings file > 32768**. It is resolved once +at startup and applies to every model reply in that invocation. + ## Supported settings | Variable | Required | Default | Description | @@ -32,13 +36,14 @@ headless modes. | `ANTHROPIC_AUTH_TOKEN` | One credential is required | none | Bearer token used for services that authenticate with `Authorization: Bearer`, such as OpenRouter's Anthropic-compatible endpoint. | | `ANTHROPIC_BASE_URL` | No | `https://api.anthropic.com` | Base URL used by the Anthropic SDK. Set it for a compatible proxy or third-party endpoint; leave it unset for the official API. | | `ANTHROPIC_MODEL` | No | `claude-sonnet-4-6` | Model passed to every Messages API call. | +| `ANTHROPIC_MAX_TOKENS` | No | `32768` | Positive integer controlling the generated tokens per model reply. Overridden by `--max-tokens`. | At least one of `ANTHROPIC_API_KEY` or `ANTHROPIC_AUTH_TOKEN` must provide a usable credential. If neither is available, the command reports the missing credentials on stderr and exits with status `1` before starting an Agent Run. The settings-file loader accepts any key whose name begins with `ANTHROPIC_`. -The four variables above are the nanoPyCodeAgent configuration contract; +The five variables above are the nanoPyCodeAgent configuration contract; additional variables are interpreted, if at all, by the installed Anthropic Python SDK and can change with that dependency. @@ -83,7 +88,8 @@ the `env` field in [Claude Code settings](https://code.claude.com/docs/en/settin "ANTHROPIC_API_KEY": "", "ANTHROPIC_AUTH_TOKEN": "", "ANTHROPIC_BASE_URL": "", - "ANTHROPIC_MODEL": "" + "ANTHROPIC_MODEL": "", + "ANTHROPIC_MAX_TOKENS": "" } } ``` @@ -96,7 +102,8 @@ rest empty or remove their keys. For example: "env": { "ANTHROPIC_AUTH_TOKEN": "your-token", "ANTHROPIC_BASE_URL": "https://example.com/anthropic", - "ANTHROPIC_MODEL": "provider/model-name" + "ANTHROPIC_MODEL": "provider/model-name", + "ANTHROPIC_MAX_TOKENS": "32768" } } ``` @@ -144,6 +151,12 @@ supply it: unset ANTHROPIC_API_KEY ANTHROPIC_AUTH_TOKEN ANTHROPIC_BASE_URL ANTHROPIC_MODEL ``` +For `ANTHROPIC_MAX_TOKENS`, a final environment value that is empty, non-integer, +zero, or negative is a usage error (exit `2`) before any model request. A valid +CLI override takes precedence even over an invalid budget in the environment. +Settings-file entries follow the string-only and placeholder rules above; +write the budget as `"32768"`, not a JSON number. + ## Precedence example Given this file: diff --git a/docs/user_docs/zh-CN/cli_reference.md b/docs/user_docs/zh-CN/cli_reference.md index 2758fb1..ebd03e3 100644 --- a/docs/user_docs/zh-CN/cli_reference.md +++ b/docs/user_docs/zh-CN/cli_reference.md @@ -9,7 +9,7 @@ ## 命令格式 ```text -nanoPyCodeAgent [-h] [-p TEXT | --prompt-file PATH] [--max-turns N] +nanoPyCodeAgent [-h] [-p TEXT | --prompt-file PATH] [--max-turns N] [--max-tokens N] [--trajectory PATH] [--version] ``` @@ -63,6 +63,7 @@ nanoPyCodeAgent -p "fix the failing tests" | `-p TEXT`、`--prompt TEXT` | 无 | 把 `TEXT` 作为一次 headless 任务运行。 | | `--prompt-file PATH` | 无 | 从 UTF-8 文件读取一次 headless 任务;文件必须可读并包含非空任务。 | | `--max-turns N` | `50` | 一次 headless run 最多允许 `N` 轮模型回复;`N` 必须是大于或等于 `1` 的整数。 | +| `--max-tokens N` | `ANTHROPIC_MAX_TOKENS` 或 `32768` | 两种模式下每次模型回复的最大生成 token 数。`N` 必须是正整数；CLI 值优先于环境变量与 settings 文件。 | | `--trajectory PATH` | 禁用 | 把 headless run 写成一份 ATIF-v1.7 JSON 文档;参见[Trajectory 输出](#trajectory-输出)。 | | `--version` | 无 | 打印 `nanoPyCodeAgent VERSION` 并成功退出。 | @@ -70,7 +71,13 @@ nanoPyCodeAgent -p "fix the failing tests" 工具,这些工具不会执行,因为已经没有下一轮回复可以使用工具结果。达到上限时,命令 会在 stderr 打印诊断,但仍属于一次正常的 headless 退出。 -每次回复还受独立的 8192 生成 token 固定上限约束。如果 provider 返回 +每次回复还受独立的生成上限约束，默认为 **32768** tokens。可用 +`--max-tokens 65536` 覆盖本次调用，或通过环境变量、settings 文件中的 +`ANTHROPIC_MAX_TOKENS` 设置持久默认值。所选模型与 provider 必须接受该上限。 +它限制整次生成；provider 将 thinking 计入生成预算时，thinking 也占用这一上限， +因此它不保证相同数量的可见回答文本。 + +如果 provider 返回 `stop_reason="max_tokens"`,agent 会停止本次 run,向 stderr 打印截断诊断,并跳过该 回复中的所有工具调用。已输出的文本会保留,trajectory 的终态 outcome 记录为 `response_truncated`。Headless 模式仍退出 `0`,不会自动重试或续写。交互模式会 @@ -90,7 +97,7 @@ Headless run 期间,stdout 包含流式模型文本以及回显的工具调用 | --- | --- | | `0` | help 或 version 输出完成;交互会话正常结束;或者 headless run 已经启动并交回控制权,即使模型放弃、工作未完成或用尽了 `--max-turns`。 | | `1` | runtime 或基础设施故障阻止了正常运行,包括缺少 API 凭据或 Anthropic/HTTP API 失败。 | -| `2` | 命令行用法无效,包括任务输入冲突或为空、轮数上限无效、prompt file 无法读取,或者 trajectory 目标无效。 | +| `2` | 命令行用法无效,包括任务输入冲突或为空、轮数或生成 token 上限无效、prompt file 无法读取,或者 trajectory 目标无效。 | 退出状态 `0` 不证明 headless 任务成功。脚本或 benchmark 必须检查产生的 workspace, 或者运行自己的 verifier。CLI 未处理的意外故障也可能让进程以非零状态和 traceback @@ -111,6 +118,10 @@ Trajectory 是独立 artifact,不会替代或重定向 stdout。它描述单次 模型回复、工具参数、工具结果、时间、用量、可获得的成本信息和终态。如果 run 启动后 发生被捕获的 API 失败,仍会产生带失败终态的 partial trajectory。 +启动提示和 Journal 的 `run.started.max_tokens` 记录最终生效的单次生成上限; +ATIF 将它保存在 `agent.extra.max_tokens` 中。这是请求预算,实际消耗仍由 usage +字段记录。旧 Journal 没有预算信息时,转换后的 ATIF 会省略此字段。 + 路径契约如下: - `--trajectory` 需要 headless 任务,不能在交互模式中使用。 diff --git a/docs/user_docs/zh-CN/configuration.md b/docs/user_docs/zh-CN/configuration.md index 1fb0a7f..de09e38 100644 --- a/docs/user_docs/zh-CN/configuration.md +++ b/docs/user_docs/zh-CN/configuration.md @@ -21,6 +21,10 @@ nanoPyCodeAgent 不加载项目 `.env` 文件,没有项目级 settings 文件, Settings 文件在 Anthropic client 创建和模型选择之前加载,所以交互模式与 headless 模式遵循同一套优先级。 +单次回复的生成上限还支持 `--max-tokens N`。此设置的优先级为 +**CLI > 环境变量 > 配置文件 > 32768**。启动时解析一次,本次调用中的每次模型回复 +都使用同一上限。 + ## 支持的设置 | 变量 | 是否必需 | 默认值 | 说明 | @@ -29,11 +33,12 @@ Settings 文件在 Anthropic client 创建和模型选择之前加载,所以交 | `ANTHROPIC_AUTH_TOKEN` | 两种凭据至少提供一种 | 无 | 需要以 `Authorization: Bearer` 认证的服务所使用的 bearer token,例如 OpenRouter 的 Anthropic-compatible endpoint。 | | `ANTHROPIC_BASE_URL` | 否 | `https://api.anthropic.com` | Anthropic SDK 使用的 base URL。兼容的 proxy 或第三方 endpoint 需要设置此项;使用官方 API 时保持未设置。 | | `ANTHROPIC_MODEL` | 否 | `claude-sonnet-4-6` | 每次 Messages API 调用所使用的模型。 | +| `ANTHROPIC_MAX_TOKENS` | 否 | `32768` | 控制每次模型回复生成 token 数的正整数,可由 `--max-tokens` 覆盖。 | `ANTHROPIC_API_KEY` 与 `ANTHROPIC_AUTH_TOKEN` 中至少要有一个提供可用凭据。两者都 不可用时,命令会在 stderr 报告缺少凭据,并在 Agent Run 启动前以状态 `1` 退出。 -Settings 文件 loader 接受名称以 `ANTHROPIC_` 开头的任何键。上面的四个变量是 +Settings 文件 loader 接受名称以 `ANTHROPIC_` 开头的任何键。上面的五个变量是 nanoPyCodeAgent 的配置契约;其他变量是否生效由安装的 Anthropic Python SDK 决定, 并可能随着该依赖变化。 @@ -76,7 +81,8 @@ nanoPyCodeAgent -p "run the test suite" "ANTHROPIC_API_KEY": "", "ANTHROPIC_AUTH_TOKEN": "", "ANTHROPIC_BASE_URL": "", - "ANTHROPIC_MODEL": "" + "ANTHROPIC_MODEL": "", + "ANTHROPIC_MAX_TOKENS": "" } } ``` @@ -88,7 +94,8 @@ nanoPyCodeAgent -p "run the test suite" "env": { "ANTHROPIC_AUTH_TOKEN": "your-token", "ANTHROPIC_BASE_URL": "https://example.com/anthropic", - "ANTHROPIC_MODEL": "provider/model-name" + "ANTHROPIC_MODEL": "provider/model-name", + "ANTHROPIC_MAX_TOKENS": "32768" } } ``` @@ -129,6 +136,10 @@ chmod 600 ~/.nanoPyCodeAgent/settings.json unset ANTHROPIC_API_KEY ANTHROPIC_AUTH_TOKEN ANTHROPIC_BASE_URL ANTHROPIC_MODEL ``` +`ANTHROPIC_MAX_TOKENS` 的最终环境值为空、非整数、零或负数时,会在任何模型请求前 +报用法错误并以状态 `2` 退出。有效的 CLI 覆盖值优先于环境中的非法预算值。 +配置文件仍遵循上述字符串与占位符规则;预算应写成 `"32768"`,而不是 JSON 数字。 + ## 优先级示例 假设文件内容如下: diff --git a/src/nanopycodeagent/agent.py b/src/nanopycodeagent/agent.py index 50ba8b1..111f51e 100644 --- a/src/nanopycodeagent/agent.py +++ b/src/nanopycodeagent/agent.py @@ -58,14 +58,13 @@ utc_now, ) from .read_tool import READ_TOOL, run_read -from .settings import load_settings_env +from .settings import DEFAULT_MAX_TOKENS, load_settings_env, resolve_max_tokens from .terminal import Spinner, print_tool_output, print_tool_use from .write_tool import WRITE_TOOL, content_preview, run_write # The model used when ANTHROPIC_MODEL is set in neither the environment nor # the config file. DEFAULT_MODEL = "claude-sonnet-4-6" -MAX_TOKENS = 8192 _TRUNCATION_NOTICE = ( "[response truncated: reached max_tokens; stopped without finishing the task. " @@ -360,6 +359,7 @@ def _run_exchange( system: str, *, max_turns: int | None = None, + max_tokens: int = DEFAULT_MAX_TOKENS, reply_prefix: str = "\nAgent> ", trajectory_path: Path | None = None, ) -> RunOutcome: @@ -381,6 +381,7 @@ def _run_exchange( "mode": "headless" if max_turns is not None else "interactive", "model": model, "max_turns": max_turns, + "max_tokens": max_tokens, "producer": { "name": "nanoPyCodeAgent", "version": _package_version(), @@ -405,6 +406,7 @@ def _run_exchange( system, emitter=emitter, max_turns=max_turns, + max_tokens=max_tokens, ) except BaseException as exc: cost_reconciliation = _reconcile_costs(client, journal, emitter) @@ -457,6 +459,7 @@ def _run_model_loop( *, emitter: EventEmitter, max_turns: int | None, + max_tokens: int, ) -> RunOutcome: """Run model replies and tool calls for an already-started Agent Run.""" turns = 0 @@ -478,7 +481,7 @@ def _run_model_loop( # the accumulated message for the conversation history. with Spinner() as spinner, client.messages.stream( model=model, - max_tokens=MAX_TOKENS, + max_tokens=max_tokens, system=system, tools=TOOLS, messages=messages, @@ -605,20 +608,22 @@ def _reconcile_costs( return outcomes -def run() -> int: +def run(*, max_tokens: int | None = None) -> int: """Start the read → ask → answer loop until the user types ``/exit``. A reply may include tool calls; they are executed and their results fed back to the model until it finishes the turn without tool use. Returns the process exit code. """ + max_tokens = resolve_max_tokens(max_tokens) client = _create_client() if client is None: return 1 model = _resolve_model() print( - f"nanoPyCodeAgent v{_package_version()} — model {model} " + f"nanoPyCodeAgent v{_package_version()} — model {model}, " + f"max tokens {max_tokens} " "(set ANTHROPIC_MODEL to override)." ) print("Type a message to chat, or /exit to quit.") @@ -640,7 +645,7 @@ def run() -> int: break messages.append({"role": "user", "content": user_input}) - _run_exchange(client, model, messages, SYSTEM_PROMPT) + _run_exchange(client, model, messages, SYSTEM_PROMPT, max_tokens=max_tokens) print("Bye!") return 0 @@ -650,6 +655,7 @@ def run_headless( task: str, *, max_turns: int = DEFAULT_MAX_TURNS, + max_tokens: int | None = None, trajectory_path: Path | None = None, ) -> int: """Work ``task`` to completion without a user, and return the exit code. @@ -662,6 +668,7 @@ def run_headless( scores the result. Only a run that could not happen at all — no credentials, an API that keeps refusing — exits non-zero. """ + max_tokens = resolve_max_tokens(max_tokens) client = _create_client() if client is None: return 1 @@ -671,7 +678,7 @@ def run_headless( # model's prose and the echoed tool calls, nothing else. print( f"nanoPyCodeAgent v{_package_version()} — model {model}, " - f"max turns {max_turns}", + f"max turns {max_turns}, max tokens {max_tokens}", file=sys.stderr, ) @@ -683,6 +690,7 @@ def run_headless( messages, HEADLESS_SYSTEM_PROMPT, max_turns=max_turns, + max_tokens=max_tokens, reply_prefix="", trajectory_path=trajectory_path, ) diff --git a/src/nanopycodeagent/atif.py b/src/nanopycodeagent/atif.py index dd68cca..ca1a5b5 100644 --- a/src/nanopycodeagent/atif.py +++ b/src/nanopycodeagent/atif.py @@ -424,6 +424,11 @@ def project_atif(entries: Sequence[JournalEntry]) -> JsonObject: "extra": { "mode": run_payload["mode"], "max_turns": run_payload["max_turns"], + **( + {"max_tokens": run_payload["max_tokens"]} + if "max_tokens" in run_payload + else {} + ), }, }, "steps": steps, diff --git a/src/nanopycodeagent/cli.py b/src/nanopycodeagent/cli.py index c895275..93173de 100644 --- a/src/nanopycodeagent/cli.py +++ b/src/nanopycodeagent/cli.py @@ -17,6 +17,7 @@ from pathlib import Path from .agent import DEFAULT_MAX_TURNS, _package_version, run, run_headless +from .settings import DEFAULT_MAX_TOKENS, resolve_max_tokens # Reserved by argparse for a misuse of the command line itself, and used here # for the same: a task that cannot be read is a mistake in how the agent was @@ -54,6 +55,15 @@ def _build_parser() -> argparse.ArgumentParser: f"(default: {DEFAULT_MAX_TURNS})" ), ) + parser.add_argument( + "--max-tokens", + type=int, + metavar="N", + help=( + "maximum generated tokens per model reply in either mode " + f"(default: ANTHROPIC_MAX_TOKENS or {DEFAULT_MAX_TOKENS})" + ), + ) parser.add_argument( "--trajectory", type=Path, @@ -122,15 +132,20 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) if args.max_turns < 1: parser.error("--max-turns must be at least 1") + try: + max_tokens = resolve_max_tokens(args.max_tokens) + except ValueError as exc: + parser.error(str(exc)) task = _read_task(args, parser) if task is None: if args.trajectory is not None: parser.error("--trajectory requires a headless task") - return run() + return run(max_tokens=max_tokens) trajectory_path = _trajectory_path(args.trajectory, parser) return run_headless( task, max_turns=args.max_turns, + max_tokens=max_tokens, trajectory_path=trajectory_path, ) diff --git a/src/nanopycodeagent/event_journal.py b/src/nanopycodeagent/event_journal.py index 592de86..edac9d0 100644 --- a/src/nanopycodeagent/event_journal.py +++ b/src/nanopycodeagent/event_journal.py @@ -299,6 +299,15 @@ def _validate_native_payload(event_type: str, payload: JsonObject) -> None: raise ValueError(f"{event_type}.duration_ms must be non-negative") if event_type == "run.started": + # Optional so journals written before configurable budgets still replay. + if "max_tokens" in payload: + max_tokens = payload["max_tokens"] + if ( + not isinstance(max_tokens, int) + or isinstance(max_tokens, bool) + or max_tokens < 1 + ): + raise ValueError("run.started.max_tokens must be a positive integer") if payload["mode"] not in {"interactive", "headless"}: raise ValueError("run.started.mode must be interactive or headless") _require_string(payload, "model", event_type) diff --git a/src/nanopycodeagent/settings.py b/src/nanopycodeagent/settings.py index 167a26e..76a2dea 100644 --- a/src/nanopycodeagent/settings.py +++ b/src/nanopycodeagent/settings.py @@ -10,6 +10,26 @@ from pathlib import Path SETTINGS_PATH = Path.home() / ".nanoPyCodeAgent" / "settings.json" +DEFAULT_MAX_TOKENS = 32768 + + +def resolve_max_tokens(override: int | None = None) -> int: + """Resolve the per-reply limit: explicit value, environment, file, default.""" + if override is not None: + if isinstance(override, bool) or not isinstance(override, int) or override < 1: + raise ValueError("--max-tokens must be a positive integer") + return override + load_settings_env() + value = os.environ.get("ANTHROPIC_MAX_TOKENS") + if value is None: + return DEFAULT_MAX_TOKENS + try: + limit = int(value) + except ValueError: + raise ValueError("ANTHROPIC_MAX_TOKENS must be a positive integer") from None + if limit < 1: + raise ValueError("ANTHROPIC_MAX_TOKENS must be a positive integer") + return limit def load_settings_env(path: Path | None = None) -> None: diff --git a/tests/conftest.py b/tests/conftest.py index 8ffeafa..435ae64 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,7 +6,10 @@ from nanopycodeagent import settings -_MANAGED_ENV = ("ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "ANTHROPIC_MODEL") +_MANAGED_ENV = ( + "ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "ANTHROPIC_MODEL", + "ANTHROPIC_MAX_TOKENS", +) @pytest.fixture(autouse=True) diff --git a/tests/test_generation_budget.py b/tests/test_generation_budget.py new file mode 100644 index 0000000..415a253 --- /dev/null +++ b/tests/test_generation_budget.py @@ -0,0 +1,117 @@ +"""Generation limits must reach every request and remain observable.""" + +import io +import json +from types import SimpleNamespace + +import pytest + +from nanopycodeagent import agent, cli, settings +from nanopycodeagent.event_journal import EventJournal, NativeEvent + +from helpers import ( + FakeClient, + FakeMessages, + FakeStream, + patch_client, + patch_client_and_input, + read_tool_use_block, + text_block, + write_settings, +) + + +class TtyStdin(io.StringIO): + def isatty(self): + return True + + +@pytest.mark.parametrize("interactive", [False, True]) +@pytest.mark.parametrize("file_value,env_value,cli_value,expected", [ + (None, None, None, 32768), + ("16384", None, None, 16384), + ("8192", "65536", None, 65536), + ("invalid", "16384", None, 16384), + ("invalid", "invalid", "1024", 1024), + (None, None, "1", 1), +]) +def test_budget_precedence_reaches_every_request_and_execution_record( + monkeypatch, tmp_path, capsys, interactive, file_value, env_value, cli_value, expected +): + if file_value is not None: + write_settings(settings.SETTINGS_PATH, {"ANTHROPIC_MAX_TOKENS": file_value}) + if env_value is not None: + monkeypatch.setenv("ANTHROPIC_MAX_TOKENS", env_value) + target = tmp_path / "input.txt" + target.write_text("hello") + usage = SimpleNamespace(input_tokens=10, output_tokens=1) + messages = FakeMessages([ + FakeStream([read_tool_use_block("read-1", path=str(target))], + stop_reason="tool_use", usage=usage), + FakeStream([text_block("done")], usage=usage), + ]) + patch_client_and_input(monkeypatch, client=FakeClient(messages), inputs=["read it", "/exit"]) + trajectory_path = tmp_path / "trajectory.json" + if interactive: + monkeypatch.setattr(cli.sys, "stdin", TtyStdin()) + args = [] + else: + args = ["-p", "read it", "--trajectory", str(trajectory_path)] + if cli_value is not None: + args += ["--max-tokens", cli_value] + + assert cli.main(args) == 0 + + assert [request["max_tokens"] for request in messages.kwargs] == [expected, expected] + captured = capsys.readouterr() + assert f"max tokens {expected}" in (captured.out if interactive else captured.err) + journal_path, = (tmp_path / "journals").glob("*.jsonl") + entries = EventJournal.replay(journal_path) + assert entries[0].payload["max_tokens"] == expected + assert entries[-1].payload["outcome"] == "completed" + if not interactive: + trajectory = json.loads(trajectory_path.read_text()) + assert trajectory["agent"]["extra"]["max_tokens"] == expected + assert trajectory["final_metrics"]["total_completion_tokens"] == 2 + + +@pytest.mark.parametrize("source", ["cli", "env", "file"]) +@pytest.mark.parametrize("value", ["0", "-1", "3.5", "invalid"]) +def test_invalid_budget_fails_before_starting_a_run(monkeypatch, tmp_path, capsys, source, value): + args = ["-p", "do not run"] + if source == "cli": + args += ["--max-tokens", value] + elif source == "env": + monkeypatch.setenv("ANTHROPIC_MAX_TOKENS", value) + else: + write_settings(settings.SETTINGS_PATH, {"ANTHROPIC_MAX_TOKENS": value}) + messages = FakeMessages([]) + patch_client(monkeypatch, FakeClient(messages)) + + with pytest.raises(SystemExit) as excinfo: + cli.main(args) + + assert excinfo.value.code == cli.EXIT_USAGE + diagnostic = "--max-tokens" if source == "cli" else "ANTHROPIC_MAX_TOKENS" + assert diagnostic in capsys.readouterr().err + assert messages.calls == [] + assert not (tmp_path / "journals").exists() + + +def test_direct_headless_entry_point_honors_environment(monkeypatch): + monkeypatch.setenv("ANTHROPIC_MAX_TOKENS", "16384") + messages = FakeMessages([[text_block("done")]]) + patch_client(monkeypatch, FakeClient(messages)) + assert agent.run_headless("say hi") == 0 + assert messages.kwargs[0]["max_tokens"] == 16384 + + +@pytest.mark.parametrize("value", [0, -1, True, 1.5, "32768", None]) +def test_journal_rejects_invalid_recorded_budgets(value): + with pytest.raises(ValueError, match="run.started.max_tokens"): + NativeEvent("run.started", { + "mode": "headless", "model": "test", "max_turns": 50, + "max_tokens": value, + "producer": {"name": "nanoPyCodeAgent", "version": "test"}, + "source_timestamp": None, + }) diff --git a/tests/test_truncation.py b/tests/test_truncation.py index f8120a2..4da9f48 100644 --- a/tests/test_truncation.py +++ b/tests/test_truncation.py @@ -29,18 +29,19 @@ def _journal_entries(): @pytest.mark.parametrize("max_turns", [1, 5]) +@pytest.mark.parametrize("max_tokens", [8192, 32768]) @pytest.mark.parametrize("content", [ [text_block("Partial answer")], [ThinkingBlock(type="thinking", thinking="Still analyzing", signature="")], [], ]) def test_truncation_stops_with_usage_cost_and_distinct_terminal( - monkeypatch, tmp_path, capsys, content, max_turns + monkeypatch, tmp_path, capsys, content, max_turns, max_tokens ): reply = FakeStream( content, stop_reason="max_tokens", - usage=SimpleNamespace(input_tokens=10, output_tokens=8192), + usage=SimpleNamespace(input_tokens=10, output_tokens=max_tokens), response_headers={"x-generation-id": "gen-truncated"}, ) messages = FakeMessages([reply]) @@ -60,11 +61,12 @@ def resolve(base_url, generation_id, credential, **kwargs): trajectory_path = tmp_path / "trajectory.json" assert cli.main([ "-p", "fix it", "--max-turns", str(max_turns), + "--max-tokens", str(max_tokens), "--trajectory", str(trajectory_path), ]) == 0 assert len(messages.calls) == 1 - assert messages.kwargs[0]["max_tokens"] == 8192 + assert messages.kwargs[0]["max_tokens"] == max_tokens captured = capsys.readouterr() assert captured.out == ("Partial answer\n" if content and content[0].type == "text" else "") assert "response truncated" in captured.err @@ -78,7 +80,7 @@ def resolve(base_url, generation_id, credential, **kwargs): assert entries[-1].payload["outcome"] == "response_truncated" completed = next(entry for entry in entries if entry.type == "model.completed") assert completed.payload["stop_reason"] == "max_tokens" - assert completed.payload["usage"]["output_tokens"] == 8192 + assert completed.payload["usage"]["output_tokens"] == max_tokens assert completed.payload["content"] == agent._native_content_blocks(content) assert not any(entry.type.startswith("tool.") for entry in entries) @@ -87,7 +89,8 @@ def resolve(base_url, generation_id, credential, **kwargs): assert trajectory["extra"]["terminal"]["outcome"] == "response_truncated" assert trajectory["steps"][1]["extra"]["stop_reason"] == "max_tokens" assert trajectory["final_metrics"]["total_prompt_tokens"] == 10 - assert trajectory["final_metrics"]["total_completion_tokens"] == 8192 + assert trajectory["agent"]["extra"]["max_tokens"] == max_tokens + assert trajectory["final_metrics"]["total_completion_tokens"] == max_tokens assert trajectory["final_metrics"]["total_cost_usd"] == 0.01 @@ -162,10 +165,11 @@ def respond(request): ) as client: patch_client(monkeypatch, client) assert agent.run_headless( - "write a file", trajectory_path=tmp_path / "trajectory.json" + "write a file", max_tokens=65536, trajectory_path=tmp_path / "trajectory.json" ) == 0 assert len(requests) == 1 + assert json.loads(requests[0].content)["max_tokens"] == 65536 assert "response truncated" in capsys.readouterr().err entries = _journal_entries() assert entries[-1].payload["outcome"] == "response_truncated"