fix: close path-traversal gap and harness scoring nits from full code review

CLAUDE.md

@@ -34,7 +34,6 @@ coder_eval/
 ├── argv_match.py                  # Structured argv matcher. STDLIB-ONLY (CE057): this file is copied into the recorder dir as a SIDECAR beside every response-serving shim, which imports it as a sibling, so `cli_called` and a `record_cli` response rule dispatch on ONE semantic
 ├── telemetry.py                   # App Insights / OpenTelemetry emission (CoderEval.Task.End et al.)
 ├── isolation/                     # driver: docker — docker_runner.py builds, runs and reaps one container per task
-├── optimize/                      # Prompt/config optimization helpers
 ├── utils.py                       # Version info helpers
 │
 ├── agents/

src/coder_eval/criteria/llm_judge.py

@@ -263,6 +263,11 @@ async def _invoke_tool_channel(
             # Handled by the unconfigured-arm guard in _check_impl_async before
             # dispatch; defensive only.
             return None, "llm_judge: no usable API route", "(no route)", None
+        case _:
+            # ApiRoute covers exactly Bedrock/Direct/LiteLLM above; this arm is
+            # unreachable but makes the match exhaustive (CodeQL: py/uninitialized-local-variable
+            # on verdict/err/response_usage below) — mirrors models/routing.py::resolve_route.
+            raise AssertionError(f"unhandled ApiRoute: {route!r}")
 
     if verdict is not None:
         return verdict, None, verdict.model_dump_json(), response_usage

src/coder_eval/orchestration/batch.py

@@ -21,14 +21,15 @@
     AgentKind,
     EvaluationResult,
     FinalStatus,
+    LLMJudgeCriterion,
     PreservationMode,
     ResolvedTask,
     RunSummary,
     SkippedTask,
     TaskDefinition,
     TaskResult,
 )
-from ..path_utils import TASK_JSON_FILENAME, format_task_log_id
+from ..path_utils import TASK_JSON_FILENAME, format_task_log_id, is_within
 from ..pricing import unpriced_models
 from ..reports_experiment import eval_result_to_task_dict
 from ..streaming.callbacks import StreamCallback
@@ -49,7 +50,7 @@ def _run_models(resolved_tasks: list[ResolvedTask]) -> Iterator[str | None]:
     for rt in resolved_tasks:
         yield rt.task.agent.model if rt.task.agent else None
         for criterion in rt.task.success_criteria:
-            yield getattr(criterion, "model", None)
+            yield criterion.model if isinstance(criterion, LLMJudgeCriterion) else None
 
 
 def check_pricing_coverage(resolved_tasks: list[ResolvedTask]) -> list[str]:
@@ -452,7 +453,11 @@ def clear_rerun_artifacts(to_run: list[ResolvedTask]) -> int:
     cleared = 0
     for rt in to_run:
         artifacts = rt.run_dir / "artifacts" / rt.task.task_id
-        if artifacts.exists():
+        # task_id is an unvalidated string (dataset rows are "<suite>/<row>"), so
+        # a crafted value like "../../../home/victim" joins to a real directory
+        # exists() happily confirms — mirror regrade.py's containment check
+        # before ever deleting anything derived from it.
+        if artifacts.exists() and is_within(artifacts, rt.run_dir):
             shutil.rmtree(artifacts, ignore_errors=True)
             cleared += 1
     return cleared

src/coder_eval/orchestration/regrade.py

@@ -41,6 +41,7 @@
     GRADE_LOG_FILENAME,
     PRE_GRADE_JSON_FILENAME,
     TASK_JSON_FILENAME,
+    is_within,
     write_text_atomic,
 )
 from coder_eval.sandbox import Sandbox
@@ -498,7 +499,7 @@ def default_workspace(run_dir: Path, prior: EvaluationResult) -> Path:
     def _contained(candidate: Path, description: str) -> Path:
         # One chokepoint, one root. `run_dir` is the operator-supplied path; a
         # candidate is only ever derived from the untrusted record.
-        if not _is_within(candidate, run_dir):
+        if not is_within(candidate, run_dir):
             raise RegradeError(
                 f"{description} resolves outside the run directory ({run_dir}). "
                 + "Pass --workspace explicitly to grade a directory outside the run."
@@ -550,15 +551,6 @@ def _contained(candidate: Path, description: str) -> Path:
     )
 
 
-def _is_within(candidate: Path, root: Path) -> bool:
-    """True when ``candidate`` resolves inside ``root``."""
-    try:
-        candidate.resolve().relative_to(root.resolve())
-    except ValueError:
-        return False
-    return True
-
-
 def verify_reference_unchanged(prior: EvaluationResult, task: TaskDefinition, task_file: Path | None) -> None:
     """Refuse to grade when the reference tree changed since the run.
 

src/coder_eval/path_utils.py

@@ -158,6 +158,21 @@ def rmtree_restrictive(root: Path) -> None:
         logger.warning("Directory %s could not be fully removed", root)
 
 
+def is_within(candidate: Path, root: Path) -> bool:
+    """True when ``candidate`` resolves inside ``root``.
+
+    Shared containment check for any path joined from an untrusted, task- or
+    record-authored field (a ``task_id``, a recorded ``sandbox_path``) before
+    it is read, written, or deleted — ``"../../etc"`` joins to a real path
+    that ``is_dir()``/``exists()`` happily confirms.
+    """
+    try:
+        candidate.resolve().relative_to(root.resolve())
+    except ValueError:
+        return False
+    return True
+
+
 def ignore_patterns_and_symlinks(patterns: list[str]) -> Callable[[str, list[str]], set[str]]:
     """``copytree`` ``ignore`` callable that drops pattern matches AND every symlink.
 

src/coder_eval/reports.py

@@ -850,7 +850,13 @@ def _attach_row_accounting(agg: CriterionAggregate, rows_total: int, rows_aggreg
     """
     excluded = rows_total - rows_aggregated
     metrics = dict(agg.metrics)
-    metrics["completion_rate"] = (rows_aggregated / rows_total) if rows_total else 0.0
+    # Omit rather than publish 0.0 when there is no denominator: a suite_thresholds
+    # gate on this metric then fails closed with actual_value=None (see
+    # _evaluate_thresholds), distinguishable from "measured and completely failed" —
+    # the same nondescript-zero shape pass_rate/error_share guard against via
+    # nothing_was_measured.
+    if rows_total:
+        metrics["completion_rate"] = rows_aggregated / rows_total
     return agg.model_copy(update={"rows_total": rows_total, "rows_excluded": excluded, "metrics": metrics})
 
 

tests/test_preservation_mode.py

@@ -141,6 +141,46 @@ def _rt(task_id: str) -> ResolvedTask:
     assert not (stale.run_dir / "artifacts" / "stale").exists()
 
 
+def test_clear_rerun_artifacts_refuses_path_traversal(tmp_path):
+    """A crafted task_id ('../../escape') must not delete outside the run dir.
+
+    task_id is an unvalidated string (dataset rows are "<suite>/<row>"), so
+    "../../../../victim" joins to a real directory `exists()` happily confirms.
+    Mirrors the containment check orchestration/regrade.py applies to the same
+    recorded-task_id shape.
+    """
+    from coder_eval.models import ResolvedTask, TaskDefinition
+    from coder_eval.orchestration.batch import clear_rerun_artifacts
+
+    task_id = "../../escaped"
+    task = TaskDefinition(
+        task_id=task_id,
+        description="d",
+        initial_prompt="p",
+        agent={"type": "claude-code"},
+        sandbox={"driver": "docker"},
+        success_criteria=[{"type": "file_exists", "path": "x.txt", "description": "x"}],
+    )
+    run_dir = tmp_path / "run" / "default" / "victim" / "00"
+    run_dir.mkdir(parents=True)
+    victim = tmp_path / "run" / "default" / "escaped"
+    victim.mkdir(parents=True)
+    (victim / "do-not-delete.txt").write_text("real data outside the run dir")
+
+    rt = ResolvedTask(
+        task=task,
+        task_file=tmp_path / "t.yaml",
+        run_dir=run_dir,
+        variant_id="default",
+        original_task_id=task_id,
+    )
+
+    cleared = clear_rerun_artifacts([rt])
+
+    assert cleared == 0
+    assert (victim / "do-not-delete.txt").exists()
+
+
 @pytest.mark.asyncio
 async def test_run_batch_dispatches_resolved_mode_for_tempdir(tmp_path):
     """run_batch must hand the driver-derived mode (tempdir→MOVE_ON_WRITE) to the Orchestrator."""

tests/test_suite_rollup.py

@@ -589,7 +589,10 @@ def test_attach_row_accounting_zero_total_no_div_error(self) -> None:
         out = _attach_row_accounting(agg, rows_total=0, rows_aggregated=0)
         assert out.rows_total == 0
         assert out.rows_excluded == 0
-        assert out.metrics["completion_rate"] == 0.0
+        # No denominator: completion_rate is omitted rather than published as
+        # 0.0 (indistinguishable from "measured and completely failed") — a
+        # suite_thresholds gate on it then fails closed via actual_value=None.
+        assert "completion_rate" not in out.metrics
 
     def test_render_includes_denominator_line_when_excluded(self) -> None:
         agg = CriterionAggregate(