Add SCML OneShot CC:Ladder

.claude/skills/create-arena-ladder/SKILL.md

@@ -104,9 +104,14 @@ Local validation is necessary but NOT sufficient — a local shim skips Docker,
    `GITHUB_TOKEN=$(gh auth token) uv run codeclash ladder make configs/ablations/ladder/make_<a>.yaml --workers N`
    → logs land under `logs/ladder/<A>/`. Fast arenas run on a laptop; big ones (e.g. SCML's
    ~1275 pairs) want an AWS box under `tmux`/`nohup`.
-4. Rank: `python -m codeclash.analysis.metrics.elo -d logs/ladder/<A> --output-dir assets/<a>_elo`
+4. Rank: `uv run python -m codeclash.analysis.metrics.elo -d logs/ladder/<A> --include-round-0 --output-dir assets/<a>_elo`
    → prints the Bradley-Terry/Elo order (weakest → strongest); that ordering IS the ladder.
-   Tip: run a cheap low-`sims` pilot first and eyeball that baselines sit near the bottom.
+   - **`--include-round-0` is REQUIRED for ladder construction.** `ladder make` uses
+     `tournament.rounds: 0`, so round 0 IS the match; without the flag every tournament is
+     dropped (round 0 is normally the excluded identical-codebases baseline) and the fit
+     crashes on an empty matrix. Do NOT pass it for normal multi-round PvP/climbing Elo.
+   - Use `uv run python`, not bare `python` — the analysis deps (matplotlib) live in the uv venv.
+   - Tip: run a cheap low-`sims` pilot first and eyeball that baselines sit near the bottom.
 
 ## Phase 5 — Assemble the ranked configs
 

codeclash/analysis/metrics/elo.py

@@ -40,6 +40,7 @@ def __init__(
         score_type: SCORING_TYPES = "per_round_tertiary",
         max_round: int = 15,
         only_specific_round: bool = False,
+        include_round_0: bool = False,
     ):
         """This class builds a win matrix from a log directory, it doesn't fit anything yet.
         It also adds a "ALL" game to the win matrix, which is the sum of all games.
@@ -62,6 +63,9 @@ def __init__(
 
         The `max_round` parameter controls the maximum number of rounds to include in the score calculation (default: 15).
         The `only_specific_round` parameter controls whether to only include the specific round (True) or all rounds up to max_round (False).
+        The `include_round_0` parameter controls whether round 0 is counted. In normal PvP/climbing
+        tournaments round 0 is the identical-codebases baseline and is excluded. For ladder
+        construction (`ladder make`, `tournament.rounds: 0`) round 0 IS the match, so set this True.
         """
         self.win_matrix: dict[str, dict[tuple[str, str], list[float]]] = defaultdict(
             lambda: defaultdict(lambda: [0.0, 0.0])
@@ -71,6 +75,7 @@ def __init__(
         self.score_type = score_type
         self.max_round = max_round
         self.only_specific_round = only_specific_round
+        self.include_round_0 = include_round_0
         self._samples: dict[str, dict[tuple[str, str], list[tuple[float, float]]]] = defaultdict(
             lambda: defaultdict(list)
         )
@@ -154,13 +159,19 @@ def _process_tournament(self, metadata_path: Path) -> None:
             return
 
         player_names = [p["name"] for p in players]
-        models = [p["config"]["model"]["model_name"].strip("@") for p in players]
+        models = []
+        for p in players:
+            try:
+                models.append(p["config"]["model"]["model_name"].strip("@"))
+            except KeyError:
+                # Ladder bots have no model config; identify by branch (flatten "/" to keep years distinct).
+                models.append(p["name"].removeprefix("human/").replace("/", "__"))
 
         # Aggregate scores for each round
         p1_round_scores = []
         p2_round_scores = []
         for idx, stats in metadata["round_stats"].items():
-            if idx == "0":
+            if idx == "0" and not self.include_round_0:
                 continue
 
             round_num = int(idx)
@@ -1547,6 +1558,12 @@ def write_latex_table_plain(results: dict[str, dict], output_dir: Path) -> None:
     parser = argparse.ArgumentParser(description="Build win matrix and fit Bradley-Terry model")
     parser.add_argument("-d", "--log_dir", type=Path, default=LOCAL_LOG_DIR)
     parser.add_argument("--print-matrix", action="store_true", help="Print win matrix")
+    parser.add_argument(
+        "--include-round-0",
+        action="store_true",
+        help="Count round 0 (normally the excluded identical-codebases baseline). REQUIRED for "
+        "ladder construction (`ladder make` uses tournament.rounds: 0, so round 0 IS the match).",
+    )
     parser.add_argument(
         "-s",
         "--score-type",
@@ -1578,7 +1595,9 @@ def write_latex_table_plain(results: dict[str, dict], output_dir: Path) -> None:
     args = parser.parse_args()
 
     builder = ScoreMatrixBuilder(
-        all_games_normalization_scheme=args.all_normalization_scheme, score_type=args.score_type
+        all_games_normalization_scheme=args.all_normalization_scheme,
+        score_type=args.score_type,
+        include_round_0=args.include_round_0,
     )
     builder.build(args.log_dir)
 
@@ -1632,22 +1651,26 @@ def write_latex_table_plain(results: dict[str, dict], output_dir: Path) -> None:
             ).run()
         write_bootstrap_metrics_table(bootstrap_results, args.output_dir, game="ALL")
 
-    logger.info("Running EloVsMaxRounds analysis")
-    EloVsMaxRounds(
-        log_dir=args.log_dir,
-        max_rounds=15,
-        all_games_normalization_scheme=args.all_normalization_scheme,
-        score_type=args.score_type,
-        regularization=args.regularization,
-        output_dir=args.output_dir,
-    ).run()
-
-    logger.info("Running EloOnlyAtRound analysis")
-    EloOnlyAtRound(
-        log_dir=args.log_dir,
-        max_rounds=15,
-        all_games_normalization_scheme=args.all_normalization_scheme,
-        score_type=args.score_type,
-        regularization=args.regularization,
-        output_dir=args.output_dir,
-    ).run()
+    # Max-round analyses are multi-round-only; skip them for single-round ladder round-robins.
+    if not args.include_round_0:
+        logger.info("Running EloVsMaxRounds analysis")
+        EloVsMaxRounds(
+            log_dir=args.log_dir,
+            max_rounds=15,
+            all_games_normalization_scheme=args.all_normalization_scheme,
+            score_type=args.score_type,
+            regularization=args.regularization,
+            output_dir=args.output_dir,
+        ).run()
+
+        logger.info("Running EloOnlyAtRound analysis")
+        EloOnlyAtRound(
+            log_dir=args.log_dir,
+            max_rounds=15,
+            all_games_normalization_scheme=args.all_normalization_scheme,
+            score_type=args.score_type,
+            regularization=args.regularization,
+            output_dir=args.output_dir,
+        ).run()
+    else:
+        logger.info("Skipping EloVsMaxRounds / EloOnlyAtRound (ladder mode: single round-0 round-robin)")

codeclash/arenas/scml/SCML.Dockerfile

@@ -4,6 +4,13 @@ ENV DEBIAN_FRONTEND=noninteractive \
     PYTHONDONTWRITEBYTECODE=1 \
     PIP_NO_CACHE_DIR=1
 
+# Pin numpy/BLAS to a single math thread per container. Each SCML simulation is
+# single-threaded compute (measured: pinned solo == unpinned solo)
+ENV OMP_NUM_THREADS=1 \
+    OPENBLAS_NUM_THREADS=1 \
+    MKL_NUM_THREADS=1 \
+    NUMEXPR_NUM_THREADS=1
+
 RUN apt-get update \
  && apt-get install -y --no-install-recommends \
     ca-certificates git build-essential jq \
@@ -12,12 +19,11 @@ RUN apt-get update \
 RUN python -m pip install --upgrade pip \
  && python -m pip install scml==0.8.2
 
+# Clone the arena repo so `origin` is set for branch_init / push (matches the other
+# arenas). Default branch holds the runtime; human/* branches overlay scml_agent.py.
+RUN git clone https://github.com/CodeClash-ai/SCML.git /workspace \
+    && cd /workspace \
+    && git remote set-url origin https://github.com/CodeClash-ai/SCML.git \
+    && git config user.email "player@codeclash.com" \
+    && git config user.name "Player"
 WORKDIR /workspace
-
-COPY codeclash/arenas/scml/runtime/ /workspace/
-
-RUN git init \
- && git config user.email "player@codeclash.com" \
- && git config user.name "Player" \
- && git add . \
- && git commit -m "Initial SCML workspace"

configs/ladder/make_scml.yaml

@@ -0,0 +1,129 @@
+# Round-robin over the SCML OneShot human bots (ported from yasserfarouk/scml-agents +
+# built-in baselines), used to RANK them via Elo/Bradley-Terry -> see scml.yaml for the ladder.
+# rounds:0 = dummy players just play the baseline SCML2024OneShot game (no code edits).
+# 51 bots -> 51*50/2 = 1275 pairwise tournaments. Bots live on CodeClash-ai/SCML; Docker required.
+#
+# COST (measured ~4.8s/sim + ~3s overhead per pair; pairs are single-threaded, so wall =
+# core-hours / workers): at sims_per_round=400 a pair takes ~32 min -> ~681 core-hours total.
+#   * 32-core box (--workers 30): ~23 h    * 64 vCPU (-w 62): ~11 h    * 96 vCPU (-w 90): ~7.5 h
+# Keep --workers <= (cores - 2): the decide() call has a 3s timeout that CPU oversubscription
+# can trip. Resumable: reruns skip pairs already logged, so it's safe to stop/resume.
+#
+# Full run (250-400 sims = publication-quality Elo; 400 chosen for SCML's high per-sim variance):
+#   uv run codeclash ladder make configs/ladder/make_scml.yaml --workers 30
+# Cheap SANITY pilot first (~1 h @ 32 cores): temporarily set sims_per_round: 30, run, rank, and
+# eyeball the ordering (baselines near the bottom) before committing to the full 400-sim pass.
+tournament:
+  rounds: 0
+game:
+  name: SCML
+  sims_per_round: 400
+  # sims_per_round: 30   # <- pilot: uncomment (and comment out 400) for a ~1h sanity-check ranking
+  args:
+    n_steps: 10
+    n_lines: 2
+players:
+- agent: dummy
+  branch_init: human/scml-baselines/greedy
+- agent: dummy
+  branch_init: human/scml-baselines/nice
+- agent: dummy
+  branch_init: human/scml-baselines/random
+- agent: dummy
+  branch_init: human/scml2021/staghunter
+- agent: dummy
+  branch_init: human/scml2021/team_50
+- agent: dummy
+  branch_init: human/scml2021/team_51
+- agent: dummy
+  branch_init: human/scml2021/team_54
+- agent: dummy
+  branch_init: human/scml2021/team_55
+- agent: dummy
+  branch_init: human/scml2021/team_61
+- agent: dummy
+  branch_init: human/scml2021/team_62
+- agent: dummy
+  branch_init: human/scml2021/team_72
+- agent: dummy
+  branch_init: human/scml2021/team_73
+- agent: dummy
+  branch_init: human/scml2021/team_86
+- agent: dummy
+  branch_init: human/scml2021/team_90
+- agent: dummy
+  branch_init: human/scml2021/team_corleone
+- agent: dummy
+  branch_init: human/scml2022/team_102
+- agent: dummy
+  branch_init: human/scml2022/team_103
+- agent: dummy
+  branch_init: human/scml2022/team_105
+- agent: dummy
+  branch_init: human/scml2022/team_106
+- agent: dummy
+  branch_init: human/scml2022/team_107
+- agent: dummy
+  branch_init: human/scml2022/team_123
+- agent: dummy
+  branch_init: human/scml2022/team_124
+- agent: dummy
+  branch_init: human/scml2022/team_126
+- agent: dummy
+  branch_init: human/scml2022/team_131
+- agent: dummy
+  branch_init: human/scml2022/team_134
+- agent: dummy
+  branch_init: human/scml2022/team_94
+- agent: dummy
+  branch_init: human/scml2022/team_96
+- agent: dummy
+  branch_init: human/scml2023/team_102
+- agent: dummy
+  branch_init: human/scml2023/team_123
+- agent: dummy
+  branch_init: human/scml2023/team_126
+- agent: dummy
+  branch_init: human/scml2023/team_127
+- agent: dummy
+  branch_init: human/scml2023/team_134
+- agent: dummy
+  branch_init: human/scml2023/team_143
+- agent: dummy
+  branch_init: human/scml2023/team_144
+- agent: dummy
+  branch_init: human/scml2023/team_145
+- agent: dummy
+  branch_init: human/scml2023/team_148
+- agent: dummy
+  branch_init: human/scml2023/team_149
+- agent: dummy
+  branch_init: human/scml2023/team_151
+- agent: dummy
+  branch_init: human/scml2023/team_poli_usp
+- agent: dummy
+  branch_init: human/scml2024/coyoteteam
+- agent: dummy
+  branch_init: human/scml2024/ozug4
+- agent: dummy
+  branch_init: human/scml2024/team_144
+- agent: dummy
+  branch_init: human/scml2024/team_164
+- agent: dummy
+  branch_init: human/scml2024/team_171
+- agent: dummy
+  branch_init: human/scml2024/team_172
+- agent: dummy
+  branch_init: human/scml2024/team_174
+- agent: dummy
+  branch_init: human/scml2024/team_193
+- agent: dummy
+  branch_init: human/scml2024/team_238
+- agent: dummy
+  branch_init: human/scml2024/team_abc
+- agent: dummy
+  branch_init: human/scml2024/team_miyajima
+- agent: dummy
+  branch_init: human/scml2024/teamyuzuru
+prompts:
+  game_description: SCML OneShot ladder