diff --git a/.claude/skills/create-arena-ladder/SKILL.md b/.claude/skills/create-arena-ladder/SKILL.md index a2c30350..5f3a15e4 100644 --- a/.claude/skills/create-arena-ladder/SKILL.md +++ b/.claude/skills/create-arena-ladder/SKILL.md @@ -104,9 +104,14 @@ Local validation is necessary but NOT sufficient — a local shim skips Docker, `GITHUB_TOKEN=$(gh auth token) uv run codeclash ladder make configs/ablations/ladder/make_.yaml --workers N` → logs land under `logs/ladder//`. Fast arenas run on a laptop; big ones (e.g. SCML's ~1275 pairs) want an AWS box under `tmux`/`nohup`. -4. Rank: `python -m codeclash.analysis.metrics.elo -d logs/ladder/ --output-dir assets/_elo` +4. Rank: `uv run python -m codeclash.analysis.metrics.elo -d logs/ladder/ --include-round-0 --output-dir assets/_elo` → prints the Bradley-Terry/Elo order (weakest → strongest); that ordering IS the ladder. - Tip: run a cheap low-`sims` pilot first and eyeball that baselines sit near the bottom. + - **`--include-round-0` is REQUIRED for ladder construction.** `ladder make` uses + `tournament.rounds: 0`, so round 0 IS the match; without the flag every tournament is + dropped (round 0 is normally the excluded identical-codebases baseline) and the fit + crashes on an empty matrix. Do NOT pass it for normal multi-round PvP/climbing Elo. + - Use `uv run python`, not bare `python` — the analysis deps (matplotlib) live in the uv venv. + - Tip: run a cheap low-`sims` pilot first and eyeball that baselines sit near the bottom. ## Phase 5 — Assemble the ranked configs diff --git a/codeclash/analysis/metrics/elo.py b/codeclash/analysis/metrics/elo.py index 929d3154..5d7a9ef6 100644 --- a/codeclash/analysis/metrics/elo.py +++ b/codeclash/analysis/metrics/elo.py @@ -40,6 +40,7 @@ def __init__( score_type: SCORING_TYPES = "per_round_tertiary", max_round: int = 15, only_specific_round: bool = False, + include_round_0: bool = False, ): """This class builds a win matrix from a log directory, it doesn't fit anything yet. It also adds a "ALL" game to the win matrix, which is the sum of all games. @@ -62,6 +63,9 @@ def __init__( The `max_round` parameter controls the maximum number of rounds to include in the score calculation (default: 15). The `only_specific_round` parameter controls whether to only include the specific round (True) or all rounds up to max_round (False). + The `include_round_0` parameter controls whether round 0 is counted. In normal PvP/climbing + tournaments round 0 is the identical-codebases baseline and is excluded. For ladder + construction (`ladder make`, `tournament.rounds: 0`) round 0 IS the match, so set this True. """ self.win_matrix: dict[str, dict[tuple[str, str], list[float]]] = defaultdict( lambda: defaultdict(lambda: [0.0, 0.0]) @@ -71,6 +75,7 @@ def __init__( self.score_type = score_type self.max_round = max_round self.only_specific_round = only_specific_round + self.include_round_0 = include_round_0 self._samples: dict[str, dict[tuple[str, str], list[tuple[float, float]]]] = defaultdict( lambda: defaultdict(list) ) @@ -154,13 +159,19 @@ def _process_tournament(self, metadata_path: Path) -> None: return player_names = [p["name"] for p in players] - models = [p["config"]["model"]["model_name"].strip("@") for p in players] + models = [] + for p in players: + try: + models.append(p["config"]["model"]["model_name"].strip("@")) + except KeyError: + # Ladder bots have no model config; identify by branch (flatten "/" to keep years distinct). + models.append(p["name"].removeprefix("human/").replace("/", "__")) # Aggregate scores for each round p1_round_scores = [] p2_round_scores = [] for idx, stats in metadata["round_stats"].items(): - if idx == "0": + if idx == "0" and not self.include_round_0: continue round_num = int(idx) @@ -1547,6 +1558,12 @@ def write_latex_table_plain(results: dict[str, dict], output_dir: Path) -> None: parser = argparse.ArgumentParser(description="Build win matrix and fit Bradley-Terry model") parser.add_argument("-d", "--log_dir", type=Path, default=LOCAL_LOG_DIR) parser.add_argument("--print-matrix", action="store_true", help="Print win matrix") + parser.add_argument( + "--include-round-0", + action="store_true", + help="Count round 0 (normally the excluded identical-codebases baseline). REQUIRED for " + "ladder construction (`ladder make` uses tournament.rounds: 0, so round 0 IS the match).", + ) parser.add_argument( "-s", "--score-type", @@ -1578,7 +1595,9 @@ def write_latex_table_plain(results: dict[str, dict], output_dir: Path) -> None: args = parser.parse_args() builder = ScoreMatrixBuilder( - all_games_normalization_scheme=args.all_normalization_scheme, score_type=args.score_type + all_games_normalization_scheme=args.all_normalization_scheme, + score_type=args.score_type, + include_round_0=args.include_round_0, ) builder.build(args.log_dir) @@ -1632,22 +1651,26 @@ def write_latex_table_plain(results: dict[str, dict], output_dir: Path) -> None: ).run() write_bootstrap_metrics_table(bootstrap_results, args.output_dir, game="ALL") - logger.info("Running EloVsMaxRounds analysis") - EloVsMaxRounds( - log_dir=args.log_dir, - max_rounds=15, - all_games_normalization_scheme=args.all_normalization_scheme, - score_type=args.score_type, - regularization=args.regularization, - output_dir=args.output_dir, - ).run() - - logger.info("Running EloOnlyAtRound analysis") - EloOnlyAtRound( - log_dir=args.log_dir, - max_rounds=15, - all_games_normalization_scheme=args.all_normalization_scheme, - score_type=args.score_type, - regularization=args.regularization, - output_dir=args.output_dir, - ).run() + # Max-round analyses are multi-round-only; skip them for single-round ladder round-robins. + if not args.include_round_0: + logger.info("Running EloVsMaxRounds analysis") + EloVsMaxRounds( + log_dir=args.log_dir, + max_rounds=15, + all_games_normalization_scheme=args.all_normalization_scheme, + score_type=args.score_type, + regularization=args.regularization, + output_dir=args.output_dir, + ).run() + + logger.info("Running EloOnlyAtRound analysis") + EloOnlyAtRound( + log_dir=args.log_dir, + max_rounds=15, + all_games_normalization_scheme=args.all_normalization_scheme, + score_type=args.score_type, + regularization=args.regularization, + output_dir=args.output_dir, + ).run() + else: + logger.info("Skipping EloVsMaxRounds / EloOnlyAtRound (ladder mode: single round-0 round-robin)") diff --git a/codeclash/arenas/scml/SCML.Dockerfile b/codeclash/arenas/scml/SCML.Dockerfile index 94c14350..a378152b 100644 --- a/codeclash/arenas/scml/SCML.Dockerfile +++ b/codeclash/arenas/scml/SCML.Dockerfile @@ -4,6 +4,13 @@ ENV DEBIAN_FRONTEND=noninteractive \ PYTHONDONTWRITEBYTECODE=1 \ PIP_NO_CACHE_DIR=1 +# Pin numpy/BLAS to a single math thread per container. Each SCML simulation is +# single-threaded compute (measured: pinned solo == unpinned solo) +ENV OMP_NUM_THREADS=1 \ + OPENBLAS_NUM_THREADS=1 \ + MKL_NUM_THREADS=1 \ + NUMEXPR_NUM_THREADS=1 + RUN apt-get update \ && apt-get install -y --no-install-recommends \ ca-certificates git build-essential jq \ @@ -12,12 +19,11 @@ RUN apt-get update \ RUN python -m pip install --upgrade pip \ && python -m pip install scml==0.8.2 +# Clone the arena repo so `origin` is set for branch_init / push (matches the other +# arenas). Default branch holds the runtime; human/* branches overlay scml_agent.py. +RUN git clone https://github.com/CodeClash-ai/SCML.git /workspace \ + && cd /workspace \ + && git remote set-url origin https://github.com/CodeClash-ai/SCML.git \ + && git config user.email "player@codeclash.com" \ + && git config user.name "Player" WORKDIR /workspace - -COPY codeclash/arenas/scml/runtime/ /workspace/ - -RUN git init \ - && git config user.email "player@codeclash.com" \ - && git config user.name "Player" \ - && git add . \ - && git commit -m "Initial SCML workspace" diff --git a/configs/ladder/make_scml.yaml b/configs/ladder/make_scml.yaml new file mode 100644 index 00000000..6b242de3 --- /dev/null +++ b/configs/ladder/make_scml.yaml @@ -0,0 +1,129 @@ +# Round-robin over the SCML OneShot human bots (ported from yasserfarouk/scml-agents + +# built-in baselines), used to RANK them via Elo/Bradley-Terry -> see scml.yaml for the ladder. +# rounds:0 = dummy players just play the baseline SCML2024OneShot game (no code edits). +# 51 bots -> 51*50/2 = 1275 pairwise tournaments. Bots live on CodeClash-ai/SCML; Docker required. +# +# COST (measured ~4.8s/sim + ~3s overhead per pair; pairs are single-threaded, so wall = +# core-hours / workers): at sims_per_round=400 a pair takes ~32 min -> ~681 core-hours total. +# * 32-core box (--workers 30): ~23 h * 64 vCPU (-w 62): ~11 h * 96 vCPU (-w 90): ~7.5 h +# Keep --workers <= (cores - 2): the decide() call has a 3s timeout that CPU oversubscription +# can trip. Resumable: reruns skip pairs already logged, so it's safe to stop/resume. +# +# Full run (250-400 sims = publication-quality Elo; 400 chosen for SCML's high per-sim variance): +# uv run codeclash ladder make configs/ladder/make_scml.yaml --workers 30 +# Cheap SANITY pilot first (~1 h @ 32 cores): temporarily set sims_per_round: 30, run, rank, and +# eyeball the ordering (baselines near the bottom) before committing to the full 400-sim pass. +tournament: + rounds: 0 +game: + name: SCML + sims_per_round: 200 + timeout: 10800 # 3h per-pair game cap; 200 sims needs ~1-1.5h (default 180s kills the run) + args: + n_steps: 10 + n_lines: 2 +players: +- agent: dummy + branch_init: human/scml-baselines/greedy +- agent: dummy + branch_init: human/scml-baselines/nice +- agent: dummy + branch_init: human/scml-baselines/random +- agent: dummy + branch_init: human/scml2021/staghunter +- agent: dummy + branch_init: human/scml2021/team_50 +- agent: dummy + branch_init: human/scml2021/team_51 +- agent: dummy + branch_init: human/scml2021/team_54 +- agent: dummy + branch_init: human/scml2021/team_55 +- agent: dummy + branch_init: human/scml2021/team_61 +- agent: dummy + branch_init: human/scml2021/team_62 +- agent: dummy + branch_init: human/scml2021/team_72 +- agent: dummy + branch_init: human/scml2021/team_73 +- agent: dummy + branch_init: human/scml2021/team_86 +- agent: dummy + branch_init: human/scml2021/team_90 +- agent: dummy + branch_init: human/scml2021/team_corleone +- agent: dummy + branch_init: human/scml2022/team_102 +- agent: dummy + branch_init: human/scml2022/team_103 +- agent: dummy + branch_init: human/scml2022/team_105 +- agent: dummy + branch_init: human/scml2022/team_106 +- agent: dummy + branch_init: human/scml2022/team_107 +- agent: dummy + branch_init: human/scml2022/team_123 +- agent: dummy + branch_init: human/scml2022/team_124 +- agent: dummy + branch_init: human/scml2022/team_126 +- agent: dummy + branch_init: human/scml2022/team_131 +- agent: dummy + branch_init: human/scml2022/team_134 +- agent: dummy + branch_init: human/scml2022/team_94 +- agent: dummy + branch_init: human/scml2022/team_96 +- agent: dummy + branch_init: human/scml2023/team_102 +- agent: dummy + branch_init: human/scml2023/team_123 +- agent: dummy + branch_init: human/scml2023/team_126 +- agent: dummy + branch_init: human/scml2023/team_127 +- agent: dummy + branch_init: human/scml2023/team_134 +- agent: dummy + branch_init: human/scml2023/team_143 +- agent: dummy + branch_init: human/scml2023/team_144 +- agent: dummy + branch_init: human/scml2023/team_145 +- agent: dummy + branch_init: human/scml2023/team_148 +- agent: dummy + branch_init: human/scml2023/team_149 +- agent: dummy + branch_init: human/scml2023/team_151 +- agent: dummy + branch_init: human/scml2023/team_poli_usp +- agent: dummy + branch_init: human/scml2024/coyoteteam +- agent: dummy + branch_init: human/scml2024/ozug4 +- agent: dummy + branch_init: human/scml2024/team_144 +- agent: dummy + branch_init: human/scml2024/team_164 +- agent: dummy + branch_init: human/scml2024/team_171 +- agent: dummy + branch_init: human/scml2024/team_172 +- agent: dummy + branch_init: human/scml2024/team_174 +- agent: dummy + branch_init: human/scml2024/team_193 +- agent: dummy + branch_init: human/scml2024/team_238 +- agent: dummy + branch_init: human/scml2024/team_abc +- agent: dummy + branch_init: human/scml2024/team_miyajima +- agent: dummy + branch_init: human/scml2024/teamyuzuru +prompts: + game_description: SCML OneShot ladder diff --git a/docs/scml_game_visualization.html b/docs/scml_game_visualization.html new file mode 100644 index 00000000..5d5f52be --- /dev/null +++ b/docs/scml_game_visualization.html @@ -0,0 +1,217 @@ + + + + + +SCML OneShot — how the game works + + + +
+

SCML OneShot — how the game works

+

A supply-chain negotiation game. Each bot is a factory that negotiates to buy inputs and sell outputs; the objective is to maximize profit. CodeClash arena · scml_agent.py

+ + +
+

1 · The supply chain (a 2-level "OneShot" world)

+

The market forces goods in at the top and demand out at the bottom (violet = exogenous, non-negotiable). Everything in the middle is settled by negotiation. Every submitted policy runs a factory at both levels, and its score is averaged.

+ + + + + + + + + EXOGENOUS SUPPLY — raw material forced in + + + + + L0 factory · "seller" + must SELL what it's forced to buy + + + L1 factory · "buyer" + must BUY to meet its demand + + + + + EXOGENOUS DEMAND — finished goods forced out + + + + + + + + + NEGOTIATED CONTRACT + [ quantity, time, unit price ] + goods → + ← money + +
+ Exogenous (forced by the market) + Seller side (has supply, needs buyers) + Buyer side (has demand, needs supply) +
+
+ +
+ +
+

2 · How they compete: bilateral negotiation

+

There are no fixed prices. Two parties exchange offers over several rounds until someone accepts, rejects, or walks away. Your bot is called at each of its turns.

+ + SELLER + BUYER (you) + + + + + + propose [10, t, $9] + + + reject → counter [10, t, $6] + + + counter [10, t, $7] + + + ✓ ACCEPT → deal at $7 + +

Every submitted policy negotiates against every other in the same world, simultaneously, on both sides of the chain.

+
+ + +
+

3 · What you actually code

+

One stateless function. The trusted runtime owns the SCML agent and calls decide() at each turn, telling you the event. Return {}/None to defer to a greedy fallback.

+
# scml_agent.py
+def decide(observation):
+    ev = observation["event"]
+    awi = observation["awi"]   # prices, needs
+
+    if ev == "propose":        # my turn to offer
+        return {"offer": [q, t, price]}
+
+    if ev == "respond":        # react to their offer
+        return {"response": "accept"}
+        # or "reject" (+ counter offer), "end"
+
+    return {}                    # → greedy fallback
+

Invalid or slow returns count as policy errors (capped); an unhandled crash floors your score.

+
+
+ + +
+

4 · How the score is decided — profit

+

A round runs several worlds. Your arena score = your average SCML profit across them. Highest average wins. Example, one day of a buyer factory:

+
+ + + + + + + +
Sell finished goods (exogenous demand: 10 @ $12)+120
Buy inputs you negotiated (10 @ $7)−70
Production cost (10 units on your lines)−15
Shortfall penalty (demand you couldn't cover)−0
Disposal penalty (inputs you overbought)−0
Profit this day+35
+
+

The core tension your bot is optimizing:

+
    +
  1. Match quantity to what you can produce (n_lines) and are obligated to deliver — over- or under-buying is penalized on both ends.
  2. +
  3. Win on price — buy cheap, sell dear — but not so greedily that negotiations collapse and you're left short.
  4. +
  5. Close in time — deals must clear within the day's negotiation rounds.
  6. +
+

No replay/animation exists in this arena: worlds run headless (no_logs=True) and emit only final scores to scml_results.json.

+
+
+
+ +
+ + diff --git a/scripts/ladder/.gitignore b/scripts/ladder/.gitignore new file mode 100644 index 00000000..b715ef09 --- /dev/null +++ b/scripts/ladder/.gitignore @@ -0,0 +1,4 @@ +# Regenerated build artifacts — bot ports live on the CodeClash-ai/ branches +# (source of truth), not in this repo. Populate ports/ locally when (re)building a ladder. +ports/ +out/ diff --git a/scripts/ladder/PORTING_GUIDE.md b/scripts/ladder/PORTING_GUIDE.md new file mode 100644 index 00000000..1522b1bd --- /dev/null +++ b/scripts/ladder/PORTING_GUIDE.md @@ -0,0 +1,80 @@ +# Porting an SCML OneShot agent to the CodeClash `decide()` contract + +You are porting ONE competition agent from `yasserfarouk/scml-agents` into a single +self-contained `scml_agent.py` that defines **one function**: `decide(observation)`. +A fully-worked, validated reference port lives at +`scripts/ladder/examples/scml_agent.py` (GreedyOneShotAgent) — read it first; +mirror its structure and defensive style. + +## The runtime (how your `decide` is called) + +The trusted runtime owns a real SCML agent (base class `GreedySyncAgent`) and calls your +`decide(observation)` at each negotiation turn. `observation` is a plain dict: + +``` +observation = { + "event": "propose" | "respond", # which decision is being asked + "player": "", + "negotiator_id": "" | None, + "awi": { # agent-world-info (all plain numbers/lists) + "current_step", "n_steps", "n_lines", "max_n_lines", + "current_balance", "current_inventory", + "current_exogenous_input_quantity", "current_exogenous_input_price", + "current_exogenous_output_quantity", "current_exogenous_output_price", + "current_disposal_cost", "current_shortfall_penalty", + "my_input_product", "my_output_product", + "is_first_level", "is_middle_level", "is_last_level", + "needed_sales", "needed_supplies", # <-- USE THESE for "how much do I still need" + "my_suppliers", "my_consumers", + }, + "nmi": { # negotiation-mechanism-info + "annotation": { "product": , "buyer": , "seller": , ... }, + "issues": [ {"name","min","max","values"}, # [0]=QUANTITY, [1]=TIME, [2]=UNIT_PRICE + {...}, {...} ], + }, + "state": { "step": , "relative_time": , "current_offer": [q,t,up] | None }, + # on "respond": observation["current_offer"] and/or state["current_offer"] holds the opponent offer + "fallback_offer": [q,t,up] | None, # what the greedy fallback would offer + "fallback_response": "accept"|"reject"|"end", +} +``` + +### What to return +- On `event == "propose"`: `{"offer": [quantity, time, unit_price]}` — three **ints**, each + within its issue's `[min, max]`. Out-of-range/non-int → counted as a policy error, fallback used. +- On `event == "respond"`: `{"response": "accept" | "reject" | "end"}`. When rejecting you MAY + include a counter: `{"response": "reject", "offer": [q,t,up]}`. +- Return `{}` or `None` at any point to defer to the trusted greedy fallback (safe default). + +### Hard rules +- **Never import `scml`, `negmas`, `numpy`, or anything outside the stdlib.** The port must be + pure-Python stdlib only — you only have the observation dict. Re-express the strategy's math + from scratch. +- **Never raise.** Wrap the body in `try/except` returning `{}` (see reference). An unhandled + exception floors the score. +- `is_selling` = `nmi["annotation"].get("product") == awi.get("my_output_product")`. +- "How much I still need" = `awi["needed_sales"]` if selling else `awi["needed_supplies"]`. +- Concession over time: use `state["relative_time"]` (0 at start → 1 at deadline) instead of + `state.step / nmi.n_steps` (n_steps isn't exposed on nmi). +- No cross-call state is guaranteed; if the original kept per-step memory (opponent price models, + `on_negotiation_success`), you may keep **module-level** dicts keyed by negotiator_id, but you + get NO success/step callbacks — approximate or drop that refinement and note it in the docstring. + +## Mapping the source's methods +- source `propose(negotiator_id, state)` / `first_proposals` → your `event=="propose"` branch. +- source `respond(...)` / `counter_all(offers, states)` → your `event=="respond"` branch. +- source `best_offer` / price helpers (`_find_good_price`, `_price_range`, `_th`) → inline as + plain functions over the observation (reference port shows the pattern). + +## If the bot is RL / learned / infeasible +Some agents (q-learning, PPO, regression on trained weights) can't be reproduced without their +weights/deps. In that case: port the **heuristic core** if one exists (many have a rule-based +path); otherwise DO NOT fake it — return a short note in your report that the bot is +RL-weight-dependent and should be skipped, and still write a best-effort heuristic port only if +faithful. Log the reason. + +## Deliverable +Write your port to `scripts/ladder/ports/__.py` +(e.g. `scml2024__team_193.py`). It must `python3 -c "import ast; ast.parse(open(FILE).read())"` +cleanly and define a top-level `decide`. Keep a concise module docstring naming the source and +noting any simplifications/dropped features. diff --git a/scripts/ladder/README.md b/scripts/ladder/README.md new file mode 100644 index 00000000..bd3535ae --- /dev/null +++ b/scripts/ladder/README.md @@ -0,0 +1,31 @@ +# Ladder build tooling + +Operational one-off scripts for constructing a **porting-based** CC:Ladder — one where the +human bots are open-source agents written against a different framework API and must be ported +into an arena's single-file submission contract before they can be ranked. Built for SCML OneShot +(`decide(observation)`), but the workflow generalizes to other arenas (e.g. Halite). + +This is scaffolding, not product code — nothing under `codeclash/` imports it. The durable +outputs of a build are: the `human/*` branches on `CodeClash-ai/` (the bots), the +`configs/ablations/ladder/*.yaml` configs, and the arena's Dockerfile wiring. Ports themselves +are **not** committed here (see `.gitignore`); they live on the branches. + +## Files +- `PORTING_GUIDE.md` — the `decide(observation)` contract + how to port a source agent. Hand this + to a porting agent (with `examples/scml_agent.py` as the worked reference). +- `examples/` — a reference port (`scml_agent.py` = GreedyOneShotAgent) + `dummy_agent.py` + an + arena smoke config (`scml_ffa.yaml`). Used by the smoke scripts. +- `validate_ports.py` — stage 1: local syntax/import/`decide` check over `ports/*.py`. +- `run_smoke_all.sh` — stage 2: run every stage-1 pass through the real runtime in Docker, + batched, to confirm each plays without crashing/erroring. Writes `ports/_stage2.json`. +- `smoke_scml.sh` — quick single-pair smoke (example greedy vs dummy) through the arena image. +- `push_branches.sh` — push each stage-2-healthy port to `CodeClash-ai/SCML` as a + `human//` branch (dedupes identical content; skips a SKIP list). +- `RUN_ON_AWS.md` — how to run the round-robin (`ladder make`) + Elo ranking on a big box. + +## Typical flow +1. Populate `ports/__.py` (fan out porting agents with `PORTING_GUIDE.md`). +2. `python3 scripts/ladder/validate_ports.py` → stage 1 +3. `bash scripts/ladder/run_smoke_all.sh` → stage 2 (needs Docker) +4. `bash scripts/ladder/push_branches.sh` → push healthy ports to branches +5. Rank + assemble configs — see `RUN_ON_AWS.md`. diff --git a/scripts/ladder/RUN_ON_AWS.md b/scripts/ladder/RUN_ON_AWS.md new file mode 100644 index 00000000..88ef613b --- /dev/null +++ b/scripts/ladder/RUN_ON_AWS.md @@ -0,0 +1,83 @@ +# Running the SCML round-robin (Elo ranking) on AWS + +Goal: run the 1,275-pair round-robin in `make_scml.yaml` to rank the 51 human bots, then +compute Elo and assemble the ladder. The 51 bots already live on `CodeClash-ai/SCML` as +`human/*` branches — `branch_init` fetches them at runtime, so nothing bot-side needs shipping; +you only need this repo branch + Docker + a GitHub token. + +## Prerequisites on the AWS box +- Docker running (`docker info`), git, and `uv` (repo uses `uv run codeclash ...`). +- A GitHub token with read access to `CodeClash-ai/SCML` (public, so a default `gh auth token` + or any classic PAT works). Export it as `GITHUB_TOKEN` for the run. +- This branch pulled: `git fetch && git checkout && uv sync` (or the repo's usual setup). + +## Step 0 — pre-build the arena image ONCE (avoids a build stampede) +`ladder make --workers N` builds the image lazily per pair; with many workers they'd all try to +build at once. Build it a single time up front (it `git clone`s CodeClash-ai/SCML and installs +`scml==0.8.2`): + +```bash +docker build -t codeclash/scml -f codeclash/arenas/scml/SCML.Dockerfile . +``` + +Sanity check one pair end-to-end before the big run: + +```bash +bash scripts/ladder/smoke_scml.sh # greedy vs dummy, should print PASS +``` + +## Step 1 (recommended) — cheap pilot ranking (~1 h on 32 cores) +Edit `configs/ablations/ladder/make_scml.yaml`: comment out `sims_per_round: 400`, uncomment +`sims_per_round: 30`. Then: + +```bash +GITHUB_TOKEN=$(gh auth token) \ + uv run codeclash ladder make configs/ablations/ladder/make_scml.yaml --workers 30 +python -m codeclash.analysis.metrics.elo -d logs/ladder/SCML --output-dir assets/scml_elo_pilot +``` + +Eyeball the printed Elo ordering: the baselines (`nice` < `random` < `greedy`) should sit near +the bottom, and disciplined bots should outrank very concessive ones. If it looks sane, proceed. +(The pilot logs live in a different pair-count than the full run only in `sims`; to force fresh +full-sim logs, run the full pass in a clean `logs/` or a separate `-o` dir — see note below.) + +## Step 2 — full ranking run (400 sims, ~23 h on 32 cores) +Restore `sims_per_round: 400` in the config, then launch under `nohup`/`tmux` so it survives +disconnects: + +```bash +tmux new -s scml +GITHUB_TOKEN=$(gh auth token) \ + uv run codeclash ladder make configs/ablations/ladder/make_scml.yaml --workers 30 \ + 2>&1 | tee scml_make.log +# detach: Ctrl-b d | reattach: tmux attach -t scml +``` + +- **Resumable:** each pair writes to `logs/ladder/SCML/PvpTournament.
_vs_/`; a rerun skips + pairs whose folder already exists. Safe to stop/restart. If it dies, just relaunch the same + command — it continues where it left off. +- **`--workers 30`** on a 32-core box (leave 2 cores headroom; the `decide` 3s timeout can trip + under CPU oversubscription). Progress = count of `logs/ladder/SCML/PvpTournament.*` dirs + (target 1275): `ls -d logs/ladder/SCML/PvpTournament.* | wc -l`. + +> NOTE on pilot→full log mixing: if you ran the Step-1 pilot into `logs/`, move or delete +> `logs/ladder/SCML/` before the full run (or point the pilot elsewhere) so the Elo fit uses only +> the 400-sim results. The make command has no `-o`; it always writes under `logs/ladder//`. + +## Step 3 — compute Elo and rank +```bash +python -m codeclash.analysis.metrics.elo -d logs/ladder/SCML --output-dir assets/scml_elo +``` +Prints the Bradley-Terry/Elo ranking and writes `assets/scml_elo/elo_results.log` (+ plots, +LaTeX/website tables). The ordering is what becomes the ladder (weakest → strongest). + +## Step 4 — assemble the ladder configs (do this once you have the ranking) +1. `configs/ablations/ladder/rungs/scml.yaml` — the ranked opponent list, worst first, strongest + last (each `{agent: dummy, branch_init: human/...}`), in Elo order from Step 3. +2. `configs/ablations/ladder/scml.yaml` — the run config: a climbing `player` (starting at the + weakest rung) + `ladder: !include ablations/ladder/rungs/scml.yaml` + a `ladder_rules` block. + Model on `battlesnake.yaml`. +3. Optional `scml__.yaml` per-model variants (swap `model: !include mini/models/...`). + +(Ping me with `logs/ladder/SCML/` or the `elo_results.log` and I'll generate the rungs + run +configs automatically.) diff --git a/scripts/ladder/examples/dummy_agent.py b/scripts/ladder/examples/dummy_agent.py new file mode 100644 index 00000000..c5f40874 --- /dev/null +++ b/scripts/ladder/examples/dummy_agent.py @@ -0,0 +1,9 @@ +"""Dummy opponent: always defer to the trusted greedy fallback. + +Returning {} at every turn means the runtime's built-in GreedySyncAgent drives this +player. Used as the baseline opponent in the pilot smoke test. +""" + + +def decide(observation): + return {} diff --git a/scripts/ladder/examples/scml_agent.py b/scripts/ladder/examples/scml_agent.py new file mode 100644 index 00000000..cf46c056 --- /dev/null +++ b/scripts/ladder/examples/scml_agent.py @@ -0,0 +1,115 @@ +"""Pilot port of SCML's GreedyOneShotAgent to the CodeClash `decide()` contract. + +Source: scml/oneshot/agents/greedy.py (GreedyOneShotAgent) in yasserfarouk/scml. +The upstream agent is an ``OneShotAgent`` subclass with methods ``propose`` / +``respond`` / ``best_offer`` and persistent per-step price memory. This arena instead +calls a stateless ``decide(observation)`` and hands plain dicts, so the port: + + * maps the two entry points via ``observation["event"]`` ("propose" / "respond"); + * reads world state from ``observation["awi"]`` (uses ``needed_sales`` / + ``needed_supplies`` directly, which the upstream ``_needed`` only approximated + via ``exogenous_contract_summary`` — cleaner and exposed here); + * derives the linear concession threshold from ``state["relative_time"]`` + (upstream used ``state.step / nmi.n_steps``; nmi.n_steps is not exposed here); + * DROPS the best-price-slack opponent memory (``_best_opp_*``), because we get no + ``on_negotiation_success`` callback in ``decide``. This keeps the faithful greedy + conceision-over-time behavior; the slack refinement is the one intentional loss. + +Returns {} anywhere the inputs are missing so the trusted greedy fallback takes over +rather than erroring (an unhandled exception floors the score). +""" + +QUANTITY, TIME, UNIT_PRICE = 0, 1, 2 +_CONCESSION_EXPONENT = 0.4 # fixed (upstream randomizes in [0.2, 1.0]); deterministic here + + +def _issue_bounds(issues, idx): + issue = issues[idx] + return int(issue["min"]), int(issue["max"]) + + +def _is_selling(awi, nmi): + """A negotiation is a sale iff its product is my output product.""" + product = nmi.get("annotation", {}).get("product") + return product == awi.get("my_output_product") + + +def _threshold(state): + """Linear concession: 1.0 (hold best price) at the start -> 0.0 (concede) at the end.""" + rt = state.get("relative_time") + if rt is None: + return 1.0 + return (1.0 - float(rt)) ** _CONCESSION_EXPONENT + + +def _good_price(awi, nmi, state): + """The price to offer/accept, conceding from best toward worst over time.""" + up_min, up_max = _issue_bounds(nmi["issues"], UNIT_PRICE) + th = _threshold(state) + if _is_selling(awi, nmi): + return int(round(up_min + th * (up_max - up_min))) # seller: high early, concede down + return int(round(up_max - th * (up_max - up_min))) # buyer: low early, concede up + + +def _my_needs(awi, nmi): + return awi.get("needed_sales", 0) if _is_selling(awi, nmi) else awi.get("needed_supplies", 0) + + +def _best_offer(awi, nmi, state): + """Quantity sized to remaining need (clamped to the issue range); price = good price.""" + needs = _my_needs(awi, nmi) + if not needs or needs <= 0: + return None + q_min, q_max = _issue_bounds(nmi["issues"], QUANTITY) + quantity = max(q_min, min(int(needs), q_max)) + t_min, t_max = _issue_bounds(nmi["issues"], TIME) + time = max(t_min, min(int(awi.get("current_step", t_min)), t_max)) + return [quantity, time, _good_price(awi, nmi, state)] + + +def _is_good_price(awi, nmi, state, price): + up_min, up_max = _issue_bounds(nmi["issues"], UNIT_PRICE) + span = up_max - up_min + if span <= 0: + return True + th = _threshold(state) + if _is_selling(awi, nmi): + return (price - up_min) >= th * span + return (up_max - price) >= th * span + + +def decide(observation): + try: + event = observation.get("event") + awi = observation.get("awi") or {} + nmi = observation.get("nmi") or {} + issues = nmi.get("issues") or [] + if len(issues) < 3: + return {} # no valid negotiation ranges -> defer to fallback + + if event == "propose": + offer = _best_offer(awi, nmi, observation.get("state") or {}) + return {"offer": offer} if offer else {} + + if event == "respond": + state = observation.get("state") or {} + current = observation.get("current_offer") or state.get("current_offer") + if not current or len(current) < 3: + return {} + + needs = _my_needs(awi, nmi) + if not needs or needs <= 0: + return {"response": "end"} # nothing more to trade + + if current[QUANTITY] > needs: # too much -> reject, counter with my best + counter = _best_offer(awi, nmi, state) + return {"response": "reject", "offer": counter} if counter else {"response": "reject"} + + if _is_good_price(awi, nmi, state, current[UNIT_PRICE]): + return {"response": "accept"} + counter = _best_offer(awi, nmi, state) + return {"response": "reject", "offer": counter} if counter else {"response": "reject"} + + return {} + except Exception: + return {} # never crash the world; fall back to greedy diff --git a/scripts/ladder/examples/scml_ffa.yaml b/scripts/ladder/examples/scml_ffa.yaml new file mode 100644 index 00000000..58b8e032 --- /dev/null +++ b/scripts/ladder/examples/scml_ffa.yaml @@ -0,0 +1,20 @@ +# Arena smoke test (Phase 3) for the SCML ladder pilot: run the ported human bot +# through the REAL arena (Docker build → clone CodeClash-ai/SCML → branch_init checkout +# → validate_code → real SCML2024OneShot games) against a fallback-greedy dummy. +# GITHUB_TOKEN=$(gh auth token) uv run codeclash run configs/ablations/ladder/scml_ffa.yaml +tournament: + rounds: 1 +game: + name: SCML + sims_per_round: 4 + args: + n_steps: 8 + n_lines: 2 +players: +- agent: dummy + name: greedy-oneshot + branch_init: human/scml-baselines/greedy-oneshot +- agent: dummy + name: baseline-fallback # default branch scml_agent.py -> returns {} -> greedy fallback +prompts: + game_description: SCML ladder smoke test diff --git a/scripts/ladder/push_branches.sh b/scripts/ladder/push_branches.sh new file mode 100644 index 00000000..9f07dee2 --- /dev/null +++ b/scripts/ladder/push_branches.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Push every stage-2-healthy port to CodeClash-ai/SCML as a human// branch. +# File stem "scml2021__team_54" -> branch "human/scml2021/team_54". +# Skips ports listed in SKIP. Dedupes exact-duplicate content (keeps first). +# bash scripts/ladder/push_branches.sh +set -euo pipefail +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +PORTS="$REPO_ROOT/scripts/ladder/ports" +SKIP="scml2023__team_139" # extreme-price bot trips per-negotiation bound mismatch; defer + +TMP=$(mktemp -d) +git clone -q "https://x-access-token:$(gh auth token)@github.com/CodeClash-ai/SCML.git" "$TMP" +cd "$TMP" +git checkout -q main + +SEEN=$(mktemp) +pushed=0; skipped=0 +PY="$(command -v python3 || command -v python)" +OKS=$("$PY" -c "import json;d=json.load(open('$PORTS/_stage2.json'));print('\n'.join(k for k,v in sorted(d.items()) if v.get('ran') and not v.get('crashed') and v.get('decisions',0)>0 and v.get('errors',1)==0))") + +for stem in $OKS; do + stem="${stem%.py}" + case " $SKIP " in *" $stem "*) echo "SKIP $stem (deferred)"; skipped=$((skipped+1)); continue;; esac + h=$(shasum "$PORTS/$stem.py" | awk '{print $1}') + if grep -q "^$h " "$SEEN"; then echo "SKIP $stem (dup of $(grep "^$h " "$SEEN" | awk '{print $2}'))"; skipped=$((skipped+1)); continue; fi + echo "$h $stem" >> "$SEEN" + branch="human/$(echo "$stem" | sed 's/__/\//')" # scml2021__team_54 -> human/scml2021/team_54 + git checkout -q main + git checkout -q -B "$branch" + cp "$PORTS/$stem.py" scml_agent.py + git add scml_agent.py + git -c user.email=player@codeclash.com -c user.name="CodeClash" commit -qm "Import $stem (SCML OneShot ladder)" + git push -q -f -u origin "$branch" 2>/dev/null + echo "PUSH $branch" + pushed=$((pushed+1)) +done +cd "$REPO_ROOT"; rm -rf "$TMP" +echo "=== pushed $pushed, skipped $skipped ===" diff --git a/scripts/ladder/run_smoke_all.sh b/scripts/ladder/run_smoke_all.sh new file mode 100644 index 00000000..8ccc2ba4 --- /dev/null +++ b/scripts/ladder/run_smoke_all.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Stage 2: run every stage-1-passing port through the REAL SCML runtime (in the arena +# Docker image), batched, to confirm each loads, makes decisions, and never crashes/floors. +# Writes a per-agent verdict table (scripts/ladder/ports/_stage2.json). Run after validate_ports.py. +# Populate scripts/ladder/ports/ with the candidate *.py first (see PORTING_GUIDE.md). +# bash scripts/ladder/run_smoke_all.sh +set -euo pipefail +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +PORTS="$REPO_ROOT/scripts/ladder/ports" +EX="$REPO_ROOT/scripts/ladder/examples" +OUT="$REPO_ROOT/scripts/ladder/out" +IMAGE="scml-smoke:latest" +PY="$(command -v python3 || command -v python)" + +cd "$REPO_ROOT" +docker build -q -f codeclash/arenas/scml/SCML.Dockerfile -t "$IMAGE" . >/dev/null +mkdir -p "$OUT" + +"$PY" - "$PORTS" "$EX" "$OUT" "$IMAGE" <<'PY' +import json, subprocess, sys +from pathlib import Path + +ports, ex, outdir, image = (Path(sys.argv[1]), Path(sys.argv[2]), Path(sys.argv[3]), sys.argv[4]) +stage1 = json.loads((ports / "_stage1.json").read_text()) +files = [ports / n for n, r in sorted(stage1.items()) if r["pass"]] + +# batch into groups of 5; each group includes the example greedy agent as a common anchor +BATCH = 5 +verdict = {} +batches = [files[i:i+BATCH] for i in range(0, len(files), BATCH)] +for bi, batch in enumerate(batches): + args = ["--agent", "anchor_greedy=/ex/scml_agent.py"] + for f in batch: + args += ["--agent", f"{f.stem}=/ports/{f.name}"] + out = f"/out/batch_{bi}.json" + cmd = ["docker", "run", "--rm", + "-v", f"{ports}:/ports:ro", "-v", f"{ex}:/ex:ro", "-v", f"{outdir}:/out", + image, "python", "run_scml.py", "--sims", "1", "--steps", "6", + "--lines", "2", "--decision-timeout", "5.0", "--output", out] + args + print(f" batch {bi+1}/{len(batches)}: {[f.stem for f in batch]}") + p = subprocess.run(cmd, capture_output=True, text=True) + res_path = outdir / f"batch_{bi}.json" + if p.returncode != 0 or not res_path.exists(): + for f in batch: + verdict[f.stem] = {"ran": False, "err": (p.stderr or p.stdout)[-200:]} + continue + res = json.loads(res_path.read_text()) + agg = {} + for d in res["details"]: + d = json.loads(d) + a = agg.setdefault(d["player"], {"decisions": 0, "errors": 0, "score": 0.0, "status": "ok"}) + a["decisions"] += d.get("decisions", 0) + a["errors"] += d.get("policy_errors", 0) + a["score"] += d.get("score", 0.0) + if d.get("status") == "error": + a["status"] = "error" + for f in batch: + a = agg.get(f.stem, {}) + crashed = a.get("status") == "error" or a.get("score", 0) <= -1e6 + verdict[f.stem] = {"ran": True, "decisions": a.get("decisions", 0), + "errors": a.get("errors", 0), "crashed": crashed} + +(ports / "_stage2.json").write_text(json.dumps(verdict, indent=2, sort_keys=True)) +print("\n=== STAGE 2 VERDICTS ===") +ok = bad = 0 +for name in sorted(verdict): + v = verdict[name] + good = v.get("ran") and not v.get("crashed") and v.get("decisions", 0) > 0 and v.get("errors", 1) == 0 + ok += good; bad += not good + tag = "OK " if good else "BAD " + print(f" {tag} {name}: {v}") +print(f"\nStage 2: {ok} healthy / {bad} problematic of {len(verdict)}") +PY diff --git a/scripts/ladder/smoke_scml.sh b/scripts/ladder/smoke_scml.sh new file mode 100644 index 00000000..140e7e9f --- /dev/null +++ b/scripts/ladder/smoke_scml.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Pilot smoke test: run the ported bot vs a dummy through the REAL SCML runtime +# (run_scml.py + scml==0.8.2) inside the arena's own Docker image. No GitHub repo +# needed — this exercises the decide() contract, validation, and real game scoring. +# +# bash scripts/ladder/smoke_scml.sh +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +EX_DIR="$REPO_ROOT/scripts/ladder/examples" +OUT_DIR="$REPO_ROOT/scripts/ladder/out" +IMAGE="scml-smoke:latest" + +cd "$REPO_ROOT" +mkdir -p "$OUT_DIR" + +echo "==> Building SCML arena image (installs scml==0.8.2 + runtime)" +docker build -q -f codeclash/arenas/scml/SCML.Dockerfile -t "$IMAGE" . >/dev/null + +echo "==> Running: greedy (example) vs dummy — 2 sims, 8 steps, 2 lines" +docker run --rm \ + -v "$EX_DIR:/ex:ro" \ + -v "$OUT_DIR:/out" \ + "$IMAGE" \ + python run_scml.py \ + --agent greedy=/ex/scml_agent.py \ + --agent dummy=/ex/dummy_agent.py \ + --sims 2 --steps 8 --lines 2 \ + --decision-timeout 5.0 \ + --output /out/scml_results.json + +echo "==> Result (scripts/ladder/out/scml_results.json):" +PY_BIN="$(command -v python3 || command -v python)" +"$PY_BIN" - "$OUT_DIR/scml_results.json" <<'PY' +import json, sys +r = json.load(open(sys.argv[1])) +print(" average_scores:", r["average_scores"]) +print(" sims :", r["sims"]) +errs = decisions = 0 +for d in r["details"]: + d = json.loads(d) + decisions += d.get("decisions", 0) + errs += d.get("policy_errors", 0) + if d.get("status") == "error": + print(" !! ERROR", d["player"], d.get("error")) +print(f" total decide() calls: {decisions} policy_errors: {errs}") +ok = decisions > 0 and errs == 0 and all(s > -1e6 for s in r["average_scores"].values()) +print(" SMOKE:", "PASS ✅" if ok else "FAIL ❌") +sys.exit(0 if ok else 1) +PY diff --git a/scripts/ladder/validate_ports.py b/scripts/ladder/validate_ports.py new file mode 100644 index 00000000..82be0004 --- /dev/null +++ b/scripts/ladder/validate_ports.py @@ -0,0 +1,61 @@ +"""Stage 1 validator: mirrors the arena's validate_code locally (no Docker/scml needed +since ports are stdlib-only). For each ports/*.py: syntax-compile, import, assert a +top-level callable `decide`, and call it with the arena's validation observation +(must return dict or None). Prints PASS/FAIL per file and writes ports/_stage1.json. +""" + +import ast +import importlib.util +import json +import sys +from pathlib import Path + +PORTS = Path(__file__).parent / "ports" +VALIDATE_OBS = {"event": "validate", "awi": {}, "state": {}, "nmi": {}} + + +def check(path: Path): + src = path.read_text() + try: + ast.parse(src) + except SyntaxError as e: + return False, f"syntax: {e}" + try: + spec = importlib.util.spec_from_file_location(f"port_{path.stem}", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + except Exception as e: + return False, f"import: {type(e).__name__}: {e}" + if not hasattr(mod, "decide") or not callable(mod.decide): + return False, "no callable decide" + # stdlib-only guard: fail if it imported scml/negmas/numpy + for banned in ("scml", "negmas", "numpy"): + if banned in src and f"import {banned}" in src: + return False, f"imports banned module: {banned}" + try: + r = mod.decide(dict(VALIDATE_OBS)) + except Exception as e: + return False, f"decide raised on validate obs: {type(e).__name__}: {e}" + if not (r is None or isinstance(r, dict)): + return False, f"decide returned {type(r).__name__}, need dict/None" + return True, "ok" + + +def main(): + results = {} + files = sorted(PORTS.glob("*.py")) + files = [f for f in files if not f.name.startswith("_")] + passed = failed = 0 + for f in files: + ok, msg = check(f) + results[f.name] = {"pass": ok, "msg": msg} + print(f" {'PASS' if ok else 'FAIL'} {f.name}" + ("" if ok else f" <- {msg}")) + passed += ok + failed += not ok + (PORTS / "_stage1.json").write_text(json.dumps(results, indent=2, sort_keys=True)) + print(f"\nStage 1: {passed} pass / {failed} fail of {len(files)} ports") + sys.exit(0) + + +if __name__ == "__main__": + main()