diff --git a/config/guerrillacheckers.ini b/config/guerrillacheckers.ini new file mode 100644 index 0000000000..3c2a2c18b8 --- /dev/null +++ b/config/guerrillacheckers.ini @@ -0,0 +1,53 @@ +[base] +env_name = guerrillacheckers +# One epoch is 4096 agents * 64 horizon = 262,144 agent steps. Save every +# 20 epochs so the historical pool receives a new policy about every 5.24M. +checkpoint_interval = 20 + +[selfplay] +enabled = 1 +max_size = 100 +seed = 42 +# Refresh each frozen bank about as often as snapshots are produced. Loading is +# deferred until all tagged games reach an episode boundary. +opp_timeout_steps = 5_000_000 + +[vec] +total_agents = 4096 +num_buffers = 8 +num_threads = 8 +num_frozen_banks = 1 +# 80% current-v-current envs and 20% current-v-historical envs. +frozen_bank_pct = 0.2 +action_mask_size = 256 + +[env] +# hard step cap; on timeout the Guerrilla is deemed to have failed (COIN wins). +max_episode_length = 256 +render_fps = 12 +# selfplay = 1: two policy slots alternate sides. In historical-pool envs, +# slot 0 is the primary and slot 1 is the frozen opponent. +# selfplay = 0: policy plays `side` against a built-in `opponent` bot. +selfplay = 1 +# Side played by slot 0 (the primary policy) in selfplay, or by the agent in +# bot mode: 0 = random per episode, 1 = Guerrilla, 2 = COIN. +side = 0 +# opponent bot when selfplay = 0: 0 = random, 1 = greedy (capture-maximizing), 2 = mcts. +opponent = 1 +# MCTS opponent knobs (used when opponent = 2). Higher iterations = stronger, slower. +mcts_iterations = 256 +mcts_exploration = 0.7 +# rollout policy: 0 = random (faithful UCT), 1 = greedy (stronger, esp. as guerrilla). +mcts_rollout = 1 + +[policy] +hidden_size = 128 +num_layers = 2 +expansion_factor = 1 + +[train] +total_timesteps = 100_000_000 +gamma = 0.98 +# This env has a sparse 256-way action head; action masks handle legality, and +# low entropy pressure keeps exploration from fighting the mask too hard. +ent_coef = 0.0001 diff --git a/ocean/guerrillacheckers/README.md b/ocean/guerrillacheckers/README.md new file mode 100644 index 0000000000..240a55c136 --- /dev/null +++ b/ocean/guerrillacheckers/README.md @@ -0,0 +1,59 @@ +# Guerrilla Checkers + +The standalone client and tournament preserve the original Puffer 40 as the +baseline for new standard 5c self-play models. + +| Checkpoint | Purpose | SHA-256 | +| --- | --- | --- | +| `guerrillacheckers_weights.bin` | Original Puffer 40 | `f58615069b9a0e50105dd54b4729d366d602fe14b957a0042e4dc88089120398` | + +Puffer 40 uses the legacy 587,264-byte biased layout. Newly trained standard +5c checkpoints use the 586,240-byte bias-free layout with a value-head row. +The standalone evaluator detects both formats directly. + +## Train + +The canonical `config/guerrillacheckers.ini` uses genuine alternating self-play +with randomized sides: slot 0 is the trainable policy, slot 1 is the current or +historical opponent, and the inactive slot receives one deterministic pass +action. Both recurrent states observe every game step, following the existing +Ocean Chess self-play convention. Twenty percent of environments use the +historical bank. Opponent swaps happen only after all tagged games reach an +episode boundary. + +```sh +./puffer train guerrillacheckers +``` + +## Evaluate + +Build the standalone client and compare a native candidate against the original +Puffer 40 from both sides: + +```sh +./build.sh guerrillacheckers --fast +./guerrillacheckers --compare-candidate 1000 \ + .runtime/checkpoints/guerrillacheckers//.bin +``` + +The recorded baseline field contains 100 games per cell, shown as Guerrilla +wins - COIN wins. + +| Guerrilla / COIN | Random | Greedy | Puffer 40 | MCTS 2K | MCTS 10K | +| --- | ---: | ---: | ---: | ---: | ---: | +| Random | 8-92 | 0-100 | 0-100 | 0-100 | 0-100 | +| Greedy | 98-2 | 33-67 | 1-99 | 0-100 | 0-100 | +| Puffer 40 | 96-4 | 99-1 | 15-85 | 12-88 | 9-91 | +| MCTS 2K | 99-1 | 100-0 | 50-50 | 30-70 | 11-89 | +| MCTS 10K | 100-0 | 100-0 | 84-16 | 81-19 | 65-35 | + +The side-specific Bradley-Terry fit is anchored at Elo 1500 across all ten +side-specific entries: + +| Bot | Guerrilla Elo | COIN Elo | +| --- | ---: | ---: | +| Random | 338 | 727 | +| Greedy | 1120 | 1189 | +| Puffer 40 | 1636 | 1879 | +| MCTS 2K | 1834 | 1969 | +| MCTS 10K | 2200 | 2108 | diff --git a/ocean/guerrillacheckers/guerrillacheckers.c b/ocean/guerrillacheckers/guerrillacheckers.c new file mode 100644 index 0000000000..fbce7fee77 --- /dev/null +++ b/ocean/guerrillacheckers/guerrillacheckers.c @@ -0,0 +1,1103 @@ +// Standalone Guerrilla Checkers client: a menu picks Human or AI (with a +// per-side level) for each side, then play is fully mouse-driven (click a +// piece or point, then a destination). AI vs AI can also run as a fast +// tournament with a running win tally. Works the same in native and +// Emscripten web builds. +#include +#include + +#include "guerrillacheckers.h" +#include "puffercpu.h" + +#define GC_DEMO_NOOP -1 + +enum { + GC_UI_MENU = 0, + GC_UI_PLAY = 1, + GC_UI_TOURNEY = 2, +}; + +enum { + GC_CTRL_HUMAN = 0, + GC_CTRL_AI = 1, +}; + +typedef struct { + const char* label; + const char* name; + int opponent; + int mcts_iterations; +} GcDemoLevel; + +#define GC_DEMO_NET_BOT -1 // sentinel opponent id: puffernet policy +#define GC_DEMO_CANDIDATE_BOT -2 + +static const GcDemoLevel gc_demo_levels[] = { + {"1", "RANDOM", GC_BOT_RANDOM, 0}, + {"2", "GREEDY", GC_BOT_GREEDY, 0}, + {"3", "PUFFER NN", GC_DEMO_NET_BOT, 0}, + {"4", "MCTS 2K", GC_BOT_MCTS, 2000}, + {"5", "MCTS 10K", GC_BOT_MCTS, 10000}, + {"6", "PUFFER 5C", GC_DEMO_CANDIDATE_BOT, 0}, +}; +#define GC_DEMO_LEVEL_COUNT 5 +#define GC_DEMO_CANDIDATE_LEVEL 5 +#define GC_CLI_MAX_BOTS 6 +#define GC_DEMO_LEVEL_NET 2 // index of the PUFFER NN entry +#define GC_DEMO_LEVEL_MCTS_2K 3 // default when the net weights are unavailable +static const char* gc_demo_level_legend = + "1 RANDOM 2 GREEDY 3 PUFFER NN 4 MCTS 2K 5 MCTS 10K"; +static const char* gc_demo_level_legend_no_net = + "1 RANDOM 2 GREEDY 4 MCTS 2K 5 MCTS 10K"; + +// Preserved default PufferLib policy. One instance per side ensures the +// recurrent MinGRU state never mixes the two players' turns. +#define GC_DEMO_NET_HIDDEN 128 +#define GC_DEMO_NET_LAYERS 2 +#define GC_DEMO_NET_WEIGHTS "resources/guerrillacheckers/guerrillacheckers_weights.bin" + +typedef struct { + Weights* weights; + Affine* encoder; + Affine* decoder; + MinGRU* gru; + int full_turn_state; +} GcDemoNet; + +enum { + GC_DEMO_NET_ORIGINAL = 0, + GC_DEMO_NET_CANDIDATE = 1, + GC_DEMO_NET_COUNT = 2, +}; +static GcDemoNet gc_demo_nets[GC_DEMO_NET_COUNT][3]; +static int gc_demo_net_loaded[GC_DEMO_NET_COUNT]; + +#define GC_DEMO_AI_WAIT 20 // frames between an action and the AI's reply +#define GC_DEMO_OVER_WAIT 45 // frames before a click can leave the game-over screen +#define GC_DEMO_TOURNEY_BUDGET 0.012 // seconds of simulation per rendered frame +#define GC_DEMO_TRAIL_MAX 8 + +// Ghost trail of the last completed turn, so the opponent's move stays +// readable after it lands. A coin capture chain accumulates into one trail, +// with every square the coin visited kept in coin_path. +typedef struct { + int side; // GC_NONE while empty + int placed[2]; + int placed_count; + int coin_path[GC_DEMO_TRAIL_MAX + 2]; + int coin_path_count; + int captured_g[GC_DEMO_TRAIL_MAX]; + int captured_g_count; + int captured_coins[GC_DEMO_TRAIL_MAX]; + int captured_coins_count; +} GcDemoTrail; + +typedef struct { + int games; + int guerrilla_wins; + int coin_wins; +} GcDemoTally; + +typedef struct { + int mode; + int ctrl[3]; // indexed by GC_GUERRILLA / GC_COIN + int ai_level[3]; // ditto + int selected; // guerrilla point or coin square awaiting a destination click + int ai_wait; + int over_wait; + int paused; // tournament simulation paused (spacebar) + GcDemoTrail trail; + GcDemoTally tally; +} GcDemoUi; + +static const Color GC_DEMO_BG = {18, 24, 28, 255}; +static const Color GC_DEMO_TEXT = {190, 204, 208, 255}; +static const Color GC_DEMO_DIM = {104, 126, 132, 255}; +static const Color GC_DEMO_GUERRILLA = {206, 72, 72, 255}; +static const Color GC_DEMO_COIN = {232, 198, 83, 255}; +static const Color GC_DEMO_SELECT = {255, 255, 255, 230}; +static const Color GC_DEMO_G_HINT = {206, 72, 72, 160}; // ghost guerrilla stones +static const Color GC_DEMO_G_HINT_DIM = {206, 72, 72, 110}; // legal first placements +static const Color GC_DEMO_C_HINT = {255, 240, 170, 230}; // coin move markers +static const Color GC_DEMO_C_SQUARE = {232, 198, 83, 60}; // coin destination squares +static const Color GC_DEMO_TRAIL = {255, 255, 255, 200}; // "just moved here" dots +static const Color GC_DEMO_G_GHOST = {206, 72, 72, 190}; // captured guerrilla X +static const Color GC_DEMO_C_GHOST = {232, 198, 83, 150}; // coin path lines +static const Color GC_DEMO_C_CAPTURED = {232, 198, 83, 190}; // captured coin X +static const Color GC_DEMO_C_ORIGIN = {232, 198, 83, 90}; // coin origin ghost piece + +static void gc_demo_allocate(GuerrillaCheckers* env) { + for (int slot = 0; slot < env->num_agents; slot++) { + env->agents[slot].observations = calloc(GC_OBS_SIZE, sizeof(uint8_t)); + env->agents[slot].actions = (float*)calloc(1, sizeof(float)); + env->agents[slot].rewards = (float*)calloc(1, sizeof(float)); + env->agents[slot].terminals = (float*)calloc(1, sizeof(float)); + env->agents[slot].action_mask = + (unsigned char*)calloc(GC_ACTIONS, sizeof(unsigned char)); + } +} + +static void gc_demo_free(GuerrillaCheckers* env) { + puf_close(env); + for (int slot = 0; slot < env->num_agents; slot++) { + free(env->agents[slot].action_mask); + free(env->agents[slot].terminals); + free(env->agents[slot].rewards); + free(env->agents[slot].actions); + free(env->agents[slot].observations); + } +} + +static int gc_demo_mouse_coin_cell(GuerrillaCheckers* env, Vector2 mouse) { + if (env->client == NULL) return -1; + int cell = env->client->cell; + int x = (int)mouse.x / cell; + int y = (int)mouse.y / cell; + if (!gc_valid_coin_xy(x, y)) return -1; + return gc_coin_pos(x, y); +} + +static int gc_demo_mouse_guerrilla_cell(GuerrillaCheckers* env, Vector2 mouse) { + if (env->client == NULL) return -1; + int cell = env->client->cell; + int best = -1; + float best_dist2 = (float)(cell * cell); + float limit = (float)(cell * cell) * 0.18f; + for (int y = 0; y < GC_G_H; y++) { + for (int x = 0; x < GC_G_W; x++) { + float dx = mouse.x - (float)((x + 1) * cell); + float dy = mouse.y - (float)((y + 1) * cell); + float dist2 = dx * dx + dy * dy; + if (dist2 < best_dist2) { + best_dist2 = dist2; + best = gc_g_pos(x, y); + } + } + } + return best_dist2 <= limit ? best : -1; +} + +// Both sides encode actions as pos * 4 + dir, so one check covers guerrilla +// first-placement points and coin source squares. +static int gc_demo_pos_has_legal_action(GuerrillaCheckers* env, int pos) { + if (pos < 0) return 0; + for (int dir = 0; dir < 4; dir++) { + if (gc_action_is_legal(env, pos * 4 + dir)) return 1; + } + return 0; +} + +static int gc_demo_guerrilla_action(int first, int second) { + static const int dirs[4] = {2, 3, 0, 1}; + for (int dir = 0; dir < 4; dir++) { + if (gc_g_neighbor(first, dirs[dir]) == second) return first * 4 + dir; + } + return GC_DEMO_NOOP; +} + +static int gc_demo_coin_action(int src, int dst) { + for (int dir = 0; dir < 4; dir++) { + if (gc_coin_neighbor(src, dir) == dst) return src * 4 + dir; + } + return GC_DEMO_NOOP; +} + +static int gc_demo_human_action(GuerrillaCheckers* env, int* selected) { + if (!IsMouseButtonPressed(MOUSE_LEFT_BUTTON)) return GC_DEMO_NOOP; + + Vector2 mouse = GetMousePosition(); + if (env->player_to_move == GC_GUERRILLA) { + int pos = gc_demo_mouse_guerrilla_cell(env, mouse); + if (pos < 0) return GC_DEMO_NOOP; + if (pos == *selected) { + *selected = GC_DEMO_NOOP; + return GC_DEMO_NOOP; + } + if (*selected >= 0) { + int action = gc_demo_guerrilla_action(*selected, pos); + if (gc_action_is_legal(env, action)) return action; + } + if (gc_demo_pos_has_legal_action(env, pos)) *selected = pos; + return GC_DEMO_NOOP; + } + + int pos = gc_demo_mouse_coin_cell(env, mouse); + if (pos < 0) return GC_DEMO_NOOP; + if (pos == *selected) { + // A capture chain must be continued with the same piece: keep it selected. + if (!env->coin_must_capture) *selected = GC_DEMO_NOOP; + return GC_DEMO_NOOP; + } + if (*selected >= 0) { + int action = gc_demo_coin_action(*selected, pos); + if (gc_action_is_legal(env, action)) return action; + } + if (gc_demo_pos_has_legal_action(env, pos)) *selected = pos; + return GC_DEMO_NOOP; +} + +static void gc_demo_trail_update(GcDemoUi* ui, GuerrillaCheckers* env, int actor, + const uint8_t* coins_before, const uint8_t* g_before) { + GcDemoTrail* trail = &ui->trail; + + int src = -1; + int dst = -1; + for (int i = 0; i < GC_COIN_CELLS; i++) { + if (coins_before[i] && !env->coin_cells[i]) src = i; + if (!coins_before[i] && env->coin_cells[i]) dst = i; + } + + // Consecutive coin actions from the same square chain into one trail + // (forced multi-captures); anything else starts a fresh trail. + int chain = actor == GC_COIN && trail->side == GC_COIN && + src >= 0 && trail->coin_path_count > 0 && + trail->coin_path[trail->coin_path_count - 1] == src; + if (!chain) { + memset(trail, 0, sizeof(*trail)); + trail->side = actor; + } + + if (actor == GC_COIN) { + if (!chain && src >= 0) trail->coin_path[trail->coin_path_count++] = src; + if (dst >= 0 && trail->coin_path_count < GC_DEMO_TRAIL_MAX + 2) { + trail->coin_path[trail->coin_path_count++] = dst; + } + for (int i = 0; i < GC_G_CELLS; i++) { + if (g_before[i] && !env->guerrilla_cells[i] && + trail->captured_g_count < GC_DEMO_TRAIL_MAX) { + trail->captured_g[trail->captured_g_count++] = i; + } + } + return; + } + + for (int i = 0; i < GC_G_CELLS; i++) { + if (!g_before[i] && env->guerrilla_cells[i] && trail->placed_count < 2) { + trail->placed[trail->placed_count++] = i; + } + } + for (int i = 0; i < GC_COIN_CELLS; i++) { + if (coins_before[i] && !env->coin_cells[i] && + trail->captured_coins_count < GC_DEMO_TRAIL_MAX) { + trail->captured_coins[trail->captured_coins_count++] = i; + } + } +} + +// Apply one legal action, advance the turn, and record the ghost trail. +static void gc_demo_apply(GuerrillaCheckers* env, GcDemoUi* ui, int action) { + uint8_t coins_before[GC_COIN_CELLS]; + uint8_t g_before[GC_G_CELLS]; + memcpy(coins_before, env->coin_cells, sizeof(coins_before)); + memcpy(g_before, env->guerrilla_cells, sizeof(g_before)); + int actor = env->player_to_move; + + gc_apply_action(env, action); + env->tick++; + gc_prepare_turn(env); + gc_demo_trail_update(ui, env, actor, coins_before, g_before); +} + +static int gc_demo_net_init(GcDemoNet* net, const char* path) { + net->weights = load_weights(path); + if (net->weights == NULL) return 0; + int num_weights = net->weights->size - 7; + int native_weights = + GC_DEMO_NET_HIDDEN * GC_OBS_SIZE + + (GC_ACTIONS + 1) * GC_DEMO_NET_HIDDEN + + GC_DEMO_NET_LAYERS * 3 * GC_DEMO_NET_HIDDEN * GC_DEMO_NET_HIDDEN; + int legacy_weights = + GC_DEMO_NET_HIDDEN * GC_OBS_SIZE + GC_DEMO_NET_HIDDEN + + GC_ACTIONS * GC_DEMO_NET_HIDDEN + GC_ACTIONS + + GC_DEMO_NET_LAYERS * 3 * GC_DEMO_NET_HIDDEN * GC_DEMO_NET_HIDDEN; + if (num_weights != legacy_weights && num_weights != native_weights) { + fprintf(stderr, "error: unsupported Puffer checkpoint size: %d floats\n", + num_weights); + free(net->weights); + net->weights = NULL; + return 0; + } + int is_native = num_weights != legacy_weights; + // Puffer 40 used actor-only recurrent updates. Standard 5c policies process + // both acting and pass timesteps. + net->full_turn_state = is_native; + net->encoder = make_affine( + net->weights, !is_native, GC_OBS_SIZE, GC_DEMO_NET_HIDDEN); + int decoder_outputs = is_native ? GC_ACTIONS + 1 : GC_ACTIONS; + net->decoder = make_affine( + net->weights, !is_native, GC_DEMO_NET_HIDDEN, decoder_outputs); + net->weights->idx = (net->weights->idx + 7) & ~7; + net->gru = make_mingru(net->weights, 1, GC_DEMO_NET_HIDDEN, GC_DEMO_NET_LAYERS); + return 1; +} + +// Clear both sides' recurrent state at the start of every game. +static void gc_demo_net_reset(void) { + for (int model = 0; model < GC_DEMO_NET_COUNT; model++) { + if (!gc_demo_net_loaded[model]) continue; + for (int side = GC_GUERRILLA; side <= GC_COIN; side++) { + MinGRU* gru = gc_demo_nets[model][side].gru; + memset(gru->state, 0, + (size_t)gru->num_layers * gru->hidden_size * sizeof(float)); + } + } +} + +static float* gc_demo_net_observe(GuerrillaCheckers* env, int model, int side) { + GcDemoNet* net = &gc_demo_nets[model][side]; + gc_compute_observations(env); + int slot = env->selfplay ? env->slot_for_side[side] : 0; + uint8_t* observations = (uint8_t*)env->agents[slot].observations; + float obs[GC_OBS_SIZE]; + for (int i = 0; i < GC_OBS_SIZE; i++) { + obs[i] = (float)observations[i]; + } + affine(net->encoder, obs); + mingru(net->gru, net->encoder->output); + affine(net->decoder, net->gru->output); + return net->decoder->output; +} + +static int gc_demo_net_action(GuerrillaCheckers* env, int model) { + int side = env->player_to_move; + int slot = gc_actor_slot(env); + unsigned char* action_mask = env->agents[slot].action_mask; + float* logits = gc_demo_net_observe(env, model, side); + + // Sample from the softmax over legal actions, matching the masked + // sampling the policy was trained with. + float max_logit = -1e30f; + for (int a = 0; a < GC_ACTIONS; a++) { + if (action_mask[a] && logits[a] > max_logit) max_logit = logits[a]; + } + float probs[GC_ACTIONS]; + float total = 0.0f; + for (int a = 0; a < GC_ACTIONS; a++) { + probs[a] = action_mask[a] ? expf(logits[a] - max_logit) : 0.0f; + total += probs[a]; + } + float r = (float)gc_rand(env) / ((float)RAND_MAX + 1.0f) * total; + int last_legal = GC_DEMO_NOOP; + for (int a = 0; a < GC_ACTIONS; a++) { + if (probs[a] <= 0.0f) continue; + last_legal = a; + r -= probs[a]; + if (r <= 0.0f) return a; + } + return last_legal; +} + +static int gc_demo_button(Rectangle rect, const char* label, int active) { + Vector2 mouse = GetMousePosition(); + int hover = CheckCollisionPointRec(mouse, rect); + Color fill = active ? (Color){96, 44, 44, 255} : + hover ? (Color){54, 70, 78, 255} : (Color){36, 48, 54, 255}; + DrawRectangleRec(rect, fill); + DrawRectangleLinesEx(rect, 2.0f, active ? GC_DEMO_GUERRILLA : GC_DEMO_DIM); + int size = 18; + int width = MeasureText(label, size); + DrawText(label, (int)(rect.x + (rect.width - (float)width) / 2.0f), + (int)(rect.y + (rect.height - (float)size) / 2.0f), size, + active ? RAYWHITE : GC_DEMO_TEXT); + return hover && IsMouseButtonPressed(MOUSE_LEFT_BUTTON); +} + +static void gc_demo_start_game(GuerrillaCheckers* env, GcDemoUi* ui) { + // Vary the seed per game so bot play differs between runs. + env->rng ^= (unsigned int)(GetTime() * 1000.0) | 1u; + puf_reset(env); + ui->selected = GC_DEMO_NOOP; + ui->ai_wait = GC_DEMO_AI_WAIT; + ui->over_wait = 0; + ui->paused = 0; + memset(&ui->trail, 0, sizeof(ui->trail)); + gc_demo_net_reset(); +} + +static void gc_demo_render_menu(GuerrillaCheckers* env, GcDemoUi* ui) { + Client* client = env->client; + BeginDrawing(); + ClearBackground(GC_DEMO_BG); + + const char* title = "GUERRILLA CHECKERS"; + DrawText(title, (client->width - MeasureText(title, 32)) / 2, 56, 32, RAYWHITE); + + static const struct { const char* name; int side; float y; } rows[] = { + {"GUERRILLA", GC_GUERRILLA, 128.0f}, + {"COIN", GC_COIN, 232.0f}, + }; + for (int r = 0; r < 2; r++) { + int side = rows[r].side; + float y = rows[r].y; + DrawText(rows[r].name, 64, (int)y + 8, 20, + side == GC_GUERRILLA ? GC_DEMO_GUERRILLA : GC_DEMO_COIN); + if (gc_demo_button((Rectangle){256, y, 112, 36}, "HUMAN", + ui->ctrl[side] == GC_CTRL_HUMAN)) { + ui->ctrl[side] = GC_CTRL_HUMAN; + } + if (gc_demo_button((Rectangle){384, y, 112, 36}, "AI", + ui->ctrl[side] == GC_CTRL_AI)) { + ui->ctrl[side] = GC_CTRL_AI; + } + if (ui->ctrl[side] == GC_CTRL_AI) { + DrawText("LEVEL", 186, (int)y + 52, 16, GC_DEMO_DIM); + int shown = 0; + for (int i = 0; i < GC_DEMO_LEVEL_COUNT; i++) { + if (i == GC_DEMO_LEVEL_NET && + !gc_demo_net_loaded[GC_DEMO_NET_ORIGINAL]) continue; + Rectangle rect = {256 + (float)shown * 52.0f, y + 44.0f, 44, 32}; + shown++; + if (gc_demo_button(rect, gc_demo_levels[i].label, + ui->ai_level[side] == i)) { + ui->ai_level[side] = i; + } + } + } + } + + const char* legend = gc_demo_net_loaded[GC_DEMO_NET_ORIGINAL] ? gc_demo_level_legend : + gc_demo_level_legend_no_net; + DrawText(legend, (client->width - MeasureText(legend, 14)) / 2, 340, 14, + GC_DEMO_DIM); + + int both_ai = ui->ctrl[GC_GUERRILLA] == GC_CTRL_AI && + ui->ctrl[GC_COIN] == GC_CTRL_AI; + if (both_ai) { + if (gc_demo_button((Rectangle){110, 400, 150, 48}, "PLAY", 0)) { + gc_demo_start_game(env, ui); + ui->mode = GC_UI_PLAY; + } + if (gc_demo_button((Rectangle){284, 400, 182, 48}, "TOURNAMENT", 0)) { + memset(&ui->tally, 0, sizeof(ui->tally)); + gc_demo_start_game(env, ui); + ui->mode = GC_UI_TOURNEY; + } + } else if (gc_demo_button((Rectangle){208, 400, 160, 48}, "PLAY", 0)) { + gc_demo_start_game(env, ui); + ui->mode = GC_UI_PLAY; + } + + const char* help = "CLICK A PIECE OR POINT, THEN A DESTINATION"; + DrawText(help, (client->width - MeasureText(help, 16)) / 2, + client->height - 56, 16, GC_DEMO_DIM); + const char* credits = "Game design by Brian Train - Code by MischaU8"; + DrawText(credits, (client->width - MeasureText(credits, 16)) / 2, + client->height - 28, 16, GC_DEMO_DIM); + EndDrawing(); +} + +static void gc_demo_draw_x(Vector2 center, float size, Color color) { + DrawLineEx((Vector2){center.x - size, center.y - size}, + (Vector2){center.x + size, center.y + size}, 2.5f, color); + DrawLineEx((Vector2){center.x - size, center.y + size}, + (Vector2){center.x + size, center.y - size}, 2.5f, color); +} + +static Vector2 gc_demo_coin_center(int pos, int cell) { + return (Vector2){(float)(gc_coin_x(pos) * cell + cell / 2), + (float)(gc_coin_y(pos) * cell + cell / 2)}; +} + +static void gc_demo_render_trail(GuerrillaCheckers* env, GcDemoUi* ui) { + GcDemoTrail* trail = &ui->trail; + if (trail->side == GC_NONE) return; + int cell = env->client->cell; + + // Newly placed guerrilla stones get a small "just moved" dot, not a ring + // (rings mean "selectable" in the move hints). + for (int i = 0; i < trail->placed_count; i++) { + Vector2 center = {(float)((gc_g_x(trail->placed[i]) + 1) * cell), + (float)((gc_g_y(trail->placed[i]) + 1) * cell)}; + DrawCircleV(center, cell * 0.07f, GC_DEMO_TRAIL); + } + for (int i = 0; i < trail->captured_g_count; i++) { + Vector2 center = {(float)((gc_g_x(trail->captured_g[i]) + 1) * cell), + (float)((gc_g_y(trail->captured_g[i]) + 1) * cell)}; + gc_demo_draw_x(center, cell * 0.10f, GC_DEMO_G_GHOST); + } + for (int i = 0; i < trail->captured_coins_count; i++) { + gc_demo_draw_x(gc_demo_coin_center(trail->captured_coins[i], cell), + cell * 0.14f, GC_DEMO_C_CAPTURED); + } + + if (trail->coin_path_count > 0) { + // Ghost piece at the origin, lines through every square the coin + // visited, and a "just moved" dot on its final position. + DrawCircleV(gc_demo_coin_center(trail->coin_path[0], cell), cell * 0.28f, + GC_DEMO_C_ORIGIN); + for (int i = 0; i + 1 < trail->coin_path_count; i++) { + DrawLineEx(gc_demo_coin_center(trail->coin_path[i], cell), + gc_demo_coin_center(trail->coin_path[i + 1], cell), 3.0f, + GC_DEMO_C_GHOST); + } + if (trail->coin_path_count > 1) { + DrawCircleV(gc_demo_coin_center( + trail->coin_path[trail->coin_path_count - 1], cell), + cell * 0.07f, GC_DEMO_TRAIL); + } + } +} + +static void gc_demo_render_hints(GuerrillaCheckers* env, GcDemoUi* ui) { + int cell = env->client->cell; + if (env->game_over || ui->ctrl[env->player_to_move] != GC_CTRL_HUMAN) return; + + if (env->player_to_move == GC_GUERRILLA) { + if (ui->selected >= 0) { + // First placement pending: draw it as a ghost stone with a + // selection ring, and ghost dots on the legal second points. + Vector2 center = {(float)((gc_g_x(ui->selected) + 1) * cell), + (float)((gc_g_y(ui->selected) + 1) * cell)}; + DrawCircleV(center, cell * 0.20f, GC_DEMO_G_HINT); + DrawRing(center, cell * 0.20f, cell * 0.20f + 3.0f, 0, 360, 32, + GC_DEMO_SELECT); + for (int dir = 0; dir < 4; dir++) { + int action = ui->selected * 4 + dir; + if (!gc_action_is_legal(env, action)) continue; + int first; + int second; + gc_decode_guerrilla_action(action, &first, &second); + if (second < 0) continue; + DrawCircle((gc_g_x(second) + 1) * cell, (gc_g_y(second) + 1) * cell, + cell * 0.13f, GC_DEMO_G_HINT); + } + } else if (env->guerrilla_count > 0) { + // Skip the hints on the opening move: every point is legal and + // 49 dots just light the whole board up. + for (int pos = 0; pos < GC_G_CELLS; pos++) { + if (!gc_demo_pos_has_legal_action(env, pos)) continue; + DrawCircle((gc_g_x(pos) + 1) * cell, (gc_g_y(pos) + 1) * cell, + cell * 0.09f, GC_DEMO_G_HINT_DIM); + } + } + return; + } + + for (int src = 0; src < GC_COIN_CELLS; src++) { + if (src == ui->selected || !env->coin_cells[src]) continue; + if (!gc_demo_pos_has_legal_action(env, src)) continue; + Vector2 center = {(float)(gc_coin_x(src) * cell + cell / 2), + (float)(gc_coin_y(src) * cell + cell / 2)}; + DrawRing(center, cell * 0.30f, cell * 0.30f + 3.0f, 0, 360, 32, GC_DEMO_C_HINT); + } + if (ui->selected >= 0) { + Vector2 center = {(float)(gc_coin_x(ui->selected) * cell + cell / 2), + (float)(gc_coin_y(ui->selected) * cell + cell / 2)}; + DrawRing(center, cell * 0.30f, cell * 0.30f + 4.0f, 0, 360, 32, GC_DEMO_SELECT); + for (int dir = 0; dir < 4; dir++) { + if (!gc_action_is_legal(env, ui->selected * 4 + dir)) continue; + int dst = gc_coin_neighbor(ui->selected, dir); + int dx = gc_coin_x(dst) * cell; + int dy = gc_coin_y(dst) * cell; + DrawRectangle(dx + 3, dy + 3, cell - 6, cell - 6, GC_DEMO_C_SQUARE); + DrawCircle(dx + cell / 2, dy + cell / 2, cell * 0.13f, GC_DEMO_C_HINT); + } + } +} + +static void gc_demo_render_paused(Client* client) { + const char* paused = "PAUSED"; + int width = MeasureText(paused, 16); + int x = (client->width - width) / 2; + DrawRectangle(x - 10, 8, width + 20, 28, (Color){0, 0, 0, 170}); + DrawText(paused, x, 14, 16, RAYWHITE); +} + +static void gc_demo_render_supply(GuerrillaCheckers* env, int bar_y) { + const char* counts = TextFormat("GUERRILLAS %d", + GC_MAX_GUERRILLAS - env->guerrilla_count); + DrawText(counts, env->client->width - 96 - MeasureText(counts, 16), bar_y + 16, + 16, GC_DEMO_GUERRILLA); +} + +// Returns 1 when the MENU button was clicked. +static int gc_demo_render_play(GuerrillaCheckers* env, GcDemoUi* ui) { + Client* client = env->client; + int cell = client->cell; + BeginDrawing(); + ClearBackground(GC_DEMO_BG); + gc_render_board(env); + gc_demo_render_trail(env, ui); + gc_demo_render_hints(env, ui); + + int bar_y = GC_BOARD_H * cell; + if (env->game_over) { + Color winner_color = env->winner == GC_GUERRILLA ? GC_DEMO_GUERRILLA : GC_DEMO_COIN; + const char* winner = env->winner == GC_GUERRILLA ? "GUERRILLA WINS" : "COIN WINS"; + DrawRectangle(0, cell * 3 - 12, client->width, cell + 60, (Color){0, 0, 0, 170}); + DrawText(winner, (client->width - MeasureText(winner, 32)) / 2, + cell * 3 + 8, 32, winner_color); + const char* again = "CLICK ANYWHERE FOR MENU"; + DrawText(again, (client->width - MeasureText(again, 16)) / 2, + cell * 3 + 52, 16, GC_DEMO_TEXT); + DrawText(winner, 12, bar_y + 13, 22, winner_color); + } else { + int side = env->player_to_move; + const char* name = side == GC_GUERRILLA ? "GUERRILLA" : "COIN"; + const char* verb = ui->ctrl[side] != GC_CTRL_AI ? "TO MOVE" : + ui->paused ? "PAUSED" : "THINKING..."; + DrawText(TextFormat("%s %s", name, verb), 12, bar_y + 13, 22, + side == GC_GUERRILLA ? GC_DEMO_GUERRILLA : GC_DEMO_COIN); + } + + if (ui->paused) gc_demo_render_paused(client); + gc_demo_render_supply(env, bar_y); + int menu_clicked = gc_demo_button( + (Rectangle){(float)client->width - 84.0f, (float)bar_y + 8.0f, 72, 32}, "MENU", 0); + EndDrawing(); + return menu_clicked; +} + +// Returns 1 when the MENU button was clicked. +static int gc_demo_render_tourney(GuerrillaCheckers* env, GcDemoUi* ui) { + Client* client = env->client; + int cell = client->cell; + BeginDrawing(); + ClearBackground(GC_DEMO_BG); + gc_render_board(env); + + if (ui->paused) gc_demo_render_paused(client); + + // One condensed tally line: "#4 G (L3) 3 WINS (30%) C (L4) 9 WINS (70%)" + GcDemoTally* tally = &ui->tally; + float games = tally->games > 0 ? (float)tally->games : 1.0f; + int bar_y = GC_BOARD_H * cell; + int x = 12; + const char* seg = TextFormat("#%d", tally->games + 1); + DrawText(seg, x, bar_y + 16, 16, RAYWHITE); + x += MeasureText(seg, 16) + 14; + seg = TextFormat("G (L%s) %d WINS (%.0f%%)", + gc_demo_levels[ui->ai_level[GC_GUERRILLA]].label, tally->guerrilla_wins, + 100.0f * (float)tally->guerrilla_wins / games); + DrawText(seg, x, bar_y + 16, 16, GC_DEMO_GUERRILLA); + x += MeasureText(seg, 16) + 14; + seg = TextFormat("C (L%s) %d WINS (%.0f%%)", + gc_demo_levels[ui->ai_level[GC_COIN]].label, tally->coin_wins, + 100.0f * (float)tally->coin_wins / games); + DrawText(seg, x, bar_y + 16, 16, GC_DEMO_COIN); + x += MeasureText(seg, 16) + 14; + + const char* counts = TextFormat("G %d", GC_MAX_GUERRILLAS - env->guerrilla_count); + int counts_x = client->width - 96 - MeasureText(counts, 16); + if (counts_x > x) { + DrawText(counts, counts_x, bar_y + 16, 16, GC_DEMO_GUERRILLA); + } + int menu_clicked = gc_demo_button( + (Rectangle){(float)client->width - 84.0f, (float)bar_y + 8.0f, 72, 32}, "MENU", 0); + EndDrawing(); + return menu_clicked; +} + +static int gc_demo_level_action(GuerrillaCheckers* env, int level_index) { + const GcDemoLevel* level = &gc_demo_levels[level_index]; + if (level->opponent == GC_DEMO_NET_BOT) { + if (gc_demo_net_loaded[GC_DEMO_NET_ORIGINAL]) { + return gc_demo_net_action(env, GC_DEMO_NET_ORIGINAL); + } + level = &gc_demo_levels[GC_DEMO_LEVEL_MCTS_2K]; + } else if (level->opponent == GC_DEMO_CANDIDATE_BOT) { + if (gc_demo_net_loaded[GC_DEMO_NET_CANDIDATE]) { + return gc_demo_net_action(env, GC_DEMO_NET_CANDIDATE); + } + level = &gc_demo_levels[GC_DEMO_LEVEL_MCTS_2K]; + } + env->opponent = level->opponent; + env->mcts_iterations = level->mcts_iterations; + return gc_bot_action(env); +} + +static int gc_demo_level_model(int level_index) { + const GcDemoLevel* level = &gc_demo_levels[level_index]; + if (level->opponent == GC_DEMO_NET_BOT && + gc_demo_net_loaded[GC_DEMO_NET_ORIGINAL]) { + return GC_DEMO_NET_ORIGINAL; + } + if (level->opponent == GC_DEMO_CANDIDATE_BOT && + gc_demo_net_loaded[GC_DEMO_NET_CANDIDATE]) { + return GC_DEMO_NET_CANDIDATE; + } + return -1; +} + +static int gc_demo_observe_waiting(GuerrillaCheckers* env, int level_index) { + int model = gc_demo_level_model(level_index); + if (model < 0) return 0; + int waiting_side = env->player_to_move == GC_GUERRILLA ? GC_COIN : GC_GUERRILLA; + GcDemoNet* net = &gc_demo_nets[model][waiting_side]; + if (net->full_turn_state) { + (void)gc_demo_net_observe(env, model, waiting_side); + return 1; + } + return 0; +} + +static int gc_demo_bot_action(GuerrillaCheckers* env, GcDemoUi* ui) { + int waiting_side = env->player_to_move == GC_GUERRILLA ? GC_COIN : GC_GUERRILLA; + if (ui->ctrl[waiting_side] == GC_CTRL_AI) { + gc_demo_observe_waiting(env, ui->ai_level[waiting_side]); + } + return gc_demo_level_action(env, ui->ai_level[env->player_to_move]); +} + +static void demo(void) { + GuerrillaCheckers env = {0}; + env.num_agents = 2; + env.max_episode_length = 256; + env.render_fps = 60; + env.selfplay = 1; // the client drives both sides turn by turn + env.side_cfg = 0; + env.opponent = GC_BOT_GREEDY; + env.mcts_iterations = 2000; + env.mcts_exploration = GC_MCTS_DEFAULT_EXPLORATION; + env.mcts_rollout = GC_MCTS_ROLLOUT_GREEDY; + env.rng = 1234u; + + gc_demo_allocate(&env); + puf_reset(&env); + + gc_demo_net_loaded[GC_DEMO_NET_ORIGINAL] = + gc_demo_net_init(&gc_demo_nets[GC_DEMO_NET_ORIGINAL][GC_GUERRILLA], + GC_DEMO_NET_WEIGHTS) && + gc_demo_net_init(&gc_demo_nets[GC_DEMO_NET_ORIGINAL][GC_COIN], + GC_DEMO_NET_WEIGHTS); + + env.client = gc_make_client(&env); + SetExitKey(KEY_NULL); // ESC navigates to the menu instead of quitting + + GcDemoUi ui = {0}; + ui.mode = GC_UI_MENU; + ui.ctrl[GC_GUERRILLA] = GC_CTRL_HUMAN; + ui.ctrl[GC_COIN] = GC_CTRL_AI; + ui.ai_level[GC_GUERRILLA] = gc_demo_net_loaded[GC_DEMO_NET_ORIGINAL] ? + GC_DEMO_LEVEL_NET : GC_DEMO_LEVEL_MCTS_2K; + ui.ai_level[GC_COIN] = ui.ai_level[GC_GUERRILLA]; + ui.selected = GC_DEMO_NOOP; + + while (!WindowShouldClose()) { + if (ui.mode == GC_UI_MENU) { + gc_demo_render_menu(&env, &ui); + continue; + } + + if (IsKeyPressed(KEY_ESCAPE)) { + ui.mode = GC_UI_MENU; + gc_demo_render_menu(&env, &ui); + continue; + } + + if (IsKeyPressed(KEY_R)) { + memset(&ui.tally, 0, sizeof(ui.tally)); + gc_demo_start_game(&env, &ui); + } + + if (ui.mode == GC_UI_TOURNEY) { + if (IsKeyPressed(KEY_SPACE)) ui.paused = !ui.paused; + // Simulate as many moves as fit in the frame budget, then render + // the current position and the running tally. + double frame_end = GetTime() + GC_DEMO_TOURNEY_BUDGET; + while (!ui.paused && GetTime() < frame_end) { + if (env.game_over) { + ui.tally.games++; + if (env.winner == GC_GUERRILLA) ui.tally.guerrilla_wins++; + else ui.tally.coin_wins++; + env.rng ^= (unsigned int)(GetTime() * 1e6) | 1u; + puf_reset(&env); + gc_demo_net_reset(); + continue; + } + gc_demo_apply(&env, &ui, gc_demo_bot_action(&env, &ui)); + } + if (gc_demo_render_tourney(&env, &ui)) ui.mode = GC_UI_MENU; + continue; + } + + if (env.game_over) { + if (ui.over_wait > 0) { + ui.over_wait--; + } else if (IsMouseButtonPressed(MOUSE_LEFT_BUTTON)) { + ui.mode = GC_UI_MENU; + continue; // consume the click before the menu renders + } + if (gc_demo_render_play(&env, &ui)) ui.mode = GC_UI_MENU; + continue; + } + + // AI vs AI games can be paused like tournaments. + if (ui.ctrl[GC_GUERRILLA] == GC_CTRL_AI && ui.ctrl[GC_COIN] == GC_CTRL_AI && + IsKeyPressed(KEY_SPACE)) { + ui.paused = !ui.paused; + } + + int action = GC_DEMO_NOOP; + if (ui.ctrl[env.player_to_move] == GC_CTRL_HUMAN) { + int prev_selected = ui.selected; + action = gc_demo_human_action(&env, &ui.selected); + // The trail marks the opponent's last move; drop it as soon as + // the player starts their own. + if (ui.selected != prev_selected) { + memset(&ui.trail, 0, sizeof(ui.trail)); + } + } else if (ui.paused) { + // Hold the position until space is pressed again. + } else if (ui.ai_wait > 0) { + // Keep presenting "THINKING..." frames before the (blocking) search + // so the status stays visible while MCTS runs. + ui.ai_wait--; + } else { + action = gc_demo_bot_action(&env, &ui); + } + + if (action != GC_DEMO_NOOP) { + int waiting_side = env.player_to_move == GC_GUERRILLA ? + GC_COIN : GC_GUERRILLA; + if (ui.ctrl[env.player_to_move] == GC_CTRL_HUMAN && + ui.ctrl[waiting_side] == GC_CTRL_AI) { + gc_demo_observe_waiting(&env, ui.ai_level[waiting_side]); + } + gc_demo_apply(&env, &ui, action); + ui.selected = GC_DEMO_NOOP; + ui.ai_wait = GC_DEMO_AI_WAIT; + if (env.game_over) { + ui.over_wait = GC_DEMO_OVER_WAIT; + } else if (env.coin_must_capture && + ui.ctrl[GC_COIN] == GC_CTRL_HUMAN) { + // Capture chains continue with the same piece: keep it selected. + ui.selected = env.coin_previous_cell; + } + } + + if (gc_demo_render_play(&env, &ui)) ui.mode = GC_UI_MENU; + } + + gc_demo_free(&env); +} + +// --------------------------------------------------------------------------- +// Headless CLI tournament (--tournament [games]): every bot as Guerrilla +// plays every bot as COIN, then W-L, decisions per second, and Elo are +// reported per (bot, side) since the two roles play very differently. +// Never touches raylib. + +#define GC_CLI_MAX_ENTITIES (2 * GC_CLI_MAX_BOTS) + +typedef struct { + int wins; + int losses; + long decisions; + double seconds; +} GcCliStats; + +static double gc_cli_now(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1e-9; +} + +// Bradley-Terry strengths from the pairwise score matrix, reported on the +// Elo scale anchored at 1500 mean. A half-win prior per pairing keeps +// shut-out entries (0 wins) at a finite rating. Guerrilla entries only ever +// meet COIN entries; the bipartite graph is fully connected, which is all +// the fit needs. +static void gc_cli_elo(int entities, + double score[GC_CLI_MAX_ENTITIES][GC_CLI_MAX_ENTITIES], + int played[GC_CLI_MAX_ENTITIES][GC_CLI_MAX_ENTITIES], double* elo) { + double p[GC_CLI_MAX_ENTITIES]; + for (int e = 0; e < entities; e++) p[e] = 1.0; + + for (int iter = 0; iter < 1000; iter++) { + double next[GC_CLI_MAX_ENTITIES]; + for (int e = 0; e < entities; e++) { + double won = 0.0; + double denom = 0.0; + for (int f = 0; f < entities; f++) { + if (f == e || played[e][f] == 0) continue; + won += score[e][f] + 0.5; + denom += ((double)played[e][f] + 1.0) / (p[e] + p[f]); + } + next[e] = won / denom; + } + double log_mean = 0.0; + for (int e = 0; e < entities; e++) log_mean += log(next[e]); + double scale = exp(log_mean / entities); + for (int e = 0; e < entities; e++) p[e] = next[e] / scale; + } + + for (int e = 0; e < entities; e++) { + elo[e] = 1500.0 + 400.0 * log10(p[e]); + } +} + +static int gc_cli_tournament(int games, const char* candidate_path, + int compare_only) { + GuerrillaCheckers env = {0}; + env.num_agents = 2; + env.selfplay = 1; + env.mcts_exploration = GC_MCTS_DEFAULT_EXPLORATION; + env.mcts_rollout = GC_MCTS_ROLLOUT_GREEDY; + env.rng = 1u; + gc_demo_allocate(&env); + + gc_demo_net_loaded[GC_DEMO_NET_ORIGINAL] = + gc_demo_net_init(&gc_demo_nets[GC_DEMO_NET_ORIGINAL][GC_GUERRILLA], + GC_DEMO_NET_WEIGHTS) && + gc_demo_net_init(&gc_demo_nets[GC_DEMO_NET_ORIGINAL][GC_COIN], + GC_DEMO_NET_WEIGHTS); + if (candidate_path != NULL) { + gc_demo_net_loaded[GC_DEMO_NET_CANDIDATE] = + gc_demo_net_init(&gc_demo_nets[GC_DEMO_NET_CANDIDATE][GC_GUERRILLA], + candidate_path) && + gc_demo_net_init(&gc_demo_nets[GC_DEMO_NET_CANDIDATE][GC_COIN], + candidate_path); + if (!gc_demo_net_loaded[GC_DEMO_NET_CANDIDATE]) { + fprintf(stderr, "error: failed to load candidate %s\n", candidate_path); + gc_demo_free(&env); + return 1; + } + } + + int roster[GC_CLI_MAX_BOTS]; + int n_bots = 0; + if (compare_only) { + if (!gc_demo_net_loaded[GC_DEMO_NET_ORIGINAL] || + !gc_demo_net_loaded[GC_DEMO_NET_CANDIDATE]) { + fprintf(stderr, + "error: comparison requires both original and candidate models\n"); + gc_demo_free(&env); + return 1; + } + roster[n_bots++] = 2; // original PUFFER NN + roster[n_bots++] = GC_DEMO_CANDIDATE_LEVEL; + } else { + for (int i = 0; i < GC_DEMO_LEVEL_COUNT; i++) { + if (gc_demo_levels[i].opponent == GC_DEMO_NET_BOT && + !gc_demo_net_loaded[GC_DEMO_NET_ORIGINAL]) { + fprintf(stderr, "note: skipping %s (missing %s)\n", + gc_demo_levels[i].name, GC_DEMO_NET_WEIGHTS); + continue; + } + roster[n_bots++] = i; + } + if (gc_demo_net_loaded[GC_DEMO_NET_CANDIDATE]) { + roster[n_bots++] = GC_DEMO_CANDIDATE_LEVEL; + } + } + + // Entity e < n_bots is roster[e] playing Guerrilla; e >= n_bots is + // roster[e - n_bots] playing COIN. + int entities = 2 * n_bots; + static GcCliStats stats[GC_CLI_MAX_ENTITIES]; + static double score[GC_CLI_MAX_ENTITIES][GC_CLI_MAX_ENTITIES]; + static int played[GC_CLI_MAX_ENTITIES][GC_CLI_MAX_ENTITIES]; + + printf("Per-side round-robin: %d bots per side, %d games per pairing, %d games total\n\n", + n_bots, games, n_bots * n_bots * games); + + for (int a = 0; a < n_bots; a++) { + for (int b = 0; b < n_bots; b++) { + int g_bot = roster[a]; + int c_bot = roster[b]; + int g_entity = a; + int c_entity = n_bots + b; + int g_wins = 0; + int c_wins = 0; + for (int g = 0; g < games; g++) { + env.rng = (0x9E3779B9u * (unsigned int)((a * 16 + b) * 100003 + g)) | 1u; + puf_reset(&env); + gc_demo_net_reset(); + + while (!env.game_over) { + int guerrilla_to_move = env.player_to_move == GC_GUERRILLA; + int mover = guerrilla_to_move ? g_entity : c_entity; + int waiter = guerrilla_to_move ? c_entity : g_entity; + double wait_t0 = gc_cli_now(); + if (gc_demo_observe_waiting(&env, + guerrilla_to_move ? c_bot : g_bot)) { + stats[waiter].seconds += gc_cli_now() - wait_t0; + } + double t0 = gc_cli_now(); + int action = gc_demo_level_action(&env, + guerrilla_to_move ? g_bot : c_bot); + stats[mover].seconds += gc_cli_now() - t0; + stats[mover].decisions++; + gc_apply_action(&env, action); + gc_prepare_turn(&env); + } + + int winner = env.winner == GC_GUERRILLA ? + g_entity : c_entity; + int loser = winner == g_entity ? c_entity : g_entity; + stats[winner].wins++; + stats[loser].losses++; + score[winner][loser] += 1.0; + played[g_entity][c_entity]++; + played[c_entity][g_entity]++; + if (winner == g_entity) g_wins++; + else c_wins++; + fprintf(stderr, "\rG %-9s vs C %-9s %3d/%d ", + gc_demo_levels[g_bot].name, gc_demo_levels[c_bot].name, + g + 1, games); + } + fprintf(stderr, "\r"); + printf("G %-9s vs C %-9s %3d-%3d\n", gc_demo_levels[g_bot].name, + gc_demo_levels[c_bot].name, g_wins, c_wins); + fflush(stdout); + } + } + + double elo[GC_CLI_MAX_ENTITIES]; + gc_cli_elo(entities, score, played, elo); + + // Sort entities by Elo, best first. + int order[GC_CLI_MAX_ENTITIES]; + for (int e = 0; e < entities; e++) order[e] = e; + for (int a = 1; a < entities; a++) { + int v = order[a]; + int b = a - 1; + while (b >= 0 && elo[order[b]] < elo[v]) { + order[b + 1] = order[b]; + b--; + } + order[b + 1] = v; + } + + printf("\n%-9s %-9s %5s %5s %10s %5s\n", "AI", "SIDE", "W", "L", "SPS", "ELO"); + for (int a = 0; a < entities; a++) { + int e = order[a]; + int level = roster[e < n_bots ? e : e - n_bots]; + GcCliStats* s = &stats[e]; + double sps = s->seconds > 0.0 ? (double)s->decisions / s->seconds : 0.0; + printf("%-9s %-9s %5d %5d %10.0f %5.0f\n", gc_demo_levels[level].name, + e < n_bots ? "GUERRILLA" : "COIN", s->wins, s->losses, sps, elo[e]); + } + + gc_demo_free(&env); + return 0; +} + +int main(int argc, char** argv) { + if (argc > 1) { + if (strcmp(argv[1], "--tournament") == 0) { + int games = argc > 2 ? atoi(argv[2]) : 100; + if (games <= 0) games = 100; + const char* candidate_path = argc > 3 ? argv[3] : NULL; + return gc_cli_tournament(games, candidate_path, 0); + } + if (strcmp(argv[1], "--compare-candidate") == 0) { + int games = argc > 2 ? atoi(argv[2]) : 100; + if (games <= 0) games = 100; + if (argc <= 3) { + fprintf(stderr, "error: --compare-candidate requires a " + "candidate checkpoint\n"); + return 1; + } + return gc_cli_tournament(games, argv[3], 1); + } + fprintf(stderr, + "usage: %s [--tournament [games-per-pairing] [candidate.bin] | " + "--compare-candidate [games-per-pairing] candidate.bin]\n", + argv[0]); + return 1; + } + demo(); + return 0; +} diff --git a/ocean/guerrillacheckers/guerrillacheckers.h b/ocean/guerrillacheckers/guerrillacheckers.h new file mode 100644 index 0000000000..5ef546c98d --- /dev/null +++ b/ocean/guerrillacheckers/guerrillacheckers.h @@ -0,0 +1,947 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "pufferenv.h" + +// Guerrilla Checkers is an asymmetric hybrid of Checkers and Go. +// https://mischa-u.itch.io/guerrilla-checkers +// https://brtrain.wordpress.com/tag/guerrilla-checkers/ +// https://nestorgames.com/rulebooks/GUERRILLACHECKERS_EN.pdf + +#define GC_BOARD_W 8 +#define GC_BOARD_H 8 +#define GC_COIN_CELLS (GC_BOARD_W * GC_BOARD_H) +#define GC_G_W (GC_BOARD_W - 1) +#define GC_G_H (GC_BOARD_H - 1) +#define GC_G_CELLS (GC_G_W * GC_G_H) +#define GC_MAX_GUERRILLAS 66 +#define GC_ACTIONS 256 +#define GC_G_ACTIONS (GC_G_CELLS * 4) +#define GC_PASS_ACTION (GC_ACTIONS - 1) +#define GC_OBS_SIZE (GC_G_CELLS + GC_COIN_CELLS + 7) +#define GC_INVALID_ACTION_REWARD -1.0f +#define GC_MAX_BANKS 8 + +// Native 5c build contract. Keep these equal to the original Puffer policy +// contract so old and new checkpoints can be evaluated on identical inputs. +#define OBS_SIZE GC_OBS_SIZE +#define NUM_ATNS 1 +#define ACT_SIZES {GC_ACTIONS} + +typedef uint8_t obs_t; +typedef Env GuerrillaCheckers; + +enum { + GC_NONE = 0, + GC_GUERRILLA = 1, + GC_COIN = 2, +}; + +// opponent kwarg: which bot plays the non-agent side when selfplay == 0. +enum { + GC_BOT_RANDOM = 0, + GC_BOT_GREEDY = 1, + GC_BOT_MCTS = 2, +}; + +typedef struct Log { + float perf; // scored-side win rate; higher is better + float score; // scored-side terminal reward + float episode_return; + float episode_length; + float invalid_rate; // mean fraction of a side's decisions that were illegal + float games_as_guerrilla; + float wins_as_guerrilla; + float games_as_coin; + float wins_as_coin; + float slot_0_score; + float slot_1_score; + float slot_0_guerrilla_score; + float slot_0_guerrilla_n; + float slot_0_coin_score; + float slot_0_coin_n; + float hist_score; + float hist_n; + float hist_score_bank[GC_MAX_BANKS]; + float hist_n_bank[GC_MAX_BANKS]; + float n; +} Log; + +typedef struct Client Client; + +struct Env { + Agent agents[2]; + int num_agents; + Log log; + Client* client; + unsigned int rng; + int tag; + int boundary_reached; + + uint8_t coin_cells[GC_COIN_CELLS]; + uint8_t guerrilla_cells[GC_G_CELLS]; + int player_to_move; + int coin_must_capture; + int coin_previous_cell; + int guerrilla_previous_cell; + int guerrilla_cells_count; + int guerrilla_count; + int guerrilla_placed_this_turn; + int game_over; + int winner; + int tick; + int legal_count; + int invalid_this_episode; + int max_episode_length; + int render_fps; + + // selfplay == 1: two logical policy slots alternate turns. Slot 0 is the + // primary policy and slot 1 is the historical bank in tagged envs. Their + // side assignment is randomized each episode. selfplay == 0: slot 0 plays + // agent_side against a built-in bot inside puf_step. + int selfplay; + int side_cfg; // configured slot-0/agent side: 0 = random, else G / COIN + int agent_side; // resolved agent side for the current episode + int slot_for_side[3]; // indexed by GC_GUERRILLA / GC_COIN + int opponent; // GC_BOT_RANDOM / GC_BOT_GREEDY / GC_BOT_MCTS + int mcts_iterations; // MCTS search budget per move (opponent == GC_BOT_MCTS) + float mcts_exploration; // UCB1 exploration constant + int mcts_rollout; // GC_MCTS_ROLLOUT_RANDOM / GC_MCTS_ROLLOUT_GREEDY +}; + +static inline int gc_actor_slot(GuerrillaCheckers* env) { + if (env->selfplay && env->num_agents == 2) { + return env->slot_for_side[env->player_to_move]; + } + return 0; +} + +static inline unsigned char* gc_actor_mask(GuerrillaCheckers* env) { + return env->agents[gc_actor_slot(env)].action_mask; +} + +void puf_init(Env* env, Dict* kwargs) { + env->max_episode_length = (int)dict_get(kwargs, "max_episode_length"); + env->render_fps = (int)dict_get(kwargs, "render_fps"); + env->selfplay = (int)dict_get(kwargs, "selfplay"); + env->side_cfg = (int)dict_get(kwargs, "side"); + env->opponent = (int)dict_get(kwargs, "opponent"); + env->mcts_iterations = (int)dict_get(kwargs, "mcts_iterations"); + env->mcts_exploration = (float)dict_get(kwargs, "mcts_exploration"); + env->mcts_rollout = (int)dict_get(kwargs, "mcts_rollout"); + env->num_agents = env->selfplay ? 2 : 1; + env->agents[0].policy = 0; + env->agents[1].policy = 1; + env->rng = env->rng * 2654435761u + 12345u; +} + +void puf_log(Log* log, Dict* out) { + dict_set(out, "perf", log->perf); + dict_set(out, "score", log->score); + dict_set(out, "episode_return", log->episode_return); + dict_set(out, "episode_length", log->episode_length); + dict_set(out, "invalid_rate", log->invalid_rate); + dict_set(out, "games_as_guerrilla", log->games_as_guerrilla); + dict_set(out, "wins_as_guerrilla", log->wins_as_guerrilla); + dict_set(out, "games_as_coin", log->games_as_coin); + dict_set(out, "wins_as_coin", log->wins_as_coin); + dict_set(out, "slot_0_score", log->slot_0_score); + dict_set(out, "slot_1_score", log->slot_1_score); + dict_set(out, "slot_0_score_as_guerrilla", + log->slot_0_guerrilla_n > 0.0f ? + log->slot_0_guerrilla_score / log->slot_0_guerrilla_n : 0.0f); + dict_set(out, "slot_0_score_as_coin", + log->slot_0_coin_n > 0.0f ? + log->slot_0_coin_score / log->slot_0_coin_n : 0.0f); + dict_set(out, "hist_score", log->hist_score); + dict_set(out, "hist_n", log->hist_n); + dict_set(out, "hist_score_bank_0", log->hist_score_bank[0]); + dict_set(out, "hist_score_bank_1", log->hist_score_bank[1]); + dict_set(out, "hist_score_bank_2", log->hist_score_bank[2]); + dict_set(out, "hist_score_bank_3", log->hist_score_bank[3]); + dict_set(out, "hist_score_bank_4", log->hist_score_bank[4]); + dict_set(out, "hist_score_bank_5", log->hist_score_bank[5]); + dict_set(out, "hist_score_bank_6", log->hist_score_bank[6]); + dict_set(out, "hist_score_bank_7", log->hist_score_bank[7]); + dict_set(out, "hist_n_bank_0", log->hist_n_bank[0]); + dict_set(out, "hist_n_bank_1", log->hist_n_bank[1]); + dict_set(out, "hist_n_bank_2", log->hist_n_bank[2]); + dict_set(out, "hist_n_bank_3", log->hist_n_bank[3]); + dict_set(out, "hist_n_bank_4", log->hist_n_bank[4]); + dict_set(out, "hist_n_bank_5", log->hist_n_bank[5]); + dict_set(out, "hist_n_bank_6", log->hist_n_bank[6]); + dict_set(out, "hist_n_bank_7", log->hist_n_bank[7]); +} + +static inline unsigned int gc_rand(GuerrillaCheckers* env) { + return (unsigned int)rand_r(&env->rng); +} + +static inline int gc_coin_pos(int x, int y) { + return y * GC_BOARD_W + x; +} + +static inline int gc_coin_x(int pos) { + return pos % GC_BOARD_W; +} + +static inline int gc_coin_y(int pos) { + return pos / GC_BOARD_W; +} + +static inline int gc_g_pos(int x, int y) { + return y * GC_G_W + x; +} + +static inline int gc_g_x(int pos) { + return pos % GC_G_W; +} + +static inline int gc_g_y(int pos) { + return pos / GC_G_W; +} + +static inline int gc_valid_coin_xy(int x, int y) { + return x >= 0 && x < GC_BOARD_W && y >= 0 && y < GC_BOARD_H; +} + +static inline int gc_valid_g_xy(int x, int y) { + return x >= 0 && x < GC_G_W && y >= 0 && y < GC_G_H; +} + +static inline int gc_valid_g_pos(int pos) { + return pos >= 0 && pos < GC_G_CELLS; +} + +static int gc_coin_jump_cell(int src, int dst) { + int sx = gc_coin_x(src); + int sy = gc_coin_y(src); + int dx = gc_coin_x(dst) - sx; + int dy = gc_coin_y(dst) - sy; + int gx = sx + (dx < 0 ? -1 : 0); + int gy = sy + (dy < 0 ? -1 : 0); + return gc_g_pos(gx, gy); +} + +static int gc_coin_neighbor(int src, int dir) { + static const int dx[4] = {-1, 1, -1, 1}; + static const int dy[4] = {-1, -1, 1, 1}; + int x = gc_coin_x(src) + dx[dir]; + int y = gc_coin_y(src) + dy[dir]; + if (!gc_valid_coin_xy(x, y)) return -1; + return gc_coin_pos(x, y); +} + +static int gc_g_neighbor(int src, int dir) { + static const int dx[4] = {-1, 0, 0, 1}; + static const int dy[4] = {0, -1, 1, 0}; + int x = gc_g_x(src) + dx[dir]; + int y = gc_g_y(src) + dy[dir]; + if (!gc_valid_g_xy(x, y)) return -1; + return gc_g_pos(x, y); +} + +static int gc_adjacent_to_existing_guerrilla(GuerrillaCheckers* env, int pos) { + for (int dir = 0; dir < 4; dir++) { + int n = gc_g_neighbor(pos, dir); + if (n >= 0 && env->guerrilla_cells[n]) return 1; + } + return 0; +} + +static int gc_adjacent_to_previous_guerrilla(GuerrillaCheckers* env, int pos) { + if (env->guerrilla_previous_cell < 0) return 0; + for (int dir = 0; dir < 4; dir++) { + if (gc_g_neighbor(env->guerrilla_previous_cell, dir) == pos) return 1; + } + return 0; +} + +static int gc_can_place_guerrilla(GuerrillaCheckers* env, int dest) { + if (!gc_valid_g_pos(dest)) return 0; + if (env->guerrilla_count == 0) return 1; + if (env->guerrilla_cells[dest]) return 0; + if (env->guerrilla_placed_this_turn == 0) { + return gc_adjacent_to_existing_guerrilla(env, dest); + } + return gc_adjacent_to_previous_guerrilla(env, dest); +} + +static int gc_coin_can_capture(GuerrillaCheckers* env, int src) { + for (int dir = 0; dir < 4; dir++) { + int dst = gc_coin_neighbor(src, dir); + if (dst < 0 || env->coin_cells[dst]) continue; + int jump_cell = gc_coin_jump_cell(src, dst); + if (gc_valid_g_pos(jump_cell) && env->guerrilla_cells[jump_cell]) { + return 1; + } + } + return 0; +} + +static int gc_can_move_coin(GuerrillaCheckers* env, int src, int dst) { + if (src < 0 || src >= GC_COIN_CELLS || dst < 0 || dst >= GC_COIN_CELLS) return 0; + if (!env->coin_cells[src] || env->coin_cells[dst]) return 0; + + int dx = abs(gc_coin_x(dst) - gc_coin_x(src)); + int dy = abs(gc_coin_y(dst) - gc_coin_y(src)); + if (dx != 1 || dy != 1) return 0; + + int jump_cell = gc_coin_jump_cell(src, dst); + if (!gc_valid_g_pos(jump_cell)) return 0; + if (env->coin_must_capture) { + if (src != env->coin_previous_cell) return 0; + if (!env->guerrilla_cells[jump_cell]) return 0; + } + return 1; +} + +static int gc_guerrilla_has_second_placement(GuerrillaCheckers* env) { + if (env->player_to_move != GC_GUERRILLA || env->guerrilla_placed_this_turn != 1) { + return 0; + } + for (int dir = 0; dir < 4; dir++) { + int n = gc_g_neighbor(env->guerrilla_previous_cell, dir); + if (n >= 0 && !env->guerrilla_cells[n]) return 1; + } + return 0; +} + +static void gc_check_victory(GuerrillaCheckers* env) { + if (env->game_over) return; + + int coin_count = 0; + for (int i = 0; i < GC_COIN_CELLS; i++) { + coin_count += env->coin_cells[i] ? 1 : 0; + } + + if (coin_count == 0) { + env->game_over = 1; + env->winner = GC_GUERRILLA; + } else if (env->guerrilla_count == GC_MAX_GUERRILLAS || + (env->guerrilla_count > 0 && env->guerrilla_cells_count == 0)) { + env->game_over = 1; + env->winner = GC_COIN; + } else if (env->player_to_move == GC_GUERRILLA && env->guerrilla_placed_this_turn == 1) { + // Call unconditionally so the function is referenced in release builds + // (assert() is compiled out under -DNDEBUG); (void) silences the unused + // result there. + int has_second = gc_guerrilla_has_second_placement(env); + assert(has_second && + "Guerrilla Checkers reached a non-terminal state with no legal second placement"); + (void)has_second; + } +} + +static void gc_check_timeout(GuerrillaCheckers* env) { + if (env->game_over) return; + if (env->tick >= env->max_episode_length) { + // Undecided at the step limit. The Guerrilla's only win condition is to + // clear the board, so failing to do so in time is a Guerrilla loss (the + // same outcome as running out of pieces). Deciding by side rather than by + // whoever happens to be on-move keeps timeouts from always falling on the + // learner in bot mode (where player_to_move is always agent_side here). + env->game_over = 1; + env->winner = GC_COIN; + } +} + +static int gc_check_guerrilla_capture(GuerrillaCheckers* env) { + int captured = 0; + for (int c = 0; c < GC_COIN_CELLS; c++) { + if (!env->coin_cells[c]) continue; + + int x = gc_coin_x(c); + int y = gc_coin_y(c); + int enemy_count = 0; + int surrounded = 1; + for (int ox = -1; ox <= 0; ox++) { + for (int oy = -1; oy <= 0; oy++) { + int gx = x + ox; + int gy = y + oy; + if (!gc_valid_g_xy(gx, gy)) continue; + enemy_count++; + if (!env->guerrilla_cells[gc_g_pos(gx, gy)]) surrounded = 0; + } + } + if (enemy_count > 0 && surrounded) { + env->coin_cells[c] = 0; + captured++; + } + } + return captured; +} + +static int gc_place_guerrilla(GuerrillaCheckers* env, int dest) { + if (!gc_can_place_guerrilla(env, dest)) return -1; + + env->guerrilla_cells[dest] = 1; + env->guerrilla_cells_count++; + env->guerrilla_count++; + env->guerrilla_placed_this_turn++; + env->guerrilla_previous_cell = dest; + + int captures = gc_check_guerrilla_capture(env); + gc_check_victory(env); + + if (!env->game_over && env->guerrilla_placed_this_turn == 2) { + env->guerrilla_placed_this_turn = 0; + env->guerrilla_previous_cell = -1; + env->player_to_move = GC_COIN; + gc_check_victory(env); + } + return captures; +} + +static int gc_move_coin(GuerrillaCheckers* env, int src, int dst) { + if (!gc_can_move_coin(env, src, dst)) return -1; + + int jump_cell = gc_coin_jump_cell(src, dst); + int captured = env->guerrilla_cells[jump_cell] ? 1 : 0; + + env->coin_cells[src] = 0; + env->coin_cells[dst] = 1; + + if (captured) { + env->guerrilla_cells[jump_cell] = 0; + env->guerrilla_cells_count--; + if (env->guerrilla_cells_count == 0) { + env->game_over = 1; + env->winner = GC_COIN; + return 1; + } + + if (gc_coin_can_capture(env, dst)) { + env->coin_must_capture = 1; + env->coin_previous_cell = dst; + gc_check_victory(env); + return 1; + } + } + + env->coin_must_capture = 0; + env->coin_previous_cell = -1; + env->player_to_move = GC_GUERRILLA; + gc_check_victory(env); + return captured; +} + +static int gc_decode_guerrilla_action(int action, int* first, int* second) { + if (action < 0 || action >= GC_G_ACTIONS) return 0; + *first = action / 4; + static const int dirs[4] = {2, 3, 0, 1}; // down/right first so action 0 is valid at reset + *second = gc_g_neighbor(*first, dirs[action % 4]); + return 1; +} + +static int gc_action_is_legal(GuerrillaCheckers* env, int action) { + if (env->game_over) return 0; + if (action < 0 || action >= GC_ACTIONS) return 0; + + if (env->player_to_move == GC_GUERRILLA) { + int first; + int second; + if (!gc_decode_guerrilla_action(action, &first, &second)) return 0; + if (!gc_can_place_guerrilla(env, first)) return 0; + + if (second < 0) return 0; + if (!gc_valid_g_pos(second) || env->guerrilla_cells[second]) return 0; + return 1; + } + + int src = action / 4; + int dir = action % 4; + int dst = gc_coin_neighbor(src, dir); + return gc_can_move_coin(env, src, dst); +} + +static int gc_enumerate_legal(GuerrillaCheckers* env, int* out) { + int n = 0; + if (env->game_over) return 0; + + if (env->player_to_move == GC_GUERRILLA) { + static const int dirs[4] = {2, 3, 0, 1}; + for (int first = 0; first < GC_G_CELLS; first++) { + if (!gc_can_place_guerrilla(env, first)) continue; + for (int action_dir = 0; action_dir < 4; action_dir++) { + int second = gc_g_neighbor(first, dirs[action_dir]); + if (second < 0 || env->guerrilla_cells[second]) continue; + out[n++] = first * 4 + action_dir; + } + } + } else { + for (int src = 0; src < GC_COIN_CELLS; src++) { + if (!env->coin_cells[src]) continue; + for (int dir = 0; dir < 4; dir++) { + int dst = gc_coin_neighbor(src, dir); + if (!gc_can_move_coin(env, src, dst)) continue; + out[n++] = src * 4 + dir; + } + } + } + return n; +} + +static int gc_rebuild_action_mask(GuerrillaCheckers* env) { + int actor_slot = gc_actor_slot(env); + for (int slot = 0; slot < env->num_agents; slot++) { + unsigned char* mask = env->agents[slot].action_mask; + if (mask == NULL) continue; + memset(mask, 0, GC_ACTIONS * sizeof(unsigned char)); + // Match the standard Ocean turn-based convention: the waiting slot has + // one deterministic pass action. puf_step reads only the actor slot, so + // this action is ignored while keeping PPO and recurrent updates valid. + // Action 255 is otherwise impossible: it is the off-board down-right + // move from the bottom-right coin square. + if (env->selfplay && slot != actor_slot) { + mask[GC_PASS_ACTION] = 1; + } + } + + int legal[GC_ACTIONS]; + int count = gc_enumerate_legal(env, legal); + unsigned char* actor_mask = gc_actor_mask(env); + if (actor_mask != NULL) { + for (int i = 0; i < count; i++) { + actor_mask[legal[i]] = 1; + } + } + + env->legal_count = count; + return count; +} + +static void gc_apply_no_legal_loss(GuerrillaCheckers* env, int legal_count) { + if (!env->game_over && legal_count == 0) { + env->game_over = 1; + env->winner = env->player_to_move == GC_GUERRILLA ? GC_COIN : GC_GUERRILLA; + } +} + +static int gc_prepare_turn(GuerrillaCheckers* env) { + int legal_count = gc_rebuild_action_mask(env); + gc_apply_no_legal_loss(env, legal_count); + if (env->game_over) gc_rebuild_action_mask(env); + return !env->game_over; +} + +// Apply one legal action for the current player; returns the number of enemy +// pieces captured. Shared by the learner and the built-in bot. +static int gc_apply_action(GuerrillaCheckers* env, int action) { + if (env->player_to_move == GC_GUERRILLA) { + int first; + int second; + gc_decode_guerrilla_action(action, &first, &second); + int captures = gc_place_guerrilla(env, first); + if (!env->game_over && second >= 0) { + int second_captures = gc_place_guerrilla(env, second); + if (second_captures > 0) captures += second_captures; + } + return captures > 0 ? captures : 0; + } + int src = action / 4; + int dst = gc_coin_neighbor(src, action % 4); + int captures = gc_move_coin(env, src, dst); + return captures > 0 ? captures : 0; +} + +static inline void gc_add_capture_candidates(int pos, int* candidates, int* n) { + if (!gc_valid_g_pos(pos)) return; + int gx = gc_g_x(pos); + int gy = gc_g_y(pos); + for (int dx = 0; dx <= 1; dx++) { + for (int dy = 0; dy <= 1; dy++) { + int cx = gx + dx; + int cy = gy + dy; + if (!gc_valid_coin_xy(cx, cy)) continue; + int coin = gc_coin_pos(cx, cy); + int seen = 0; + for (int i = 0; i < *n; i++) { + if (candidates[i] == coin) { + seen = 1; + break; + } + } + if (!seen) candidates[(*n)++] = coin; + } + } +} + +// Immediate captures a candidate move would make, without mutating the board +// (used to rank moves for the greedy bot). +static int gc_action_capture_score(GuerrillaCheckers* env, int action) { + if (env->player_to_move == GC_GUERRILLA) { + int first; + int second; + gc_decode_guerrilla_action(action, &first, &second); + int candidates[8]; + int candidate_count = 0; + gc_add_capture_candidates(first, candidates, &candidate_count); + gc_add_capture_candidates(second, candidates, &candidate_count); + int count = 0; + // Captures are applied eagerly after every Guerrilla placement, so a + // reachable board cannot already contain a fully surrounded coin. Only + // coins whose corner set includes one of the newly placed stones can + // become newly surrounded by this action. + for (int i = 0; i < candidate_count; i++) { + int c = candidates[i]; + if (!env->coin_cells[c]) continue; + int x = gc_coin_x(c); + int y = gc_coin_y(c); + int enemy_count = 0; + int surrounded = 1; + for (int ox = -1; ox <= 0; ox++) { + for (int oy = -1; oy <= 0; oy++) { + int gx = x + ox; + int gy = y + oy; + if (!gc_valid_g_xy(gx, gy)) continue; + enemy_count++; + int gp = gc_g_pos(gx, gy); + int occupied = env->guerrilla_cells[gp] || gp == first || + (second >= 0 && gp == second); + if (!occupied) surrounded = 0; + } + } + if (enemy_count > 0 && surrounded) count++; + } + return count; + } + int src = action / 4; + int dst = gc_coin_neighbor(src, action % 4); + if (dst < 0) return 0; + int jump_cell = gc_coin_jump_cell(src, dst); + return gc_valid_g_pos(jump_cell) && env->guerrilla_cells[jump_cell] ? 1 : 0; +} + +static int gc_greedy_pick(GuerrillaCheckers* rng_env, GuerrillaCheckers* state, + const int* legal, int n) { + int best[GC_ACTIONS]; + int best_count = 0; + int best_score = -1; + for (int i = 0; i < n; i++) { + int score = gc_action_capture_score(state, legal[i]); + if (score > best_score) { + best_score = score; + best_count = 0; + best[best_count++] = legal[i]; + } else if (score == best_score) { + best[best_count++] = legal[i]; + } + } + return best[gc_rand(rng_env) % (unsigned int)best_count]; +} + +// MCTS opponent (opponent == GC_BOT_MCTS). Depends on the game primitives above. +#include "mcts.h" + +// Pick a move for the built-in opponent. +static int gc_bot_action(GuerrillaCheckers* env) { + if (env->opponent == GC_BOT_MCTS) return gc_mcts_action(env); + + int legal[GC_ACTIONS]; + int n = gc_enumerate_legal(env, legal); + assert(n > 0 && + "Guerrilla Checkers bot reached a non-terminal state with no legal moves"); + if (env->opponent != GC_BOT_GREEDY) { + return legal[gc_rand(env) % (unsigned int)n]; + } + return gc_greedy_pick(env, env, legal, n); +} + +static void gc_play_bot_turns(GuerrillaCheckers* env) { + while (!env->game_over && env->player_to_move != env->agent_side) { + if (!gc_prepare_turn(env)) return; + if (env->player_to_move == env->agent_side) return; + int action = gc_bot_action(env); + gc_apply_action(env, action); + } +} + +static void gc_compute_observations(GuerrillaCheckers* env) { + for (int slot = 0; slot < env->num_agents; slot++) { + uint8_t* observations = (uint8_t*)env->agents[slot].observations; + int idx = 0; + for (int i = 0; i < GC_G_CELLS; i++) { + observations[idx++] = env->guerrilla_cells[i] ? 1 : 0; + } + for (int i = 0; i < GC_COIN_CELLS; i++) { + observations[idx++] = env->coin_cells[i] ? 1 : 0; + } + + // Preserve the original checkpoint contract exactly: both slots see + // the same absolute board encoding, including the current side id. + observations[idx++] = env->player_to_move == GC_GUERRILLA ? 1 : 2; + observations[idx++] = env->coin_must_capture ? 1 : 0; + observations[idx++] = env->coin_previous_cell < 0 ? 0 : + (uint8_t)(env->coin_previous_cell + 1); + observations[idx++] = env->guerrilla_previous_cell < 0 ? 0 : + (uint8_t)(env->guerrilla_previous_cell + 1); + observations[idx++] = (uint8_t)env->guerrilla_placed_this_turn; + observations[idx++] = (uint8_t)env->guerrilla_count; + observations[idx++] = (uint8_t)env->guerrilla_cells_count; + assert(idx == GC_OBS_SIZE); + } +} + +static void gc_finish_step(GuerrillaCheckers* env, int actor_slot, + int score_side, float reward) { + gc_prepare_turn(env); + + if (env->game_over) { + float guerrilla_win = env->winner == GC_GUERRILLA ? 1.0f : 0.0f; + float coin_win = env->winner == GC_COIN ? 1.0f : 0.0f; + if (env->selfplay || score_side == GC_GUERRILLA) { + env->log.games_as_guerrilla += 1.0f; + env->log.wins_as_guerrilla += guerrilla_win; + } + if (env->selfplay || score_side == GC_COIN) { + env->log.games_as_coin += 1.0f; + env->log.wins_as_coin += coin_win; + } + + float scored_win; + if (env->selfplay) { + int winner_slot = env->slot_for_side[env->winner]; + int loser_slot = 1 - winner_slot; + *env->agents[winner_slot].rewards = 1.0f; + *env->agents[loser_slot].rewards = -1.0f; + *env->agents[0].terminals = 1.0f; + *env->agents[1].terminals = 1.0f; + + scored_win = winner_slot == 0 ? 1.0f : 0.0f; + env->log.slot_0_score += scored_win; + env->log.slot_1_score += 1.0f - scored_win; + if (env->slot_for_side[GC_GUERRILLA] == 0) { + env->log.slot_0_guerrilla_score += scored_win; + env->log.slot_0_guerrilla_n += 1.0f; + } else { + env->log.slot_0_coin_score += scored_win; + env->log.slot_0_coin_n += 1.0f; + } + if (env->tag > 0 && env->tag <= GC_MAX_BANKS) { + int bank = env->tag - 1; + env->log.hist_score += scored_win; + env->log.hist_n += 1.0f; + env->log.hist_score_bank[bank] += scored_win; + env->log.hist_n_bank[bank] += 1.0f; + env->boundary_reached = 1; + } + } else { + scored_win = env->winner == score_side ? 1.0f : 0.0f; + reward = scored_win ? 1.0f : -1.0f; + *env->agents[0].rewards = reward; + *env->agents[0].terminals = 1.0f; + } + + env->log.perf += scored_win; + env->log.score += env->selfplay ? scored_win : reward; + // Bounded [0,1]: fraction of this episode's decisions that were illegal. + env->log.invalid_rate += (float)env->invalid_this_episode / + (float)(env->tick > 0 ? env->tick : 1); + env->log.episode_length += (float)env->tick; + env->log.n += 1.0f; + } else { + *env->agents[actor_slot].rewards = fmaxf(-1.0f, fminf(1.0f, reward)); + } + + for (int slot = 0; slot < env->num_agents; slot++) { + env->log.episode_return += *env->agents[slot].rewards; + } + gc_compute_observations(env); +} + +void puf_reset(Env* env) { + memset(env->coin_cells, 0, sizeof(env->coin_cells)); + memset(env->guerrilla_cells, 0, sizeof(env->guerrilla_cells)); + + static const int starts[6][2] = { + {3, 2}, {2, 3}, {4, 3}, {3, 4}, {5, 4}, {4, 5}, + }; + for (int i = 0; i < 6; i++) { + env->coin_cells[gc_coin_pos(starts[i][0], starts[i][1])] = 1; + } + + env->player_to_move = GC_GUERRILLA; + env->coin_must_capture = 0; + env->coin_previous_cell = -1; + env->guerrilla_previous_cell = -1; + env->guerrilla_cells_count = 0; + env->guerrilla_count = 0; + env->guerrilla_placed_this_turn = 0; + env->game_over = 0; + env->winner = GC_NONE; + env->tick = 0; + env->legal_count = 0; + env->invalid_this_episode = 0; + for (int slot = 0; slot < env->num_agents; slot++) { + *env->agents[slot].rewards = 0.0f; + *env->agents[slot].terminals = 0.0f; + } + + if (env->selfplay) { + int guerrilla_slot; + if (env->side_cfg == GC_GUERRILLA) guerrilla_slot = 0; + else if (env->side_cfg == GC_COIN) guerrilla_slot = 1; + else guerrilla_slot = (int)(gc_rand(env) & 1u); + env->slot_for_side[GC_GUERRILLA] = guerrilla_slot; + env->slot_for_side[GC_COIN] = 1 - guerrilla_slot; + env->agent_side = GC_NONE; + } else if (env->side_cfg == GC_GUERRILLA || env->side_cfg == GC_COIN) { + env->agent_side = env->side_cfg; + } else { + env->agent_side = (gc_rand(env) & 1u) ? GC_GUERRILLA : GC_COIN; + } + // When the agent plays COIN, let the Guerrilla bot open so the first + // observation is on the agent's turn. + if (!env->selfplay) { + gc_play_bot_turns(env); + } + + gc_prepare_turn(env); + gc_compute_observations(env); +} + +static void gc_reset_after_terminal_step(GuerrillaCheckers* env) { + if (*env->agents[0].terminals != 1.0f) return; + float rewards[2] = {0}; + float terminals[2] = {0}; + for (int slot = 0; slot < env->num_agents; slot++) { + rewards[slot] = *env->agents[slot].rewards; + terminals[slot] = *env->agents[slot].terminals; + } + puf_reset(env); + for (int slot = 0; slot < env->num_agents; slot++) { + *env->agents[slot].rewards = rewards[slot]; + *env->agents[slot].terminals = terminals[slot]; + } +} + +void puf_step(Env* env) { + for (int slot = 0; slot < env->num_agents; slot++) { + *env->agents[slot].rewards = 0.0f; + *env->agents[slot].terminals = 0.0f; + } + + int actor = env->player_to_move; + int actor_slot = gc_actor_slot(env); + int action = (int)env->agents[actor_slot].actions[0]; + assert(env->legal_count > 0 && + "Guerrilla Checkers step reached a non-terminal state with no legal moves"); + int legal = gc_action_is_legal(env, action); + env->tick++; + + int score_side = env->selfplay ? GC_NONE : env->agent_side; + + if (!legal) { + // Policy rollout paths consume MY_ACTION_MASK before sampling, so a + // masked policy never lands here and invalid_rate stays ~0. As a safety + // net for unmasked/eval use, treat an + // illegal action as a negative no-op (not an instant forfeit) so stalling + // never beats legal play; the timeout below bounds any stall. + env->invalid_this_episode++; + gc_check_timeout(env); + gc_finish_step(env, actor_slot, score_side, GC_INVALID_ACTION_REWARD); + gc_reset_after_terminal_step(env); + return; + } + + int captures = gc_apply_action(env, action); + float reward = 0.0f; + if (captures > 0) { + reward += (actor == GC_GUERRILLA ? 0.05f : 0.03f) * (float)captures; + } + + // Bot mode: play the opponent's reply(ies) so the game returns to the + // agent's turn (or ends) before we score this step. + if (!env->selfplay) { + gc_play_bot_turns(env); + } + + gc_check_timeout(env); + gc_finish_step(env, actor_slot, score_side, reward); + gc_reset_after_terminal_step(env); +} + +void puf_close(Env* env) { + if (env->client != NULL) { + CloseWindow(); + free(env->client); + env->client = NULL; + } +} + +struct Client { + int width; + int height; + int cell; +}; + +static Client* gc_make_client(GuerrillaCheckers* env) { + Client* client = (Client*)calloc(1, sizeof(Client)); + client->cell = 72; + client->width = GC_BOARD_W * client->cell; + client->height = GC_BOARD_H * client->cell + 48; + InitWindow(client->width, client->height, "PufferLib Guerrilla Checkers"); + SetTargetFPS(env->render_fps); + return client; +} + +// Board squares and pieces only; callers wrap this in Begin/EndDrawing and +// draw their own status text (and, for the standalone client, move hints). +static void gc_render_board(GuerrillaCheckers* env) { + int cell = env->client->cell; + + for (int y = 0; y < GC_BOARD_H; y++) { + for (int x = 0; x < GC_BOARD_W; x++) { + Color square = ((x + y) & 1) ? (Color){54, 70, 78, 255} : (Color){36, 48, 54, 255}; + DrawRectangle(x * cell, y * cell, cell, cell, square); + DrawRectangleLines(x * cell, y * cell, cell, cell, (Color){15, 20, 24, 255}); + int pos = gc_coin_pos(x, y); + if (env->coin_cells[pos]) { + DrawCircle(x * cell + cell / 2, y * cell + cell / 2, cell * 0.28f, + (Color){232, 198, 83, 255}); + DrawCircleLines(x * cell + cell / 2, y * cell + cell / 2, cell * 0.28f, + (Color){96, 74, 22, 255}); + } + } + } + + for (int y = 0; y < GC_G_H; y++) { + for (int x = 0; x < GC_G_W; x++) { + int pos = gc_g_pos(x, y); + int cx = x * cell + cell; + int cy = y * cell + cell; + DrawCircle(cx, cy, 5.0f, (Color){104, 126, 132, 255}); + if (env->guerrilla_cells[pos]) { + DrawCircle(cx, cy, cell * 0.20f, (Color){206, 72, 72, 255}); + DrawCircleLines(cx, cy, cell * 0.20f, (Color){88, 28, 28, 255}); + } + } + } +} + +void puf_render(Env* env) { + if (IsKeyDown(KEY_ESCAPE)) exit(0); + if (env->client == NULL) env->client = gc_make_client(env); + + int cell = env->client->cell; + BeginDrawing(); + ClearBackground((Color){18, 24, 28, 255}); + gc_render_board(env); + + const char* side = env->player_to_move == GC_GUERRILLA ? "Guerrilla" : "COIN"; + const char* status = env->game_over ? + (env->winner == GC_GUERRILLA ? "Guerrilla wins" : + "COIN wins") : side; + DrawText(status, 12, GC_BOARD_H * cell + 14, 22, RAYWHITE); + DrawText(TextFormat("turn %d guerrillas %d/%d", env->tick, + env->guerrilla_count, GC_MAX_GUERRILLAS), + 250, GC_BOARD_H * cell + 16, 18, (Color){190, 204, 208, 255}); + EndDrawing(); +} diff --git a/ocean/guerrillacheckers/mcts.h b/ocean/guerrillacheckers/mcts.h new file mode 100644 index 0000000000..8ad7551160 --- /dev/null +++ b/ocean/guerrillacheckers/mcts.h @@ -0,0 +1,181 @@ +// MCTS (UCT) opponent for Guerrilla Checkers, ported from +// nico/guerrillacheckers/src/mcts.nim. Included by guerrillacheckers.h AFTER the +// GuerrillaCheckers struct and the game primitives it relies on are defined: +// gc_rand, gc_enumerate_legal, gc_apply_action, gc_action_capture_score, and the +// GC_* board/action constants. Not meant to be included standalone. +// +// Perfect-information single-tree UCT: wins are stored from the perspective of +// the player who moved into a node, so maximizing a child's win rate selects the +// best move for the player to move at its parent. One tree is built per decision. + +#ifndef GUERRILLACHECKERS_MCTS_H +#define GUERRILLACHECKERS_MCTS_H + +// MCTS simulation policy: uniform-random (faithful to the reference) or a +// greedy capture-maximizing playout (much stronger on the wide Guerrilla side, +// at a higher per-iteration cost). +enum { + GC_MCTS_ROLLOUT_RANDOM = 0, + GC_MCTS_ROLLOUT_GREEDY = 1, +}; + +// UCB1 exploration constant from the reference engine (~sqrt(2)/2). +#define GC_MCTS_DEFAULT_EXPLORATION 0.7f +typedef struct GcMctsNode { + int move; // action leading to this node (-1 at the root) + int parent; // pool index of parent (-1 at the root) + int first_child; // pool index of first child (-1) + int next_sibling; // pool index of next sibling (-1) + int child_count; + int player_just_moved; // GC_NONE at the root + int visits; + int select_count; // UCB1 log numerator; same value old siblings shared + double wins; +} GcMctsNode; + +// Collect legal actions on an MCTS clone and, if the side to move is stuck, +// resolve it as a loss (the env applies this in gc_prepare_turn, which MCTS +// does not call). Returns the number of legal actions. +static int gc_mcts_legal(GuerrillaCheckers* s, int* out) { + if (s->game_over) return 0; + int n = gc_enumerate_legal(s, out); + if (n == 0) { + gc_apply_no_legal_loss(s, n); + } + return n; +} + +static int gc_mcts_has_child(GcMctsNode* pool, int node, int move) { + for (int c = pool[node].first_child; c >= 0; c = pool[c].next_sibling) { + if (pool[c].move == move) return 1; + } + return 0; +} + +// UCB1 child selection. All children are legal here (perfect information), so we +// consider every child. In this tree all siblings share the same availability +// count, so the parent tracks the old per-sibling value. +static int gc_mcts_ucb_select(GcMctsNode* pool, int node, double exploration) { + int best = -1; + double best_score = -1.0e300; + double select_count = (double)pool[node].select_count; + for (int c = pool[node].first_child; c >= 0; c = pool[c].next_sibling) { + double visits = (double)pool[c].visits; + double score = pool[c].wins / visits + + exploration * sqrt(log(select_count) / visits); + if (score > best_score) { + best_score = score; + best = c; + } + } + pool[node].select_count += 1; + return best; +} + +// Choose a playout move on state `s`: uniform-random, or (greedy mode) the +// highest immediate-capture move with a random tie-break. `env` supplies rng and +// the rollout mode; `s` is the rollout clone whose side-to-move is scored. +static int gc_mcts_rollout_pick(GuerrillaCheckers* env, GuerrillaCheckers* s, + int* legal, int n) { + if (env->mcts_rollout != GC_MCTS_ROLLOUT_GREEDY) { + return legal[gc_rand(env) % (unsigned int)n]; + } + return gc_greedy_pick(env, s, legal, n); +} + +static void gc_mcts_init_node(GcMctsNode* n, int move, int parent, int sibling, + int player_just_moved) { + n->move = move; + n->parent = parent; + n->first_child = -1; + n->next_sibling = sibling; + n->child_count = 0; + n->player_just_moved = player_just_moved; + n->visits = 0; + n->select_count = 1; + n->wins = 0.0; +} + +static int gc_mcts_action(GuerrillaCheckers* env) { + int root_legal[GC_ACTIONS]; + int root_n = gc_enumerate_legal(env, root_legal); + if (root_n <= 1) return root_n == 1 ? root_legal[0] : 0; // nothing to search + + int itermax = env->mcts_iterations > 0 ? env->mcts_iterations : 1; + double exploration = env->mcts_exploration > 0.0f ? + (double)env->mcts_exploration : (double)GC_MCTS_DEFAULT_EXPLORATION; + + // Each iteration expands at most one node, so itermax + 1 nodes suffice. + GcMctsNode* pool = (GcMctsNode*)malloc((size_t)(itermax + 1) * sizeof(GcMctsNode)); + gc_mcts_init_node(&pool[0], -1, -1, -1, GC_NONE); + int node_count = 1; + + for (int iter = 0; iter < itermax; iter++) { + // Clone only the board state; MCTS never touches obs/mask/reward buffers. + GuerrillaCheckers s = *env; + memset(s.agents, 0, sizeof(s.agents)); + s.client = NULL; + + int node = 0; + int legal[GC_ACTIONS]; + int n = gc_mcts_legal(&s, legal); + + // Select: descend while the node is fully expanded and non-terminal. + while (n > 0) { + int fully_expanded = pool[node].child_count == n; + if (!fully_expanded) break; + node = gc_mcts_ucb_select(pool, node, exploration); + gc_apply_action(&s, pool[node].move); + n = gc_mcts_legal(&s, legal); + } + + // Expand: add one random untried move. + if (n > 0) { + int untried[GC_ACTIONS]; + int un = 0; + for (int i = 0; i < n; i++) { + if (!gc_mcts_has_child(pool, node, legal[i])) untried[un++] = legal[i]; + } + int m = untried[gc_rand(env) % (unsigned int)un]; + int player = s.player_to_move; + gc_apply_action(&s, m); + int child = node_count++; + gc_mcts_init_node(&pool[child], m, node, pool[node].first_child, player); + pool[node].first_child = child; + pool[node].child_count++; + node = child; + } + + // Simulate: rollout to a terminal state (random or greedy playout). + while (!s.game_over) { + int rollout[GC_ACTIONS]; + int rn = gc_mcts_legal(&s, rollout); + if (rn == 0) break; // game_over was set by gc_mcts_legal + gc_apply_action(&s, gc_mcts_rollout_pick(env, &s, rollout, rn)); + } + + // Backpropagate the terminal result to the root. + for (int bn = node; ; bn = pool[bn].parent) { + pool[bn].visits += 1; + if (pool[bn].player_just_moved != GC_NONE && + s.winner == pool[bn].player_just_moved) { + pool[bn].wins += 1.0; + } + if (pool[bn].parent < 0) break; + } + } + + // Best move = most-visited child of the root. + int best_move = root_legal[0]; + int best_visits = -1; + for (int c = pool[0].first_child; c >= 0; c = pool[c].next_sibling) { + if (pool[c].visits > best_visits) { + best_visits = pool[c].visits; + best_move = pool[c].move; + } + } + free(pool); + return best_move; +} + +#endif // GUERRILLACHECKERS_MCTS_H diff --git a/resources/guerrillacheckers/guerrillacheckers_weights.bin b/resources/guerrillacheckers/guerrillacheckers_weights.bin new file mode 100644 index 0000000000..02698875f2 Binary files /dev/null and b/resources/guerrillacheckers/guerrillacheckers_weights.bin differ