diff --git a/README.md b/README.md index 65de4694..a32b9f11 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,8 @@ var eval = braintrust.evalBuilder() (expected, result) -> expected.equals(result) ? 1.0 : 0.0)) .build(); var result = eval.run(); +// TODO: document the concurrency contract - cases run 10-at-a-time by default; +// task/scorers must be thread-safe. See Eval.Builder#maxConcurrency and #executor. System.out.println("\n\n" + result.createReportString()); ``` diff --git a/braintrust-sdk/src/main/java/dev/braintrust/config/BraintrustConfig.java b/braintrust-sdk/src/main/java/dev/braintrust/config/BraintrustConfig.java index 5a34ff31..c17c2df7 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/config/BraintrustConfig.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/config/BraintrustConfig.java @@ -64,6 +64,13 @@ public final class BraintrustConfig extends BaseConfig { /** Custom X509 trust manager for OTLP exporter. Builder-only field, not backed by envars. */ private final X509TrustManager x509TrustManager; + /** + * Maximum number of eval cases the remote eval devserver evaluates concurrently. Matches {@link + * dev.braintrust.eval.Eval#DEFAULT_MAX_CONCURRENCY}. + */ + private final int devserverMaxConcurrency = + getConfig("BRAINTRUST_DEVSERVER_MAX_CONCURRENCY", 10); + /** CORS origins to allow when running remote eval devserver */ private final String devserverCorsOriginWhitelistCsv = getConfig( @@ -260,6 +267,12 @@ public Builder devserverCorsOriginWhitelistCsv(String csv) { return this; } + public Builder devserverMaxConcurrency(int maxConcurrency) { + envOverrides.put( + "BRAINTRUST_DEVSERVER_MAX_CONCURRENCY", String.valueOf(maxConcurrency)); + return this; + } + public BraintrustConfig build() { return new BraintrustConfig(envOverrides, sslContext, x509TrustManager); } diff --git a/braintrust-sdk/src/main/java/dev/braintrust/devserver/Devserver.java b/braintrust-sdk/src/main/java/dev/braintrust/devserver/Devserver.java index 3618fd70..287598d8 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/devserver/Devserver.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/devserver/Devserver.java @@ -28,6 +28,7 @@ import java.nio.charset.StandardCharsets; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; import java.util.function.Function; @@ -87,6 +88,20 @@ public class Devserver { private final @Nullable String orgName; private final Map> evals; private @Nullable HttpServer server; + + /** Threads for HTTP request handling. */ + private @Nullable ExecutorService executor; + + /** + * Threads that eval cases run on, kept separate from the HTTP pool. Playground runs submit + * cases here while occupying an HTTP thread, so sharing a bounded pool with request handling + * would deadlock. Experiment snapshots keep using it after their response has been sent, so it + * outlives individual requests. + */ + private @Nullable ExecutorService evalExecutor; + + private final int maxConcurrency; + private final @Nullable Consumer traceBuilderHook; private final @Nullable Consumer configBuilderHook; @@ -101,6 +116,14 @@ private Devserver(Builder builder) { this.orgName = builder.orgName; this.traceBuilderHook = builder.traceBuilderHook; this.configBuilderHook = builder.configBuilderHook; + this.maxConcurrency = + builder.maxConcurrency != null + ? builder.maxConcurrency + : config.devserverMaxConcurrency(); + if (maxConcurrency < 1) { + throw new IllegalArgumentException( + "maxConcurrency must be at least 1, got " + maxConcurrency); + } Map> evalMap = new HashMap<>(); for (RemoteEval eval : builder.evals) { if (evalMap.containsKey(eval.getName())) { @@ -130,7 +153,9 @@ public synchronized void start() throws IOException { } server = HttpServer.create(new InetSocketAddress(host, port), 0); - server.setExecutor(Executors.newCachedThreadPool()); + executor = Executors.newCachedThreadPool(); + server.setExecutor(executor); + evalExecutor = createEvalExecutor(maxConcurrency); server.createContext("/", withCors(this::handleHealthCheck)); server.createContext("/list", withCors(this::handleList)); @@ -146,10 +171,34 @@ public synchronized void stop() { if (server != null) { server.stop(0); server = null; + // Lets evals still running in the background finish, but rejects new work. A snapshot + // that is mid-run when the server stops will abort once it tries to submit its next + // case. + if (executor != null) { + executor.shutdown(); + executor = null; + } + if (evalExecutor != null) { + evalExecutor.shutdown(); + evalExecutor = null; + } log.info("Braintrust dev server stopped"); } } + /** A fixed pool of daemon threads for eval cases, sized to {@code maxConcurrency}. */ + private static ExecutorService createEvalExecutor(int maxConcurrency) { + var counter = new java.util.concurrent.atomic.AtomicInteger(); + return Executors.newFixedThreadPool( + maxConcurrency, + r -> { + var thread = + new Thread(r, "braintrust-devserver-eval-" + counter.incrementAndGet()); + thread.setDaemon(true); + return thread; + }); + } + private void handleHealthCheck(HttpExchange exchange) throws IOException { if (!"GET".equals(exchange.getRequestMethod())) { sendResponse(exchange, 405, "text/plain", "Method Not Allowed"); @@ -417,134 +466,152 @@ private void handleStreamingEval( final var braintrustParent = parentInfo.braintrustParent(); final var braintrustGeneration = parentInfo.generation(); - // NOTE: this code is serial but written in a thread-safe manner to support - // concurrent dataset fetching and eval execution - extractDataset( - request, - apiClient, - eval.getInputConverter(), - eval.getOutputConverter()) - .forEach( + // Cases are evaluated concurrently on the eval pool. The cursor is drained by + // this (request) thread, which only ever waits for permits and never runs a case + // itself, so it cannot deadlock against the pool. + var casePool = this.evalExecutor; + var caseContext = Context.current(); + var drainError = + ConcurrentCases.drain( + extractDataset( + request, + apiClient, + eval.getInputConverter(), + eval.getOutputConverter()) + .openCursor(), + casePool != null ? casePool : Runnable::run, + maxConcurrency, datasetCase -> { - var evalSpan = - tracer.spanBuilder("eval") - .setNoParent() - .setSpanKind(SpanKind.CLIENT) - .setAttribute( - PARENT, - braintrustParent.toParentValue()) - .startSpan(); - Context evalContext = Context.current().with(evalSpan); - evalContext = - BraintrustContext.setParentInBaggage( - evalContext, - braintrustParent.type(), - braintrustParent.id()); - // Make the eval context (with span and baggage) current - try (var rootScope = evalContext.makeCurrent()) { - final TaskResult taskResult; - { // run task - var taskSpan = tracer.spanBuilder("task").startSpan(); - try (var unused = - Context.current() - .with(taskSpan) - .makeCurrent()) { - var task = eval.getTask(); - try { - taskResult = - task.apply( - datasetCase, mergedParameters); - } catch (Exception e) { - taskSpan.setStatus( - StatusCode.ERROR, e.getMessage()); - taskSpan.recordException(e); - taskSpan.end(); - evalSpan.setStatus( - StatusCode.ERROR, e.getMessage()); - log.debug( - "Task threw exception for input: " - + datasetCase.input(), - e); - // Set eval span attributes so Braintrust can - // resolve the trace - setEvalSpanAttributesForError( - evalSpan, - braintrustParent, - braintrustGeneration, - datasetCase); - // Send progress event even on error so the - // Playground can link to the trace + try (var caseScope = caseContext.makeCurrent()) { + var evalSpan = + tracer.spanBuilder("eval") + .setNoParent() + .setSpanKind(SpanKind.CLIENT) + .setAttribute( + PARENT, + braintrustParent.toParentValue()) + .startSpan(); + Context evalContext = Context.current().with(evalSpan); + evalContext = + BraintrustContext.setParentInBaggage( + evalContext, + braintrustParent.type(), + braintrustParent.id()); + // Make the eval context (with span and baggage) current + try (var rootScope = evalContext.makeCurrent()) { + final TaskResult taskResult; + { // run task + var taskSpan = + tracer.spanBuilder("task").startSpan(); + try (var unused = + Context.current() + .with(taskSpan) + .makeCurrent()) { + var task = eval.getTask(); + try { + taskResult = + task.apply( + datasetCase, + mergedParameters); + } catch (Exception e) { + taskSpan.setStatus( + StatusCode.ERROR, e.getMessage()); + taskSpan.recordException(e); + taskSpan.end(); + evalSpan.setStatus( + StatusCode.ERROR, e.getMessage()); + log.debug( + "Task threw exception for input: " + + datasetCase.input(), + e); + // Set eval span attributes so Braintrust + // can + // resolve the trace + setEvalSpanAttributesForError( + evalSpan, + braintrustParent, + braintrustGeneration, + datasetCase); + // Send progress event even on error so the + // Playground can link to the trace + sendProgressEvent( + os, + evalSpan.getSpanContext() + .getSpanId(), + datasetCase.origin(), + eval.getName(), + null); + // run scoreForTaskException on each scorer + List> allScorersForError = + new ArrayList<>(eval.getScorers()); + allScorersForError.addAll(remoteScorers); + for (var scorer : allScorersForError) { + runScoreForTaskException( + tracer, + evalSpan, + braintrustParent, + braintrustGeneration, + scorer, + e, + datasetCase, + scoresByName); + } + return; + } + // Send progress event for task completion sendProgressEvent( os, evalSpan.getSpanContext().getSpanId(), datasetCase.origin(), eval.getName(), - null); - // run scoreForTaskException on each scorer - List> allScorersForError = - new ArrayList<>(eval.getScorers()); - allScorersForError.addAll(remoteScorers); - for (var scorer : allScorersForError) { - runScoreForTaskException( - tracer, - evalSpan, - braintrustParent, - braintrustGeneration, - scorer, - e, - datasetCase, - scoresByName); - } - return; + taskResult.result()); + setTaskSpanAttributes( + taskSpan, + braintrustParent, + braintrustGeneration, + datasetCase, + taskResult); + } finally { + taskSpan.end(); } - // Send progress event for task completion - sendProgressEvent( - os, - evalSpan.getSpanContext().getSpanId(), - datasetCase.origin(), - eval.getName(), - taskResult.result()); - setTaskSpanAttributes( - taskSpan, + // setting eval span attributes here because we need + // the + // task output + setEvalSpanAttributes( + evalSpan, braintrustParent, braintrustGeneration, datasetCase, taskResult); - } finally { - taskSpan.end(); } - // setting eval span attributes here because we need the - // task output - setEvalSpanAttributes( - evalSpan, - braintrustParent, - braintrustGeneration, - datasetCase, - taskResult); - } - // run scorers - one score span per scorer - // Combine local scorers from RemoteEval with remote scorers - // from request - List> allScorers = - new ArrayList<>(eval.getScorers()); - allScorers.addAll(remoteScorers); - for (var scorer : allScorers) { - runScorer( - tracer, - evalSpan, - braintrustParent, - braintrustGeneration, - scorer, - taskResult, - scoresByName); + // run scorers - one score span per scorer + // Combine local scorers from RemoteEval with remote + // scorers + // from request + List> allScorers = + new ArrayList<>(eval.getScorers()); + allScorers.addAll(remoteScorers); + for (var scorer : allScorers) { + runScorer( + tracer, + evalSpan, + braintrustParent, + braintrustGeneration, + scorer, + taskResult, + scoresByName); + } + } catch (IOException e) { + throw new RuntimeException( + "Failed to send progress event", e); + } finally { + evalSpan.end(); } - } catch (IOException e) { - throw new RuntimeException( - "Failed to send progress event", e); - } finally { - evalSpan.end(); } }); + if (drainError != null) { + throw new RuntimeException("Failed to evaluate dataset", drainError); + } // Aggregate scores Map scoreSummaries = new LinkedHashMap<>(); @@ -593,11 +660,15 @@ private void handleStreamingEval( /** * Handles an experiment "snapshot" run: a remote eval triggered as an Experiment from the UI * (no playground parent). Rather than re-implementing experiment creation and span emission, it - * builds a first-class {@link Eval} and runs it synchronously — so snapshots get the exact same - * behavior as a normal {@code Eval.run()} (experiment creation with {@code ensure_new}, dataset - * id/version linkage for Braintrust-backed datasets, standard span shape). When the run - * completes it streams a single {@code summary} (with the created experiment's id/name/url) and - * a {@code done} event. + * builds a first-class {@link Eval} — so snapshots get the exact same behavior as a normal + * {@code Eval.run()} (experiment creation with {@code ensure_new}, dataset id/version linkage + * for Braintrust-backed datasets, standard span shape). + * + *

The eval is started with {@link Eval#start()} rather than run to completion: as soon as + * the experiment exists this streams a single {@code summary} (with the created experiment's + * id/name/url) and a {@code done} event, then returns while the cases keep evaluating on the + * server's executor. Errors creating the experiment surface to the caller; failures once the + * run is underway are logged by the eval. * *

Unlike playground runs, snapshots do not stream per-case {@code progress} events: the user * is handed the experiment link and views results in the experiment UI. @@ -619,10 +690,7 @@ private void handleExperimentSnapshot( List> allScorers = new ArrayList<>(eval.getScorers()); allScorers.addAll(remoteScorers); - // TODO: when async evals are supported, simply begin the eval and hand back the link to the - // user and finish eval in the background - - var evalResult = + var evalBuilder = Eval.builder() .name(experimentName) .config(braintrust.config()) @@ -643,9 +711,16 @@ private void handleExperimentSnapshot( : request.getParameters()) // Each snapshot run should produce a distinct experiment even if a prior // run used the same name (the backend dedupes the name on conflict). - .ensureNew(true) - .build() - .run(); + .ensureNew(true); + + // Run cases on the eval pool so they outlive this request, and hand the experiment link + // back as soon as the experiment exists rather than waiting for the run to finish. + evalBuilder.maxConcurrency(maxConcurrency); + var pool = this.evalExecutor; + if (pool != null) { + evalBuilder.executor(pool); + } + var evalResult = evalBuilder.build().start(); // Snapshots don't stream per-scorer progress. The scores are recorded on the experiment // and visible via the experiment link. @@ -844,7 +919,12 @@ private void recordScores( } Map scorerScores = new LinkedHashMap<>(); for (Score score : scores) { - scoresByName.computeIfAbsent(score.name(), k -> new ArrayList<>()).add(score.value()); + // Cases score concurrently. computeIfAbsent is atomic but the add() is not, so the + // list itself has to be synchronized. + scoresByName + .computeIfAbsent( + score.name(), k -> Collections.synchronizedList(new ArrayList<>())) + .add(score.value()); scorerScores.put(score.name(), score.value()); } setScoreSpanAttributes( @@ -853,7 +933,10 @@ private void recordScores( private void sendSSEEvent(OutputStream os, String eventType, String data) throws IOException { String event = "event: " + eventType + "\n" + "data: " + data + "\n\n"; - synchronized (this) { + // Lock the stream, not the server: concurrent cases of one run must not interleave a + // partial event, but concurrent *requests* write to different streams and shouldn't + // serialize against each other (or against the synchronized start()/stop()). + synchronized (os) { os.write(event.getBytes(StandardCharsets.UTF_8)); } } @@ -1378,6 +1461,7 @@ public static class Builder { private @Nullable Consumer traceBuilderHook = null; private @Nullable Consumer configBuilderHook = null; + private @Nullable Integer maxConcurrency = null; public Devserver build() { if (evals.isEmpty()) { @@ -1394,6 +1478,20 @@ public Builder config(BraintrustConfig config) { return this; } + /** + * Sets the maximum number of eval cases evaluated concurrently, for both playground runs + * and experiment snapshots. Defaults to {@link BraintrustConfig#devserverMaxConcurrency()} + * ({@code BRAINTRUST_DEVSERVER_MAX_CONCURRENCY}, itself defaulting to 10). + * + *

Cases run concurrently, so a {@link RemoteEval}'s task and scorers must be thread-safe + * — which {@code RemoteEval} already requires. Pass {@code 1} to evaluate cases one at a + * time. + */ + public Builder maxConcurrency(int maxConcurrency) { + this.maxConcurrency = maxConcurrency; + return this; + } + public Builder registerEval(RemoteEval eval) { this.evals.add(eval); return this; diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/Classifier.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/Classifier.java index e37b3e3e..d6e476cb 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/Classifier.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/Classifier.java @@ -14,6 +14,8 @@ * @param type of the input data * @param type of the output data */ +// TODO: document the concurrency contract - Eval invokes this from multiple threads +// (see Eval.Builder#maxConcurrency), so implementations must be thread-safe. public interface Classifier { String INVALID_CLASSIFICATION_MESSAGE = "When returning structured classifier results, each classification must be a non-empty" diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/ConcurrentCases.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/ConcurrentCases.java new file mode 100644 index 00000000..26c3356d --- /dev/null +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/ConcurrentCases.java @@ -0,0 +1,87 @@ +package dev.braintrust.eval; + +import java.util.concurrent.Executor; +import java.util.concurrent.Semaphore; +import java.util.function.Consumer; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; + +/** + * Runs the cases of a dataset concurrently. + * + *

This is shared machinery behind {@link Eval} and the remote-eval devserver rather than an API + * for SDK users; it is public only because those two live in different packages. + */ +@Slf4j +public final class ConcurrentCases { + private ConcurrentCases() {} + + /** + * Drains {@code cursor} on the calling thread, running {@code caseConsumer} for each case on + * {@code executor}, with at most {@code maxConcurrency} cases in flight. Blocks until every + * case that was submitted has finished, then closes the cursor. + * + *

The drain is deliberately single-threaded: {@link Dataset.Cursor} is + * {@code @NotThreadSafe} and its {@code next()} may make network calls, so one thread pulls + * cases and fans them out. The calling thread only ever waits for permits — it never runs a + * case itself, so passing an {@code executor} that this thread belongs to cannot deadlock. + * + *

A throw from {@code caseConsumer} is contained to its own case and logged; it does not + * abort the remaining cases. Callers that need to record per-case outcomes should do so inside + * {@code caseConsumer}. This method does not propagate the caller's {@link + * io.opentelemetry.context.Context} onto worker threads — wrap {@code caseConsumer} if you need + * that. + * + * @return the error that aborted the drain (for example a failure fetching the next page of a + * dataset), or null if every case was submitted + */ + public static @Nullable Throwable drain( + Dataset.Cursor cursor, + Executor executor, + int maxConcurrency, + Consumer caseConsumer) { + var inFlight = new Semaphore(maxConcurrency); + Throwable fatal = null; + try (cursor) { + for (var next = cursor.next(); next.isPresent(); next = cursor.next()) { + var item = next.get(); + inFlight.acquire(); + try { + executor.execute( + () -> { + try { + caseConsumer.accept(item); + } catch (Throwable t) { + // Contain the failure to this case: one bad case must not + // abort the rest of the run. + log.warn("Eval case failed", t); + } finally { + inFlight.release(); + } + }); + } catch (RuntimeException e) { + // e.g. RejectedExecutionException from a caller-supplied executor. Release the + // permit we took so the drain below can't hang. + inFlight.release(); + throw e; + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + fatal = e; + } catch (Throwable t) { + fatal = t; + } finally { + // Wait for every in-flight case, including when the drain above aborted. + try { + inFlight.acquire(maxConcurrency); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + if (fatal == null) { + fatal = e; + } + } + } + return fatal; + } +} diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/Dataset.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/Dataset.java index 671ff612..44af7060 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/Dataset.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/Dataset.java @@ -32,6 +32,8 @@ default void forEach(Consumer> consumer) { } } + // TODO: document the concurrency contract - Eval drains a cursor from a single coordinator + // thread even when evaluating cases concurrently, so implementations need no locking. @NotThreadSafe interface Cursor extends AutoCloseable { /** diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java index ed3613a2..13246fc6 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java @@ -17,7 +17,12 @@ import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; import java.util.*; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -26,11 +31,20 @@ /** * An evaluation framework for testing AI models. * + *

Cases are evaluated concurrently. By default up to {@value #DEFAULT_MAX_CONCURRENCY} + * cases run at once, so the {@link Task}, {@link Scorer}s and {@link Classifier}s supplied to an + * eval must be safe to invoke from multiple threads. Use {@link Builder#maxConcurrency(int)} to + * change the bound, or {@code maxConcurrency(1)} to evaluate cases one at a time. Use {@link + * Builder#executor(Executor)} to supply the threads the cases run on. + * * @param The type of input data for the evaluation * @param The type of output produced by the task */ @Slf4j public final class Eval { + /** Default number of eval cases evaluated concurrently. */ + public static final int DEFAULT_MAX_CONCURRENCY = 10; + private static final AttributeKey PARENT = AttributeKey.stringKey(BraintrustTracing.PARENT_KEY); private final @Nonnull String experimentName; @@ -47,6 +61,8 @@ public final class Eval { private final @Nonnull Map metadata; private final @Nonnull Parameters parameters; private final boolean ensureNew; + private final int maxConcurrency; + private final @Nullable Executor executor; private Eval(Builder builder) { this.experimentName = builder.experimentName; @@ -65,11 +81,39 @@ private Eval(Builder builder) { this.metadata = Map.copyOf(builder.metadata); this.parameters = builder.buildParameters(); this.ensureNew = builder.ensureNew; + this.maxConcurrency = builder.maxConcurrency; + this.executor = builder.executor; } - /** Runs the evaluation and returns results. */ + /** + * Runs the evaluation to completion and returns the results. + * + *

Cases are evaluated concurrently; see {@link Builder#maxConcurrency(int)}. + */ public EvalResult run() { - try (var cursor = dataset.openCursor()) { + var result = start(); + result.awaitCompletion(); + return result; + } + + /** + * Creates the experiment and begins evaluating cases in the background, returning as soon as + * the experiment exists. + * + *

The returned {@link EvalResult} carries the experiment id, name and url immediately — so + * callers can surface the link right away — while its cases are still being evaluated. Use + * {@link EvalResult#isDone()} and {@link EvalResult#awaitCompletion()} to observe the run. + * + *

Errors raised while creating the experiment are thrown from this method. Errors raised + * once the run is underway are reported through {@link EvalResult#awaitCompletion()}. + */ + public EvalResult start() { + var state = new EvalRunState(); + var cursor = dataset.openCursor(); + final EvalResult result; + final ExecutorService ownedExecutor; + final Executor caseExecutor; + try { Optional datasetVersion = Optional.empty(); Optional datasetId = Optional.empty(); if (dataset instanceof DatasetBrainstoreImpl) { @@ -94,8 +138,6 @@ public EvalResult run() { var experiment = new ExperimentsApi(client).postExperiment(createExperiment); - cursor.forEach(datasetCase -> evalOne(experiment.getId().toString(), datasetCase)); - // Use the experiment's actual name from the response: with ensure_new the backend may // dedupe a conflicting name (e.g. "foo" -> "foo-2f8ca776"), and the URL must point at // the real, created experiment. @@ -109,11 +151,122 @@ public EvalResult run() { project.getName()) .toASCIIString(), resolvedName); - return new EvalResult(experiment.getId().toString(), resolvedName, experimentUrl); - } + result = + new EvalResult( + experiment.getId().toString(), resolvedName, experimentUrl, state); + + if (executor != null) { + ownedExecutor = null; + caseExecutor = executor; + } else { + ownedExecutor = createDefaultExecutor(); + caseExecutor = ownedExecutor; + } + } catch (Throwable t) { + cursor.close(); + throw t; + } + + // Each case re-establishes the context that was current when the run was started. The eval + // span itself is created with setNoParent(), so this carries baggage rather than parentage. + var callerContext = Context.current(); + var experimentId = Objects.requireNonNull(result.getExperimentId()); + var coordinator = + new Thread( + () -> + evalAllCases( + cursor, + caseExecutor, + ownedExecutor, + experimentId, + callerContext, + state), + "braintrust-eval-coordinator"); + coordinator.setDaemon(true); + coordinator.start(); + return result; + } + + /** + * Drains the dataset cursor on this (coordinator) thread, submitting each case to {@code + * caseExecutor}. A semaphore bounds the number of cases in flight so the whole dataset is never + * materialized in memory, and so the executor's queue can't grow without limit. + * + *

Runs on the coordinator thread, never on a worker: waiting for cases to finish from inside + * the pool that runs them would deadlock once the pool is saturated. + */ + private void evalAllCases( + Dataset.Cursor> cursor, + Executor caseExecutor, + @Nullable ExecutorService ownedExecutor, + String experimentId, + Context callerContext, + EvalRunState state) { + Throwable fatal = + ConcurrentCases.drain( + cursor, + caseExecutor, + maxConcurrency, + datasetCase -> { + try (var unused = callerContext.makeCurrent()) { + if (evalOne(experimentId, datasetCase)) { + state.caseSucceeded(); + } else { + state.caseFailed(); + } + } catch (Throwable t) { + state.caseFailed(); + log.warn("Eval case failed for input: {}", datasetCase.input(), t); + } + }); + if (ownedExecutor != null) { + ownedExecutor.shutdown(); + } + if (fatal != null) { + // Callers that never await (e.g. a run started with start() and left to finish in the + // background) would otherwise never see this. + log.error( + "Eval aborted for experiment {} after {} case(s)", + experimentId, + state.getCasesSucceeded() + state.getCasesFailed(), + fatal); + } else { + log.debug( + "Eval complete for experiment {}: {} succeeded, {} failed", + experimentId, + state.getCasesSucceeded(), + state.getCasesFailed()); + } + state.complete(fatal); } - private void evalOne(String experimentId, DatasetCase datasetCase) { + /** + * The executor used when the caller did not supply one: a fixed pool of daemon threads sized to + * {@link #maxConcurrency}, owned by this run and shut down when it finishes. + * + *

On Java 21+, pass {@code Executors.newVirtualThreadPerTaskExecutor()} to {@link + * Builder#executor(Executor)} if you want to run many cases concurrently. + */ + private ExecutorService createDefaultExecutor() { + var counter = new AtomicInteger(); + return Executors.newFixedThreadPool( + maxConcurrency, + r -> { + var thread = + new Thread(r, "braintrust-eval-worker-" + counter.incrementAndGet()); + thread.setDaemon(true); + return thread; + }); + } + + /** + * Evaluates a single case. Runs entirely on one thread so that the OpenTelemetry scopes it + * opens stay thread-confined. + * + * @return false if the task threw (scorers fell back to {@link Scorer#scoreForTaskException}), + * true otherwise + */ + private boolean evalOne(String experimentId, DatasetCase datasetCase) { var rootSpan = tracer.spanBuilder("eval") // TODO: allow names for eval cases .setNoParent() // each eval case is its own trace @@ -164,7 +317,7 @@ private void evalOne(String experimentId, DatasetCase datasetCase for (var scorer : scorers) { runScoreForTaskException(experimentId, rootSpan, scorer, e, datasetCase); } - return; + return false; } taskSpan.end(); } @@ -221,6 +374,7 @@ private void evalOne(String experimentId, DatasetCase datasetCase } finally { rootSpan.end(); } + return true; } /** @@ -406,6 +560,8 @@ public static final class Builder { private @Nonnull List tags = List.of(); private @Nonnull Map metadata = Map.of(); private boolean ensureNew = false; + private int maxConcurrency = DEFAULT_MAX_CONCURRENCY; + private @Nullable Executor executor; public Eval build() { if (config == null) { @@ -453,6 +609,40 @@ public Builder apiClient(BraintrustApiClient apiClient) { return apiClient(apiClient.openApiClient()); } + /** + * Sets the maximum number of eval cases evaluated concurrently. Defaults to {@value + * Eval#DEFAULT_MAX_CONCURRENCY}. + * + *

Because cases run concurrently, the {@link Task}, {@link Scorer}s and {@link + * Classifier}s must be safe to invoke from multiple threads. Pass {@code 1} to evaluate + * cases one at a time. + * + *

This bounds how many cases are in flight at once. If you also supply an {@link + * #executor(Executor)} with fewer threads than this, that executor is the real limit and + * the remaining cases queue. + */ + public Builder maxConcurrency(int maxConcurrency) { + if (maxConcurrency < 1) { + throw new IllegalArgumentException( + "maxConcurrency must be at least 1, got " + maxConcurrency); + } + this.maxConcurrency = maxConcurrency; + return this; + } + + /** + * Sets the executor that eval cases run on. Defaults to a fixed pool of daemon threads + * sized to {@link #maxConcurrency(int)}, created and shut down by the eval. + * + *

An executor supplied here is never shut down by the SDK — the caller owns its + * lifecycle. On Java 21+, pass {@code Executors.newVirtualThreadPerTaskExecutor()} if you + * want to run many cases concurrently. + */ + public Builder executor(@Nonnull Executor executor) { + this.executor = Objects.requireNonNull(executor); + return this; + } + public Builder tracer(Tracer tracer) { this.tracer = tracer; return this; diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/EvalResult.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/EvalResult.java index 7b2b44b4..09c69b74 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/EvalResult.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/EvalResult.java @@ -1,24 +1,83 @@ package dev.braintrust.eval; +import java.time.Duration; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.Getter; -import lombok.SneakyThrows; -/** Results of all eval cases of an experiment. */ +/** + * Results of all eval cases of an experiment. + * + *

The experiment identifiers are available as soon as this object exists, but the run itself may + * still be in progress: {@link Eval#start()} hands back a result whose cases are still being + * evaluated, so that callers can surface the experiment link immediately. Use {@link #isDone()} and + * {@link #awaitCompletion()} to observe the run. A result returned by {@link Eval#run()} is always + * already complete. + */ public class EvalResult { @Getter private final @Nullable String experimentId; @Getter private final @Nullable String experimentName; @Getter private final String experimentUrl; + private final @Nonnull EvalRunState state; - @SneakyThrows EvalResult( - @Nullable String experimentId, @Nullable String experimentName, String experimentUrl) { + @Nullable String experimentId, + @Nullable String experimentName, + String experimentUrl, + @Nonnull EvalRunState state) { this.experimentId = experimentId; this.experimentName = experimentName; this.experimentUrl = experimentUrl; + this.state = state; + } + + /** true if the eval has completed execution */ + public boolean isDone() { + return state.isDone(); + } + + /** + * wait until the eval finishes running, or return right away if already done. + * + *

If the run aborted with an error, that error is rethrown here. + */ + public void awaitCompletion() { + state.await(); + } + + /** + * wait until the eval finishes running, or return right away if already done, giving up after + * {@code timeout}. + * + * @return true if the eval completed, false if the timeout elapsed while it was still running + */ + public boolean awaitCompletion(@Nonnull Duration timeout) { + return state.await(timeout); + } + + /** number of cases that finished evaluating successfully so far */ + public int getCasesSucceeded() { + return state.getCasesSucceeded(); + } + + /** + * number of cases that failed so far. A case fails when its task throws and no scorer fallback + * recovers it, or when scoring itself fails. Failed cases do not abort the run. + */ + public int getCasesFailed() { + return state.getCasesFailed(); } public String createReportString() { + if (!isDone()) { + return "Experiment is running (%d cases done so far). View live results in braintrust: %s" + .formatted(getCasesSucceeded() + getCasesFailed(), experimentUrl); + } + var failed = getCasesFailed(); + if (failed > 0) { + return "Experiment complete with %d failed case(s) of %d. View results in braintrust: %s" + .formatted(failed, getCasesSucceeded() + failed, experimentUrl); + } return "Experiment complete. View results in braintrust: %s".formatted(experimentUrl); } } diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/EvalRunState.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/EvalRunState.java new file mode 100644 index 00000000..9f67343b --- /dev/null +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/EvalRunState.java @@ -0,0 +1,112 @@ +package dev.braintrust.eval; + +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import javax.annotation.Nullable; + +/** + * Mutable execution state of a single eval run, shared between the {@link Eval} executing the run + * and the {@link EvalResult} handed back to the caller. + * + *

An {@link Eval} may hand back its {@link EvalResult} before the run has finished (see {@link + * Eval#start()}), so this state is written by the eval's coordinator and worker threads while the + * caller reads it. All members are thread-safe. + */ +final class EvalRunState { + private final CountDownLatch completed = new CountDownLatch(1); + private final AtomicInteger casesSucceeded = new AtomicInteger(); + private final AtomicInteger casesFailed = new AtomicInteger(); + private final AtomicReference failure = new AtomicReference<>(); + + /** Returns a state that is already complete, for callers that never ran anything. */ + static EvalRunState alreadyComplete() { + var state = new EvalRunState(); + state.complete(null); + return state; + } + + boolean isDone() { + return completed.getCount() == 0; + } + + /** + * Blocks until the run completes. If the run failed with an error that aborted it, that error + * is rethrown here. + */ + void await() { + try { + completed.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("interrupted while awaiting eval completion", e); + } + throwIfFailed(); + } + + /** + * Blocks until the run completes or the timeout elapses. + * + * @return true if the run completed, false if the timeout elapsed first + */ + boolean await(Duration timeout) { + final boolean done; + try { + done = completed.await(timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("interrupted while awaiting eval completion", e); + } + if (done) { + throwIfFailed(); + } + return done; + } + + private void throwIfFailed() { + var t = failure.get(); + if (t == null) { + return; + } + if (t instanceof RuntimeException re) { + throw re; + } + if (t instanceof Error err) { + throw err; + } + throw new RuntimeException("eval failed", t); + } + + /** + * Marks the run complete. A non-null {@code error} means the run aborted before every case was + * evaluated (as opposed to individual cases failing, which is tracked by {@link #caseFailed()} + * and does not abort the run). + */ + void complete(@Nullable Throwable error) { + failure.compareAndSet(null, error); + completed.countDown(); + } + + void caseSucceeded() { + casesSucceeded.incrementAndGet(); + } + + void caseFailed() { + casesFailed.incrementAndGet(); + } + + int getCasesSucceeded() { + return casesSucceeded.get(); + } + + int getCasesFailed() { + return casesFailed.get(); + } + + @Nullable + Throwable getFailure() { + return failure.get(); + } +} diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/Scorer.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/Scorer.java index 300a5607..95e45c6f 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/Scorer.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/Scorer.java @@ -15,6 +15,8 @@ * @param type of the input data * @param type of the output data */ +// TODO: document the concurrency contract - Eval invokes this from multiple threads +// (see Eval.Builder#maxConcurrency), so implementations must be thread-safe. public interface Scorer { String getName(); diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/Task.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/Task.java index 9072a9b1..00bcc9b4 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/Task.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/Task.java @@ -14,6 +14,8 @@ * @param type of the input data * @param type of the output data */ +// TODO: document the concurrency contract - Eval invokes this from multiple threads +// (see Eval.Builder#maxConcurrency), so implementations must be thread-safe. public interface Task { /** * Executes this task against a single dataset case, with access to merged eval parameters. diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/TracedClassifier.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/TracedClassifier.java index db67d78b..e2d4217b 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/TracedClassifier.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/TracedClassifier.java @@ -13,6 +13,8 @@ * @param type of the input data * @param type of the output data */ +// TODO: document the concurrency contract - Eval invokes this from multiple threads +// (see Eval.Builder#maxConcurrency), so implementations must be thread-safe. public interface TracedClassifier extends Classifier { /** diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/TracedScorer.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/TracedScorer.java index 352a7130..94a72e1a 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/TracedScorer.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/TracedScorer.java @@ -13,6 +13,8 @@ * @param type of the input data * @param type of the output data */ +// TODO: document the concurrency contract - Eval invokes this from multiple threads +// (see Eval.Builder#maxConcurrency), so implementations must be thread-safe. public interface TracedScorer extends Scorer { /** diff --git a/braintrust-sdk/src/test/java/dev/braintrust/devserver/DevserverConcurrencyTest.java b/braintrust-sdk/src/test/java/dev/braintrust/devserver/DevserverConcurrencyTest.java new file mode 100644 index 00000000..363bd5c3 --- /dev/null +++ b/braintrust-sdk/src/test/java/dev/braintrust/devserver/DevserverConcurrencyTest.java @@ -0,0 +1,263 @@ +package dev.braintrust.devserver; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.braintrust.BraintrustUtils; +import dev.braintrust.TestHarness; +import dev.braintrust.eval.Scorer; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.*; + +/** Covers concurrent case execution on the devserver's playground path. */ +@Slf4j +class DevserverConcurrencyTest { + private static final int TEST_PORT = 8302; + private static final String TEST_URL = "http://localhost:" + TEST_PORT; + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + private static final int MAX_CONCURRENCY = 4; + + private static final String BLOCKING_EVAL = "concurrency-blocking-eval"; + private static final String SCORES_EVAL = "concurrency-scores-eval"; + + private static final BraintrustUtils.Parent PLAYGROUND_PARENT = + new BraintrustUtils.Parent("playground_id", "ceea7422-3507-4d1c-a5f7-7acf41d9fac2"); + + private static Devserver server; + private static Thread serverThread; + private static TestHarness testHarness; + + // Observed concurrency of the blocking eval's task, reset per test. + private static final AtomicInteger inFlight = new AtomicInteger(); + private static final AtomicInteger peak = new AtomicInteger(); + // When set, every task counts down and then waits for all of them to arrive. + private static volatile CountDownLatch gate; + + @BeforeAll + static void setUp() throws Exception { + testHarness = TestHarness.setup(); + + var blockingEval = + RemoteEval.builder() + .name(BLOCKING_EVAL) + .taskFunction( + input -> { + peak.accumulateAndGet(inFlight.incrementAndGet(), Math::max); + try { + var g = gate; + if (g != null) { + g.countDown(); + assertTrue( + g.await(10, TimeUnit.SECONDS), + "tasks did not run concurrently"); + } else { + Thread.sleep(25); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } finally { + inFlight.decrementAndGet(); + } + return "ok"; + }) + .scorer(Scorer.of("static_scorer", (expected, result) -> 1.0)) + .build(); + + // Each case's input *is* its score, so a lost score in the shared aggregate changes the + // reported average. With identical scores per case the race would be invisible. + var scoresEval = + RemoteEval.builder() + .name(SCORES_EVAL) + .taskFunction(input -> input) + .scorer( + Scorer.of( + "varying_scorer", + (expected, result) -> Double.parseDouble(result))) + .build(); + + server = + Devserver.builder() + .config(testHarness.braintrust().config()) + .registerEval(blockingEval) + .registerEval(scoresEval) + .maxConcurrency(MAX_CONCURRENCY) + .host("localhost") + .port(TEST_PORT) + .build(); + + serverThread = + new Thread( + () -> { + try { + server.start(); + } catch (Exception e) { + log.error("unable to start dev server", e); + } + }); + serverThread.start(); + Thread.sleep(1000); + } + + @AfterAll + @SneakyThrows + static void tearDown() { + if (server != null) { + server.stop(); + } + if (serverThread != null) { + serverThread.join(30_000); + if (serverThread.isAlive()) { + serverThread.interrupt(); + } + } + } + + @BeforeEach + void resetCounters() { + inFlight.set(0); + peak.set(0); + gate = null; + } + + @Test + void casesRunConcurrently() throws Exception { + // Every task blocks until all MAX_CONCURRENCY of them have arrived, so the run can only + // complete if they genuinely execute at the same time. + gate = new CountDownLatch(MAX_CONCURRENCY); + + var events = runPlaygroundEval(BLOCKING_EVAL, inputs(MAX_CONCURRENCY)); + + assertEquals( + MAX_CONCURRENCY, + events.stream().filter(e -> "progress".equals(e.get("event"))).count(), + "every case should report progress"); + assertEquals(MAX_CONCURRENCY, peak.get(), "expected all cases in flight at once"); + } + + @Test + void maxConcurrencyIsRespected() throws Exception { + int caseCount = MAX_CONCURRENCY * 3; + var events = runPlaygroundEval(BLOCKING_EVAL, inputs(caseCount)); + + assertEquals( + caseCount, events.stream().filter(e -> "progress".equals(e.get("event"))).count()); + assertTrue(peak.get() > 1, "expected concurrent execution, peak was " + peak.get()); + assertTrue( + peak.get() <= MAX_CONCURRENCY, "exceeded maxConcurrency, peak was " + peak.get()); + } + + @Test + void scoreAggregationIsCorrectUnderConcurrency() throws Exception { + // Every case scores differently, so the summary average is only right if each concurrent + // case's score made it into the shared aggregate. The case count is high to make the + // narrow computeIfAbsent-then-add window land reliably. + int caseCount = 300; + var caseInputs = new ArrayList(); + double expectedSum = 0.0; + for (int i = 1; i <= caseCount; i++) { + double value = i / (double) caseCount; + expectedSum += value; + caseInputs.add(String.valueOf(value)); + } + double expectedAverage = expectedSum / caseCount; + + var events = runPlaygroundEval(SCORES_EVAL, caseInputs); + + var summary = + events.stream() + .filter(e -> "summary".equals(e.get("event"))) + .findFirst() + .orElseThrow(() -> new AssertionError("no summary event")); + JsonNode scores = JSON_MAPPER.readTree(summary.get("data")).get("scores"); + assertNotNull(scores.get("varying_scorer"), "summary should carry the scorer's average"); + assertEquals( + expectedAverage, + scores.get("varying_scorer").get("score").asDouble(), + 1e-9, + "every case's score must be counted exactly once"); + } + + private static List inputs(int count) { + var list = new ArrayList(); + for (int i = 0; i < count; i++) { + list.add("case-" + i); + } + return list; + } + + /** POSTs a streaming playground eval with inline cases and returns the parsed SSE events. */ + private List> runPlaygroundEval(String evalName, List caseInputs) + throws Exception { + var evalRequest = new EvalRequest(); + evalRequest.setName(evalName); + evalRequest.setStream(true); + evalRequest.setParent( + Map.of( + "object_type", PLAYGROUND_PARENT.type(), + "object_id", PLAYGROUND_PARENT.id())); + + var dataSpec = new EvalRequest.DataSpec(); + var cases = new ArrayList(); + for (String input : caseInputs) { + var c = new EvalRequest.EvalCaseData(); + c.setInput(input); + c.setExpected(input); + cases.add(c); + } + dataSpec.setData(cases); + evalRequest.setData(dataSpec); + + String requestBody = JSON_MAPPER.writeValueAsString(evalRequest); + + HttpURLConnection conn = + (HttpURLConnection) new URI(TEST_URL + "/eval").toURL().openConnection(); + conn.setRequestMethod("POST"); + conn.setRequestProperty("Content-Type", "application/json"); + conn.setRequestProperty("x-bt-auth-token", testHarness.braintrustApiKey()); + conn.setRequestProperty("x-bt-project-id", TestHarness.defaultProjectId()); + conn.setRequestProperty("x-bt-org-name", TestHarness.defaultOrgName()); + conn.setDoOutput(true); + conn.getOutputStream().write(requestBody.getBytes(StandardCharsets.UTF_8)); + conn.getOutputStream().flush(); + + assertEquals(200, conn.getResponseCode()); + return readSSEEvents(conn); + } + + private List> readSSEEvents(HttpURLConnection conn) throws Exception { + var events = new ArrayList>(); + try (BufferedReader reader = + new BufferedReader( + new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8))) { + String line; + String currentEvent = null; + var currentData = new StringBuilder(); + while ((line = reader.readLine()) != null) { + if (line.startsWith("event: ")) { + currentEvent = line.substring(7); + } else if (line.startsWith("data: ")) { + currentData.append(line.substring(6)); + } else if (line.isEmpty() && currentEvent != null) { + events.add(Map.of("event", currentEvent, "data", currentData.toString())); + currentEvent = null; + currentData = new StringBuilder(); + } + } + } + return events; + } +} diff --git a/braintrust-sdk/src/test/java/dev/braintrust/devserver/DevserverTest.java b/braintrust-sdk/src/test/java/dev/braintrust/devserver/DevserverTest.java index f3ad627c..6d8159b6 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/devserver/DevserverTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/devserver/DevserverTest.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.*; @@ -759,19 +760,29 @@ void testExperimentEval() throws Exception { String experimentId = summaryData.get("experimentId").asText(); // Spans should be parented to experiment_id: using the standard Eval span shape. - List allSpans = testHarness.awaitExportedSpans(); + // The snapshot response is sent as soon as the experiment exists — cases keep evaluating + // on the server's executor after that — so poll until this experiment's spans arrive + // rather than assuming the run finished when the response did. String expectedParent = "experiment_id:" + experimentId; - var evalSpans = - allSpans.stream() - .filter(s -> s.getName().equals("eval")) - .filter( - s -> - expectedParent.equals( - s.getAttributes() - .get( - AttributeKey.stringKey( - "braintrust.parent")))) - .toList(); + List evalSpans = List.of(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (true) { + evalSpans = + testHarness.awaitExportedSpans().stream() + .filter(s -> s.getName().equals("eval")) + .filter( + s -> + expectedParent.equals( + s.getAttributes() + .get( + AttributeKey.stringKey( + "braintrust.parent")))) + .toList(); + if (evalSpans.size() >= 2 || System.nanoTime() > deadline) { + break; + } + Thread.sleep(100); + } assertEquals(2, evalSpans.size(), "Should have 2 eval spans parented to the experiment"); for (SpanData evalSpan : evalSpans) { diff --git a/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalConcurrencyTest.java b/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalConcurrencyTest.java new file mode 100644 index 00000000..c2c6ae47 --- /dev/null +++ b/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalConcurrencyTest.java @@ -0,0 +1,257 @@ +package dev.braintrust.eval; + +import static org.junit.jupiter.api.Assertions.*; + +import dev.braintrust.TestHarness; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.SneakyThrows; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** Covers concurrent case execution and the run-state exposed on {@link EvalResult}. */ +public class EvalConcurrencyTest { + private TestHarness testHarness; + + @BeforeEach + void beforeEach() { + testHarness = TestHarness.setup(); + } + + private DatasetCase[] cases(int n) { + @SuppressWarnings("unchecked") + DatasetCase[] cases = new DatasetCase[n]; + for (int i = 0; i < n; i++) { + cases[i] = DatasetCase.of("input-" + i, "fruit"); + } + return cases; + } + + @Test + @SneakyThrows + public void casesRunConcurrently() { + int caseCount = 10; + // Every task blocks until all of them have arrived. This can only complete if the tasks + // genuinely run at the same time. + var allArrived = new CountDownLatch(caseCount); + var peak = new AtomicInteger(); + var inFlight = new AtomicInteger(); + + var eval = + testHarness + .braintrust() + .evalBuilder() + .name("concurrency-test") + .cases(cases(caseCount)) + .maxConcurrency(caseCount) + .taskFunction( + input -> { + peak.accumulateAndGet(inFlight.incrementAndGet(), Math::max); + allArrived.countDown(); + try { + assertTrue( + allArrived.await(10, TimeUnit.SECONDS), + "tasks did not run concurrently"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } finally { + inFlight.decrementAndGet(); + } + return "fruit"; + }) + .scorers(Scorer.of("s", r -> "fruit".equals(r) ? 1.0 : 0.0)) + .build(); + + var result = eval.run(); + assertEquals(caseCount, peak.get(), "expected all cases in flight at once"); + assertTrue(result.isDone()); + assertEquals(caseCount, result.getCasesSucceeded()); + assertEquals(0, result.getCasesFailed()); + } + + @Test + @SneakyThrows + public void maxConcurrencyIsRespected() { + int caseCount = 12; + int limit = 3; + var peak = new AtomicInteger(); + var inFlight = new AtomicInteger(); + + var eval = + testHarness + .braintrust() + .evalBuilder() + .name("max-concurrency-test") + .cases(cases(caseCount)) + .maxConcurrency(limit) + .taskFunction( + input -> { + peak.accumulateAndGet(inFlight.incrementAndGet(), Math::max); + try { + Thread.sleep(25); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + inFlight.decrementAndGet(); + return "fruit"; + }) + .scorers(Scorer.of("s", r -> 1.0)) + .build(); + + var result = eval.run(); + assertTrue(peak.get() > 1, "expected concurrent execution, peak was " + peak.get()); + assertTrue(peak.get() <= limit, "exceeded maxConcurrency, peak was " + peak.get()); + assertEquals(caseCount, result.getCasesSucceeded()); + } + + @Test + @SneakyThrows + public void serialWhenMaxConcurrencyIsOne() { + var peak = new AtomicInteger(); + var inFlight = new AtomicInteger(); + var eval = + testHarness + .braintrust() + .evalBuilder() + .name("serial-test") + .cases(cases(6)) + .maxConcurrency(1) + .taskFunction( + input -> { + peak.accumulateAndGet(inFlight.incrementAndGet(), Math::max); + try { + Thread.sleep(10); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + inFlight.decrementAndGet(); + return "fruit"; + }) + .scorers(Scorer.of("s", r -> 1.0)) + .build(); + eval.run(); + assertEquals(1, peak.get(), "maxConcurrency(1) must evaluate cases one at a time"); + } + + @Test + @SneakyThrows + public void startReturnsBeforeCompletionAndAwaitBlocks() { + var release = new CountDownLatch(1); + var eval = + testHarness + .braintrust() + .evalBuilder() + .name("start-test") + .cases(cases(4)) + .maxConcurrency(4) + .taskFunction( + input -> { + try { + assertTrue(release.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return "fruit"; + }) + .scorers(Scorer.of("s", r -> 1.0)) + .build(); + + var result = eval.start(); + // The experiment link is available immediately, while cases are still running. + assertNotNull(result.getExperimentUrl()); + assertFalse(result.isDone()); + assertFalse( + result.awaitCompletion(Duration.ofMillis(100)), + "awaitCompletion should time out while cases are still running"); + assertTrue(result.createReportString().contains("running")); + + release.countDown(); + result.awaitCompletion(); + assertTrue(result.isDone()); + assertEquals(4, result.getCasesSucceeded()); + assertTrue(result.createReportString().contains("complete")); + } + + @Test + @SneakyThrows + public void failingTaskIsContainedToItsOwnCase() { + var eval = + testHarness + .braintrust() + .evalBuilder() + .name("failing-task-test") + .cases(cases(6)) + .maxConcurrency(3) + .taskFunction( + input -> { + if ("input-2".equals(input)) { + throw new RuntimeException("boom"); + } + return "fruit"; + }) + .scorers(Scorer.of("s", r -> 1.0)) + .build(); + + var result = eval.run(); + assertTrue(result.isDone()); + assertEquals(1, result.getCasesFailed()); + assertEquals(5, result.getCasesSucceeded(), "other cases must still be evaluated"); + assertTrue(result.createReportString().contains("failed")); + } + + @Test + @SneakyThrows + public void outOfRangeScoreFailsOnlyThatCase() { + var eval = + testHarness + .braintrust() + .evalBuilder() + .name("bad-score-test") + .cases(cases(5)) + .maxConcurrency(2) + .taskFunction(input -> input) + .scorers(Scorer.of("s", r -> "input-1".equals(r.result()) ? 42.0 : 1.0)) + .build(); + + var result = eval.run(); + assertTrue(result.isDone()); + assertEquals(1, result.getCasesFailed()); + assertEquals(4, result.getCasesSucceeded(), "an invalid score must not abort the run"); + } + + @Test + @SneakyThrows + public void callerSuppliedExecutorIsNotShutDown() { + ExecutorService executor = Executors.newFixedThreadPool(4); + try { + var eval = + testHarness + .braintrust() + .evalBuilder() + .name("byo-executor-test") + .cases(cases(8)) + .executor(executor) + .taskFunction(input -> "fruit") + .scorers(Scorer.of("s", r -> 1.0)) + .build(); + var result = eval.run(); + assertEquals(8, result.getCasesSucceeded()); + assertFalse(executor.isShutdown(), "SDK must not shut down a caller-supplied executor"); + } finally { + executor.shutdownNow(); + } + } + + @Test + @SneakyThrows + public void rejectsInvalidMaxConcurrency() { + assertThrows( + IllegalArgumentException.class, + () -> Eval.builder().maxConcurrency(0)); + } +} diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-08983257c32b.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-08983257c32b.json new file mode 100644 index 00000000..e1ed3149 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-08983257c32b.json @@ -0,0 +1 @@ +{"id":"a2899b42-b4c6-4189-94ec-ae309cb28bd9","project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","name":"serial-test","description":null,"created":"2026-08-26T02:02:00.029Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","metadata":null,"tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-409b337a72c9.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-409b337a72c9.json new file mode 100644 index 00000000..862e9646 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-409b337a72c9.json @@ -0,0 +1 @@ +{"id":"e48d6ebf-42f0-45fa-b91c-8ce73f04f9f4","project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","name":"failing-task-test","description":null,"created":"2026-08-26T02:02:02.086Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","metadata":null,"tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-4c3bb7c53c36.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-4c3bb7c53c36.json new file mode 100644 index 00000000..f4367e54 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-4c3bb7c53c36.json @@ -0,0 +1 @@ +{"id":"7f8cb336-e1bc-4d4e-9541-03c4dee2e1b3","project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","name":"byo-executor-test","description":null,"created":"2026-08-26T02:01:58.096Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","metadata":null,"tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-81c93d1c26be.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-81c93d1c26be.json new file mode 100644 index 00000000..b8eb1de5 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-81c93d1c26be.json @@ -0,0 +1 @@ +{"id":"e7c0ab1f-9757-45cc-84f0-cb271d9bc398","project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","name":"concurrency-test","description":null,"created":"2026-08-26T02:02:01.048Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","metadata":null,"tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-a6fe1b460300.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-a6fe1b460300.json new file mode 100644 index 00000000..6acca0d4 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-a6fe1b460300.json @@ -0,0 +1 @@ +{"id":"3061a4e5-bfa5-4e84-a9c4-14fbe08bd653","project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","name":"start-test","description":null,"created":"2026-08-26T02:02:03.020Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","metadata":null,"tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-d25fd7674fdc.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-d25fd7674fdc.json new file mode 100644 index 00000000..3d253393 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-d25fd7674fdc.json @@ -0,0 +1 @@ +{"id":"37adfc69-2ad9-4c38-ad1e-fe8d3294add5","project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","name":"max-concurrency-test","description":null,"created":"2026-08-26T02:01:59.022Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","metadata":null,"tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-d2dc6ea2cede.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-d2dc6ea2cede.json new file mode 100644 index 00000000..3428c91e --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_experiment-d2dc6ea2cede.json @@ -0,0 +1 @@ +{"id":"73e8737d-a8eb-4ab1-b049-a3338c63eab3","project_id":"f1e858a4-58e3-408f-983f-016760d7fa25","name":"bad-score-test","description":null,"created":"2026-08-26T02:01:57.012Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","metadata":null,"tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-30867e7088bd.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-30867e7088bd.json new file mode 100644 index 00000000..89452950 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-30867e7088bd.json @@ -0,0 +1 @@ +{"objects":[{"id":"f1e858a4-58e3-408f-983f-016760d7fa25","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"java-unit-test","description":null,"created":"2026-05-07T16:55:48.127Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"localjava","description":null}]}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-410ff9f133da.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-410ff9f133da.json new file mode 100644 index 00000000..89452950 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-410ff9f133da.json @@ -0,0 +1 @@ +{"objects":[{"id":"f1e858a4-58e3-408f-983f-016760d7fa25","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"java-unit-test","description":null,"created":"2026-05-07T16:55:48.127Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"localjava","description":null}]}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-4891298f2969.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-4891298f2969.json new file mode 100644 index 00000000..89452950 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-4891298f2969.json @@ -0,0 +1 @@ +{"objects":[{"id":"f1e858a4-58e3-408f-983f-016760d7fa25","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"java-unit-test","description":null,"created":"2026-05-07T16:55:48.127Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"localjava","description":null}]}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-543a07178006.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-543a07178006.json new file mode 100644 index 00000000..89452950 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-543a07178006.json @@ -0,0 +1 @@ +{"objects":[{"id":"f1e858a4-58e3-408f-983f-016760d7fa25","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"java-unit-test","description":null,"created":"2026-05-07T16:55:48.127Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"localjava","description":null}]}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-c0742bb3c63f.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-c0742bb3c63f.json new file mode 100644 index 00000000..89452950 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-c0742bb3c63f.json @@ -0,0 +1 @@ +{"objects":[{"id":"f1e858a4-58e3-408f-983f-016760d7fa25","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"java-unit-test","description":null,"created":"2026-05-07T16:55:48.127Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"localjava","description":null}]}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-dd7665d7a48a.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-dd7665d7a48a.json new file mode 100644 index 00000000..89452950 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-dd7665d7a48a.json @@ -0,0 +1 @@ +{"objects":[{"id":"f1e858a4-58e3-408f-983f-016760d7fa25","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"java-unit-test","description":null,"created":"2026-05-07T16:55:48.127Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"localjava","description":null}]}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-e74886687a34.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-e74886687a34.json new file mode 100644 index 00000000..89452950 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-e74886687a34.json @@ -0,0 +1 @@ +{"objects":[{"id":"f1e858a4-58e3-408f-983f-016760d7fa25","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"java-unit-test","description":null,"created":"2026-05-07T16:55:48.127Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"localjava","description":null}]}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-e7e35e493e43.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-e7e35e493e43.json new file mode 100644 index 00000000..89452950 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-e7e35e493e43.json @@ -0,0 +1 @@ +{"objects":[{"id":"f1e858a4-58e3-408f-983f-016760d7fa25","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"java-unit-test","description":null,"created":"2026-05-07T16:55:48.127Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"localjava","description":null}]}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-08983257c32b.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-08983257c32b.json new file mode 100644 index 00000000..cd210823 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-08983257c32b.json @@ -0,0 +1,46 @@ +{ + "id" : "5b64a1d5-cf8f-3a99-a25d-94e2daff200e", + "name" : "v1_experiment", + "request" : { + "url" : "/v1/experiment", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"project_id\":\"f1e858a4-58e3-408f-983f-016760d7fa25\",\"name\":\"serial-test\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_experiment-08983257c32b.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "expires" : "0", + "x-amz-apigw-id" : "CsBlmFdhoAMEoHQ=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "447", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-6a8e4956-424345d80bd8e92b0e22fb03;Parent=12ec1581766a219b;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Wed, 26 Aug 2026 02:03:02 GMT", + "Via" : "1.1 4c9457912580c6114eec78b8fa604a20.cloudfront.net (CloudFront), 1.1 dbfd9bcc806d4c322e72b461b2458112.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "6a8e4956000000004567386e5d7de4e6", + "x-amzn-RequestId" : "3990b4fe-108f-4ffc-9661-e632711fc7d8", + "X-Amz-Cf-Id" : "-28FlDnNkNHrLughq7R1wHPr8ItsCCpqVs0eM_FhI0vs3Kw852ataQ==", + "etag" : "W/\"1bf-97m5BoF5VHRxK0jj4iLHHlNNYB4\"", + "cache-control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "surrogate-control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "5b64a1d5-cf8f-3a99-a25d-94e2daff200e", + "persistent" : true, + "insertionIndex" : 210 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-409b337a72c9.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-409b337a72c9.json new file mode 100644 index 00000000..7f0827f0 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-409b337a72c9.json @@ -0,0 +1,46 @@ +{ + "id" : "72603ff3-d67e-3e6c-acaa-8f9e27a4992d", + "name" : "v1_experiment", + "request" : { + "url" : "/v1/experiment", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"project_id\":\"f1e858a4-58e3-408f-983f-016760d7fa25\",\"name\":\"failing-task-test\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_experiment-409b337a72c9.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "expires" : "0", + "x-amz-apigw-id" : "CsBl-FkIIAMEbJg=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "453", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-6a8e4958-50f5298247cac1bf7a4d68af;Parent=6b2a342b76aca91a;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Wed, 26 Aug 2026 02:03:05 GMT", + "Via" : "1.1 4c9457912580c6114eec78b8fa604a20.cloudfront.net (CloudFront), 1.1 7605973575a3551426b82751020317de.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "6a8e4958000000001f178dfd50dd6607", + "x-amzn-RequestId" : "837e00cf-449b-48ee-8218-faabb5ba308d", + "X-Amz-Cf-Id" : "KSJri7sAWSDW6rLak3Of8xmh86mszishhUPJCC691jDk2dZMX6sUkA==", + "etag" : "W/\"1c5-0xoT2bc4TJrBN/DjLJgLlbr9jio\"", + "cache-control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "surrogate-control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "72603ff3-d67e-3e6c-acaa-8f9e27a4992d", + "persistent" : true, + "insertionIndex" : 204 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-4c3bb7c53c36.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-4c3bb7c53c36.json new file mode 100644 index 00000000..79c4bb88 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-4c3bb7c53c36.json @@ -0,0 +1,46 @@ +{ + "id" : "8964d68a-597d-3eb5-b4e9-e3aa97abb792", + "name" : "v1_experiment", + "request" : { + "url" : "/v1/experiment", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"project_id\":\"f1e858a4-58e3-408f-983f-016760d7fa25\",\"name\":\"byo-executor-test\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_experiment-4c3bb7c53c36.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "expires" : "0", + "x-amz-apigw-id" : "CsBlRE43IAMEJ7Q=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "453", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-6a8e4954-0d6ab108322ecdc147ad6e21;Parent=196ee23585497a9e;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Wed, 26 Aug 2026 02:03:00 GMT", + "Via" : "1.1 9257f9c4051fe8bd6cc4a09855b66350.cloudfront.net (CloudFront), 1.1 a53bab1af200813b8f27e3c0a28b4964.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "6a8e495400000000415a2f7944e75981", + "x-amzn-RequestId" : "bacf0ce5-8e5a-47ec-89db-cbc55bb72372", + "X-Amz-Cf-Id" : "CZSUA_mx7nSo-fD-OcNbLU1eFyyp2a7QzYwDZeUT-GV4jPaEce3Uzw==", + "etag" : "W/\"1c5-XHOo3Sc+fQrk1dpnwRSa1Vcfu6M\"", + "cache-control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "surrogate-control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "8964d68a-597d-3eb5-b4e9-e3aa97abb792", + "persistent" : true, + "insertionIndex" : 216 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-81c93d1c26be.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-81c93d1c26be.json new file mode 100644 index 00000000..fcb3954f --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-81c93d1c26be.json @@ -0,0 +1,46 @@ +{ + "id" : "2d9ed8b9-81c9-35f1-b790-3fff0d2237b0", + "name" : "v1_experiment", + "request" : { + "url" : "/v1/experiment", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"project_id\":\"f1e858a4-58e3-408f-983f-016760d7fa25\",\"name\":\"concurrency-test\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_experiment-81c93d1c26be.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "expires" : "0", + "x-amz-apigw-id" : "CsBlzGmDoAMEKhg=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "452", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-6a8e4957-0a402d845b695c8b4eee5a76;Parent=76c310315fe47e19;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Wed, 26 Aug 2026 02:03:04 GMT", + "Via" : "1.1 5e599a9eda8861379cfef6a522da18e4.cloudfront.net (CloudFront), 1.1 d9d466ed70d93f34739969f91577ec74.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "6a8e4957000000007dcb99f9684445c7", + "x-amzn-RequestId" : "0d7d7317-7f79-4f65-a28e-6a696e6e741a", + "X-Amz-Cf-Id" : "HcNIn4WG3_o7b4P_x1qs6chJaGCYvOmAVswR-HvkQEllfU7UMv5EFw==", + "etag" : "W/\"1c4-grPA721Zvp9tyi/CCXRTJj+1nVk\"", + "cache-control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "surrogate-control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "2d9ed8b9-81c9-35f1-b790-3fff0d2237b0", + "persistent" : true, + "insertionIndex" : 207 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-a6fe1b460300.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-a6fe1b460300.json new file mode 100644 index 00000000..26616b3d --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-a6fe1b460300.json @@ -0,0 +1,46 @@ +{ + "id" : "b79ecd18-2d9e-3c6a-a81c-855252f53364", + "name" : "v1_experiment", + "request" : { + "url" : "/v1/experiment", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"project_id\":\"f1e858a4-58e3-408f-983f-016760d7fa25\",\"name\":\"start-test\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_experiment-a6fe1b460300.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "expires" : "0", + "x-amz-apigw-id" : "CsBmLHqZIAMEZFA=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "446", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-6a8e495a-277957087858222b532928fb;Parent=1607b6e71c251554;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Wed, 26 Aug 2026 02:03:06 GMT", + "Via" : "1.1 d08613e1dd8ad614e47875ae31a8af20.cloudfront.net (CloudFront), 1.1 7605973575a3551426b82751020317de.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "6a8e495a000000005001dc2799514d3b", + "x-amzn-RequestId" : "933ba4ef-3e59-49c3-9ad7-9fa8e73ff17d", + "X-Amz-Cf-Id" : "49eI5-KMRBjAzP2VP4Mjk-MZbC_1EULuqZFhYTZnyOMeXTb0W9RWyw==", + "etag" : "W/\"1be-+xWx4aXHBjzJNFK6Xn4fSlVfCe4\"", + "cache-control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "surrogate-control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "b79ecd18-2d9e-3c6a-a81c-855252f53364", + "persistent" : true, + "insertionIndex" : 201 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-d25fd7674fdc.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-d25fd7674fdc.json new file mode 100644 index 00000000..1844ceb7 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-d25fd7674fdc.json @@ -0,0 +1,46 @@ +{ + "id" : "8414d439-b521-35f4-8bee-0a54e09dee2b", + "name" : "v1_experiment", + "request" : { + "url" : "/v1/experiment", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"project_id\":\"f1e858a4-58e3-408f-983f-016760d7fa25\",\"name\":\"max-concurrency-test\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_experiment-d25fd7674fdc.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "expires" : "0", + "x-amz-apigw-id" : "CsBlaHpbIAMEdCA=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "456", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-6a8e4955-6828c4714617366c631ab29e;Parent=30496073e7ca4516;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Wed, 26 Aug 2026 02:03:01 GMT", + "Via" : "1.1 940972e9e344075576fe20d5db482122.cloudfront.net (CloudFront), 1.1 ddea1c07643e5e0bfceb34480eebdc52.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "6a8e49550000000052b11cf9384fb2f2", + "x-amzn-RequestId" : "5716a763-dad3-41c1-87fa-ee03714102d4", + "X-Amz-Cf-Id" : "YDGChvbW1KdJPiQSSVo7edUDVJ9JWPN3s2kuL4iVqcYAtWa8E3DqWQ==", + "etag" : "W/\"1c8-pRyrCQTW84pbwgU7+aHf5A2snnM\"", + "cache-control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "surrogate-control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "8414d439-b521-35f4-8bee-0a54e09dee2b", + "persistent" : true, + "insertionIndex" : 213 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-d2dc6ea2cede.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-d2dc6ea2cede.json new file mode 100644 index 00000000..3c9bbf8c --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_experiment-d2dc6ea2cede.json @@ -0,0 +1,46 @@ +{ + "id" : "dfcd863e-d812-3f23-a64a-7e1eaa98d7be", + "name" : "v1_experiment", + "request" : { + "url" : "/v1/experiment", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"project_id\":\"f1e858a4-58e3-408f-983f-016760d7fa25\",\"name\":\"bad-score-test\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_experiment-d2dc6ea2cede.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "expires" : "0", + "x-amz-apigw-id" : "CsBlHFMhoAMETzg=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "450", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-6a8e4953-6e76055d1d17dc9a78030a8f;Parent=0e214336a7312682;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Wed, 26 Aug 2026 02:02:59 GMT", + "Via" : "1.1 5e599a9eda8861379cfef6a522da18e4.cloudfront.net (CloudFront), 1.1 a40ac7dad0e348fc93799233c9af5960.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "6a8e49530000000012362efe06e97c4f", + "x-amzn-RequestId" : "f6ed9394-f535-40f9-851a-9df2e1237752", + "X-Amz-Cf-Id" : "wdKNjlkizf2-HaEQTmbqqfojpCJ80Q5sHka74T7cz0oYE-Cyifk45Q==", + "etag" : "W/\"1c2-LL5Gg+/1eJtV3ksHwkyQRHY7WIc\"", + "cache-control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "surrogate-control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "dfcd863e-d812-3f23-a64a-7e1eaa98d7be", + "persistent" : true, + "insertionIndex" : 219 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-30867e7088bd.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-30867e7088bd.json new file mode 100644 index 00000000..537c05ad --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-30867e7088bd.json @@ -0,0 +1,45 @@ +{ + "id" : "6606a3c7-099c-3495-b32d-59502385df8a", + "name" : "v1_project", + "request" : { + "urlPath" : "/v1/project", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project-30867e7088bd.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "expires" : "0", + "x-amz-apigw-id" : "CsBmCHzOIAMEu4A=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "361", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-6a8e4959-0a723cd601323c4072490e17;Parent=6ac554dec80a6bf4;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Wed, 26 Aug 2026 02:03:05 GMT", + "Via" : "1.1 5e599a9eda8861379cfef6a522da18e4.cloudfront.net (CloudFront), 1.1 d525041695bdb6325f78ebba5c11b8a2.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "6a8e4959000000004c4ef1a48f0ad63f", + "x-amzn-RequestId" : "4674fb83-f815-4ee7-82ff-7155023ffc37", + "X-Amz-Cf-Id" : "6lLu9M2xEiJJp9xA1qulLHxoA2gCSgeD1Uv1Au3Xxs_GNrIhs6_G0A==", + "etag" : "W/\"169-XiwCuJsCqAZuAH8JspCgkYonnKw\"", + "cache-control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "surrogate-control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "6606a3c7-099c-3495-b32d-59502385df8a", + "persistent" : true, + "scenarioName" : "scenario-2-v1-project", + "requiredScenarioState" : "scenario-2-v1-project-8", + "insertionIndex" : 203 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-410ff9f133da.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-410ff9f133da.json new file mode 100644 index 00000000..e799d307 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-410ff9f133da.json @@ -0,0 +1,46 @@ +{ + "id" : "2c0e6f0a-b1c8-3b66-8435-51de30a1a7dd", + "name" : "v1_project", + "request" : { + "urlPath" : "/v1/project", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project-410ff9f133da.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "expires" : "0", + "x-amz-apigw-id" : "CsBleGshIAMEFrw=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "361", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-6a8e4955-2fe0eb9d34c137a71de65076;Parent=46111e8bf2ad1f80;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Wed, 26 Aug 2026 02:03:01 GMT", + "Via" : "1.1 d08613e1dd8ad614e47875ae31a8af20.cloudfront.net (CloudFront), 1.1 170efbc424be9181bda5d0fcd6e41f30.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "6a8e4955000000001b73442c5145d953", + "x-amzn-RequestId" : "1db41f87-9dc7-400a-9926-5cd46466853c", + "X-Amz-Cf-Id" : "iTuszd8A40Vkp9ACKXchl1u6M8VyFpemgn9iV-GgwZ8kdOVH9VLIrg==", + "etag" : "W/\"169-XiwCuJsCqAZuAH8JspCgkYonnKw\"", + "cache-control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "surrogate-control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "2c0e6f0a-b1c8-3b66-8435-51de30a1a7dd", + "persistent" : true, + "scenarioName" : "scenario-2-v1-project", + "requiredScenarioState" : "scenario-2-v1-project-5", + "newScenarioState" : "scenario-2-v1-project-6", + "insertionIndex" : 212 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-4891298f2969.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-4891298f2969.json new file mode 100644 index 00000000..921500e6 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-4891298f2969.json @@ -0,0 +1,46 @@ +{ + "id" : "f0270406-fe88-3aec-a5a7-ee6d2435ae22", + "name" : "v1_project", + "request" : { + "urlPath" : "/v1/project", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project-4891298f2969.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "expires" : "0", + "x-amz-apigw-id" : "CsBk4EnHIAMENBA=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "361", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-6a8e4951-621c71ec59f8141813e01799;Parent=22a0b81fad2ad5af;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Wed, 26 Aug 2026 02:02:58 GMT", + "Via" : "1.1 79a7455da856598d6db0b6edabec6574.cloudfront.net (CloudFront), 1.1 a53bab1af200813b8f27e3c0a28b4964.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "6a8e4951000000003d2cda141f6b444d", + "x-amzn-RequestId" : "c970926e-a46f-4572-8ca8-be8784209ee6", + "X-Amz-Cf-Id" : "k11VAzxBTO8LKMy1b4tKhezahWzNw_v2dnnSTiOeXq_EGSudgku_cA==", + "etag" : "W/\"169-XiwCuJsCqAZuAH8JspCgkYonnKw\"", + "cache-control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "surrogate-control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "f0270406-fe88-3aec-a5a7-ee6d2435ae22", + "persistent" : true, + "scenarioName" : "scenario-2-v1-project", + "requiredScenarioState" : "Started", + "newScenarioState" : "scenario-2-v1-project-2", + "insertionIndex" : 223 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-543a07178006.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-543a07178006.json new file mode 100644 index 00000000..82942338 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-543a07178006.json @@ -0,0 +1,46 @@ +{ + "id" : "1c224e28-c695-3150-80a1-b131fa0ac88b", + "name" : "v1_project", + "request" : { + "urlPath" : "/v1/project", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project-543a07178006.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "expires" : "0", + "x-amz-apigw-id" : "CsBl3Em2oAMEJwA=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "361", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-6a8e4958-59bbb01b1eec443c5a850363;Parent=6506dd58c4c3d4f8;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Wed, 26 Aug 2026 02:03:04 GMT", + "Via" : "1.1 9257f9c4051fe8bd6cc4a09855b66350.cloudfront.net (CloudFront), 1.1 77f3c89ffd619275648d49ad13868570.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "6a8e4958000000003bde946306d0cf1b", + "x-amzn-RequestId" : "e005ef1a-6751-4166-8ed5-c102005f08bf", + "X-Amz-Cf-Id" : "m3aOPGH-6x5mqiElWUtaJREczwpFDV41Jvcsjo_56brRHqs6VThFlA==", + "etag" : "W/\"169-XiwCuJsCqAZuAH8JspCgkYonnKw\"", + "cache-control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "surrogate-control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "1c224e28-c695-3150-80a1-b131fa0ac88b", + "persistent" : true, + "scenarioName" : "scenario-2-v1-project", + "requiredScenarioState" : "scenario-2-v1-project-7", + "newScenarioState" : "scenario-2-v1-project-8", + "insertionIndex" : 206 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-c0742bb3c63f.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-c0742bb3c63f.json new file mode 100644 index 00000000..7f220305 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-c0742bb3c63f.json @@ -0,0 +1,46 @@ +{ + "id" : "02e68d25-de8f-38c9-b80e-6c8c4390e791", + "name" : "v1_project", + "request" : { + "urlPath" : "/v1/project", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project-c0742bb3c63f.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "expires" : "0", + "x-amz-apigw-id" : "CsBltEEjoAMEedA=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "361", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-6a8e4957-4c606da744809ecd56caea26;Parent=3cb9701604c287b2;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Wed, 26 Aug 2026 02:03:03 GMT", + "Via" : "1.1 487082619948f670d3b30fb3db8fbabc.cloudfront.net (CloudFront), 1.1 e6b2537b87653726af8a79e6da505188.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "6a8e4957000000000480684f2f070b17", + "x-amzn-RequestId" : "af5e5be4-5097-49b2-9d16-6f249fd88fb0", + "X-Amz-Cf-Id" : "luyinWb5qt8g9doJMPeeayv80rS_OO5VeN45HpyDwYZch3Ni08uZFA==", + "etag" : "W/\"169-XiwCuJsCqAZuAH8JspCgkYonnKw\"", + "cache-control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "surrogate-control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "02e68d25-de8f-38c9-b80e-6c8c4390e791", + "persistent" : true, + "scenarioName" : "scenario-2-v1-project", + "requiredScenarioState" : "scenario-2-v1-project-6", + "newScenarioState" : "scenario-2-v1-project-7", + "insertionIndex" : 209 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-dd7665d7a48a.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-dd7665d7a48a.json new file mode 100644 index 00000000..d24880d6 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-dd7665d7a48a.json @@ -0,0 +1,46 @@ +{ + "id" : "0254f542-fbbd-37af-903b-d925ff1ca4af", + "name" : "v1_project", + "request" : { + "urlPath" : "/v1/project", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project-dd7665d7a48a.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "expires" : "0", + "x-amz-apigw-id" : "CsBlUG_XoAMEjTQ=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "361", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-6a8e4954-5c1319636fed16174720b092;Parent=677b49084d48bb24;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Wed, 26 Aug 2026 02:03:00 GMT", + "Via" : "1.1 79a7455da856598d6db0b6edabec6574.cloudfront.net (CloudFront), 1.1 82fa7f20ab5a12301da8e01f9493e222.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "6a8e4954000000004771e334eca2d3f4", + "x-amzn-RequestId" : "8585c6b2-3869-4653-b077-e6bacc6daf45", + "X-Amz-Cf-Id" : "Am3vXQqyyGn8Ix-GFJCnb6kpHUazzovywLOQRADB2dK3i5Gt9hmvAA==", + "etag" : "W/\"169-XiwCuJsCqAZuAH8JspCgkYonnKw\"", + "cache-control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "surrogate-control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "0254f542-fbbd-37af-903b-d925ff1ca4af", + "persistent" : true, + "scenarioName" : "scenario-2-v1-project", + "requiredScenarioState" : "scenario-2-v1-project-4", + "newScenarioState" : "scenario-2-v1-project-5", + "insertionIndex" : 215 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-e74886687a34.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-e74886687a34.json new file mode 100644 index 00000000..bf3c5b9d --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-e74886687a34.json @@ -0,0 +1,46 @@ +{ + "id" : "b0b988b5-0416-376e-8651-152095d5b692", + "name" : "v1_project", + "request" : { + "urlPath" : "/v1/project", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project-e74886687a34.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "expires" : "0", + "x-amz-apigw-id" : "CsBlKHgSIAMEFrg=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "361", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-6a8e4953-52a0e2fc5e630eb219f32712;Parent=46e909b73245dac5;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Wed, 26 Aug 2026 02:02:59 GMT", + "Via" : "1.1 4c9457912580c6114eec78b8fa604a20.cloudfront.net (CloudFront), 1.1 b3ccaedda78c63d5967b57382ceb4cbe.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "6a8e4953000000005b9ff4c37362807a", + "x-amzn-RequestId" : "30fed0ad-693e-4301-a15f-52d7c803ac6e", + "X-Amz-Cf-Id" : "B8bHKJc8FmxcuRCWlwD5nXLjf33IEhVn9SyZ0JPifnyS1gN38HCvlw==", + "etag" : "W/\"169-XiwCuJsCqAZuAH8JspCgkYonnKw\"", + "cache-control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "surrogate-control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "b0b988b5-0416-376e-8651-152095d5b692", + "persistent" : true, + "scenarioName" : "scenario-2-v1-project", + "requiredScenarioState" : "scenario-2-v1-project-3", + "newScenarioState" : "scenario-2-v1-project-4", + "insertionIndex" : 218 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-e7e35e493e43.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-e7e35e493e43.json new file mode 100644 index 00000000..a8c60f3a --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-e7e35e493e43.json @@ -0,0 +1,46 @@ +{ + "id" : "d2f428e5-f001-3e3a-9b24-5e1d7065518c", + "name" : "v1_project", + "request" : { + "urlPath" : "/v1/project", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project-e7e35e493e43.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "expires" : "0", + "x-amz-apigw-id" : "CsBlAGbXoAMEuDQ=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "361", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-6a8e4952-56a98b936ee8f43c421b9174;Parent=713a3d277e37a969;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Wed, 26 Aug 2026 02:02:58 GMT", + "Via" : "1.1 487082619948f670d3b30fb3db8fbabc.cloudfront.net (CloudFront), 1.1 98b0bd91dae8cd13230da037f5a4d5e0.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "6a8e49520000000001ac59a8fef3716b", + "x-amzn-RequestId" : "004d0f0b-2d2f-4474-8682-a2203dfc75da", + "X-Amz-Cf-Id" : "NYfyjIXsX5hh7ECi8kulYn_h3KUDvhS-djffcI2KNZsK9N-dUlcK5g==", + "etag" : "W/\"169-XiwCuJsCqAZuAH8JspCgkYonnKw\"", + "cache-control" : "no-store, no-cache, must-revalidate, proxy-revalidate", + "surrogate-control" : "no-store", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "d2f428e5-f001-3e3a-9b24-5e1d7065518c", + "persistent" : true, + "scenarioName" : "scenario-2-v1-project", + "requiredScenarioState" : "scenario-2-v1-project-2", + "newScenarioState" : "scenario-2-v1-project-3", + "insertionIndex" : 221 +} \ No newline at end of file