Experiments

Experiments let you systematically evaluate a prompt + dataset + evaluator combination. Each run executes every dataset item through the prompt and scores the output.

Setup

import { AISDK } from '@browserstack/ai-sdk';

const testOps = new AISDK({
  publicKey: process.env.AISDK_PUBLIC_KEY,
  secretKey: process.env.AISDK_SECRET_KEY,
});

const experiments = testOps.experiments;       // Experiments
const experimentRuns = testOps.experimentRuns; // ExperimentRuns

import os
from browserstack_ai_sdk import AISDK

client = AISDK(
    public_key=os.environ["AISDK_PUBLIC_KEY"],
    secret_key=os.environ["AISDK_SECRET_KEY"],
)

import com.browserstack.aisdk.AISDK;
import com.browserstack.aisdk.eval.ExperimentsClient;
import com.browserstack.aisdk.eval.ExperimentRunsClient;

AISDK sdk = AISDK.fromEnv();
ExperimentsClient experiments = sdk.experiments();
ExperimentRunsClient experimentRuns = sdk.experimentRuns();

Create an Experiment

An experiment requires a name, an evaluator list, and either a datasetRunTagId or both promptId + datasetId.

With a prompt and dataset

import { CreateExperimentWithPromptRequest } from '@browserstack/ai-sdk';

const experiment = await experiments.create({
  name: 'qa-accuracy-test',
  description: 'Evaluate QA accuracy on golden set',
  promptId: 'prompt-id-abc',
  datasetId: 'dataset-id-xyz',
  evaluatorListId: 'eval-list-id-123',
  concurrency: 5,
} satisfies CreateExperimentWithPromptRequest);

console.log(experiment.id);

experiment = client.experiments.create({
    "name": "rag-eval-v2",
    "evaluatorListId": "eval-list-id",
    "datasetId": "dataset-id",
    "promptId": "prompt-id",
})

import com.browserstack.aisdk.eval.model.CreateExperimentRequest;
import com.browserstack.aisdk.eval.model.ExperimentResponse;

ExperimentResponse experiment = experiments.create(
    CreateExperimentRequest.builder()
        .name("gpt-4o-faithfulness-v2")
        .evaluatorListId("evl_abc123")
        .promptId("prm_xyz789")
        .datasetId("ds_def456")
        .concurrency(5)
        .build()
);

System.out.println("Experiment ID: " + experiment.getId());

You must provide either datasetRunTagId alone, or both promptId and datasetId together. Mixing them throws IllegalArgumentException.

With a dataset run tag

import { CreateExperimentWithTagRequest } from '@browserstack/ai-sdk';

const experiment = await experiments.create({
  name: 'tag-based-experiment',
  datasetRunTagId: 'tag-id-abc',
  evaluatorListId: 'eval-list-id-123',
  concurrency: 3,
} satisfies CreateExperimentWithTagRequest);

experiment = client.experiments.create({
    "name": "rag-eval-v1",
    "evaluatorListId": "eval-list-id",
    "datasetRunTagId": "tag-id-from-dataset-run",
})
print(experiment)

List and Get Experiments

const result = await experiments.list(
  20, // limit (1-100)
  1   // page
);

for (const exp of result.experiments) {
  console.log(exp.id, exp.name, exp.createdAt);
}

const experiment = await experiments.find('experiment-id-abc');
console.log(experiment.name, experiment.status);

experiments = client.experiments.list(page=1, limit=50)

experiment = client.experiments.get("experiment-id")
print(experiment)

ListExperimentsResponse list = experiments.list(20, 1);
list.getExperiments().forEach(e -> System.out.println(e.getId() + " " + e.getName()));

ExperimentResponse experiment = experiments.find("exp_abc123");
System.out.println("Found: " + experiment.getName() + " (" + experiment.getId() + ")");

Create and Monitor Runs

Create a Run

const run = await experimentRuns.create(
  'experiment-id-abc', // experimentId
  'output',            // llmColumnName
  5,                   // concurrency (1-100)
  { temperature: 0.0 } // optional runConfig
);

console.log(run.id, run.status); // 'PENDING'

run = client.experiment_runs.create({
    "experimentId": "experiment-id",
    "name": "run-2026-04-01",
})
print(run)

import com.browserstack.aisdk.eval.model.CreateExperimentRunRequest;
import com.browserstack.aisdk.eval.model.ExperimentRunResponse;

ExperimentRunResponse run = experimentRuns.create(
    CreateExperimentRunRequest.builder()
        .experimentId(experiment.getId())
        .build()
);

System.out.println("Run ID: " + run.getId());

Poll for Completion

const result = await experimentRuns.subscribe(
  run.id,
  120_000, // timeout in ms
  5_000    // poll interval in ms
);

if (result.finalStatus === 'COMPLETED') {
  console.log('Run completed:', result.experimentRunData);
} else {
  console.error('Run failed or timed out:', result.finalStatus);
}

// Default: polls every 5s, times out after 5 minutes
ExperimentRunResponse finalRun = experimentRuns.subscribe(
    run.getId(),
    update -> System.out.println("Status: " + update.getStatus())
);

System.out.println("Final status: " + finalRun.getStatus());

With a custom timeout:

long tenMinutes = 10 * 60 * 1_000L;

ExperimentRunResponse finalRun = experimentRuns.subscribe(
    run.getId(),
    update -> System.out.println("Run status: " + update.getStatus()),
    tenMinutes
);

Summarize a Run

summarize() returns scores, metrics, diffs, and UI deep-links for a completed run in a single call. Use it in CI to gate on threshold status, post PR comments, or feed dashboards.

Parameters

Parameter	Default	Description
`compare_to_run_id` / `compareToRunId` / `compareToRunID`	UI-pinned baseline (or none)	Explicit baseline run ID. When omitted (or `""` in Go, `None` in Python, not passed in TypeScript/Java), the server uses the experiment's UI-pinned baseline (set via "Set as baseline" in the experiment view). If neither is set, no comparison is performed and every `diff` is `null`. The server does NOT auto-pick a baseline by recency.

Basic call

const summary = await experimentRuns.summarize("run-uuid-1");

console.log(summary.summary.verdict);       // "PASS" | "REGRESSION"
console.log(summary.urls.scoresTab);        // deep-link to Scores tab in the UI
const exactMatch = summary.scores.find(s => s.name === "exact_match");
console.log(exactMatch?.thresholdStatus); // "pass" | "fail"

Pass an options object to pin the baseline:

// With an explicit baseline
const summary = await experimentRuns.summarize("run-uuid-1", {
  compareToRunId: "run-uuid-0",
});

summary = client.experiment_runs.summarize("run-uuid-1")

print(summary.summary.verdict)                              # "PASS" or "REGRESSION"
print(summary.urls.scores_tab)                              # deep-link to Scores tab
exact_match = next((s for s in summary.scores if s.name == "exact_match"), None)
print(exact_match.threshold_status if exact_match else None)  # "pass" or "fail"

With an explicit baseline:

summary = client.experiment_runs.summarize(
    "run-uuid-1",
    compare_to_run_id="run-uuid-0",
)

ExperimentRunSummary summary = experimentRuns.summarize(runId);

System.out.println(summary.getSummary().getVerdict());                              // "PASS" or "REGRESSION"
System.out.println(summary.getUrls().getScoresTab());                               // deep-link to Scores tab
ExperimentRunScoreSummary exactMatch = summary.getScores().stream()
    .filter(s -> "exact_match".equals(s.getName()))
    .findFirst()
    .orElse(null);
System.out.println(exactMatch != null ? exactMatch.getThresholdStatus() : null);    // "pass" or "fail"

Two overloads are available, matching the list() / get() pattern used elsewhere in the Java SDK:

// Uses UI-pinned baseline if any, else no comparison
ExperimentRunSummary summary = experimentRuns.summarize(runId);

// Explicit baseline
ExperimentRunSummary summary = experimentRuns.summarize(runId, "run-uuid-0");

summary, err := client.ExperimentRuns.Summarize(ctx, "run-uuid-1", "")
if err != nil {
    log.Fatal(err)
}

fmt.Println(summary.Summary.Verdict)                              // "PASS" or "REGRESSION"
fmt.Println(summary.Urls.ScoresTab)                               // deep-link to Scores tab
var exactMatch *crud.ExperimentRunScoreSummary
for i := range summary.Scores {
    if summary.Scores[i].Name == "exact_match" {
        exactMatch = &summary.Scores[i]
        break
    }
}
if exactMatch != nil {
    fmt.Println(exactMatch.ThresholdStatus)                       // "pass" or "fail"
}

Pass an empty string for compareToRunID to use auto-resolution.

// Uses UI-pinned baseline if any, else no comparison
summary, err := client.ExperimentRuns.Summarize(ctx, "run-uuid-1", "")

// Explicit baseline
summary, err := client.ExperimentRuns.Summarize(ctx, "run-uuid-1", "run-uuid-0")

CI usage — post a PR comment

A common pattern is running summarize() at the end of a CI job and posting the result as a pull-request comment.

// GitHub Actions step — gate on regression and post a PR comment.
// The SDK returns structured `scores` and `metrics`; render the comment
// however you like (table, one-line summary, Slack blocks, etc.).
const summary = await experimentRuns.summarize(run.id);

if (summary.summary.verdict === "REGRESSION") {
  console.error(`Run regressed: ${summary.summary.regressedCount} scorer(s) degraded`);
  process.exit(1);
}

const scoreLines = summary.scores.map(
  (s) => `- **${s.name}** — ${s.score?.toFixed(3) ?? "—"} (${s.thresholdStatus ?? "—"})`,
);
const body = [
  `**Eval summary — ${summary.summary.verdict}**`,
  ...scoreLines,
  `\n[View full scores](${summary.urls.scoresTab})`,
].join("\n");

await octokit.issues.createComment({ body, /* owner, repo, issue_number */ });

# GitHub Actions step — gate on regression and post a PR comment.
# The SDK returns structured `scores` and `metrics`; render the comment
# however you like (table, one-line summary, Slack blocks, etc.).
summary = client.experiment_runs.summarize(run["id"])

if summary.summary.verdict == "REGRESSION":
    print(f"Run regressed: {summary.summary.regressed_count} scorer(s) degraded")
    sys.exit(1)

score_lines = [
    f"- **{s.name}** — {s.score:.3f} ({s.threshold_status or '—'})"
    for s in summary.scores
    if s.score is not None
]
body = "\n".join([
    f"**Eval summary — {summary.summary.verdict}**",
    *score_lines,
    f"\n[View full scores]({summary.urls.scores_tab})",
])

github.create_pull_request_comment(body=body)

// GitHub Actions step — gate on regression and post a PR comment.
// The SDK returns structured `scores` and `metrics`; render the comment
// however you like (table, one-line summary, Slack blocks, etc.).
ExperimentRunSummary summary = experimentRuns.summarize(run.getId());

if ("REGRESSION".equals(summary.getSummary().getVerdict())) {
    System.err.println("Run regressed: "
        + summary.getSummary().getRegressedCount() + " scorer(s) degraded");
    System.exit(1);
}

StringBuilder body = new StringBuilder("**Eval summary — ")
    .append(summary.getSummary().getVerdict()).append("**\n");
for (ExperimentRunScoreSummary s : summary.getScores()) {
    body.append("- **").append(s.getName()).append("** — ")
        .append(s.getScore() != null ? String.format("%.3f", s.getScore()) : "—")
        .append(" (").append(s.getThresholdStatus() != null ? s.getThresholdStatus() : "—").append(")\n");
}
body.append("\n[View full scores](").append(summary.getUrls().getScoresTab()).append(")");

githubClient.createPullRequestComment(body.toString());

// GitHub Actions step — gate on regression and post a PR comment.
// The SDK returns structured `scores` and `metrics`; render the comment
// however you like (table, one-line summary, Slack blocks, etc.).
summary, err := client.ExperimentRuns.Summarize(ctx, run.ID, "")
if err != nil {
    log.Fatal(err)
}

if summary.Summary.Verdict == "REGRESSION" {
    fmt.Fprintf(os.Stderr, "Run regressed: %d scorer(s) degraded\n",
        summary.Summary.RegressedCount)
    os.Exit(1)
}

var lines []string
lines = append(lines, fmt.Sprintf("**Eval summary — %s**", summary.Summary.Verdict))
for _, s := range summary.Scores {
    score := "—"
    if s.Score != nil {
        score = fmt.Sprintf("%.3f", *s.Score)
    }
    status := "—"
    if s.ThresholdStatus != nil {
        status = *s.ThresholdStatus
    }
    lines = append(lines, fmt.Sprintf("- **%s** — %s (%s)", s.Name, score, status))
}
lines = append(lines, fmt.Sprintf("\n[View full scores](%s)", summary.Urls.ScoresTab))
body := strings.Join(lines, "\n")
githubClient.CreatePullRequestComment(ctx, body)

Response shape

The returned object has these top-level fields:

Field	Type	Description
`projectId`	`string`	Project ID
`projectName`	`string`	Project name
`experimentId`	`string`	Experiment ID
`experimentName`	`string`	Experiment name
`experimentRunId`	`string`	This run's ID
`experimentRunName`	`string`	This run's name
`comparisonExperimentRunId`	`string \| null`	Baseline run ID (`null` when no baseline)
`comparisonExperimentRunName`	`string \| null`	Baseline run name (`null` when no baseline)
`urls.project`	`string`	UI link to the project
`urls.experiment`	`string`	UI link to the experiment
`urls.run`	`string`	UI link to the run page
`urls.scoresTab`	`string`	Direct link to the Scores tab
`urls.analyticsTab`	`string`	Direct link to the Analytics tab
`scores`	`array`	Ordered list of evaluator score summaries — worst-regressed first; look up by `name` (see below)
`metrics`	`object`	Map of metric name → metric summary (see below)
`summary.verdict`	`"PASS" \| "REGRESSION"`	Overall run verdict
`summary.failureCount`	`number`	Scorers below threshold
`summary.improvedCount`	`number`	Scorers that improved vs baseline
`summary.regressedCount`	`number`	Scorers that regressed vs baseline

Each score summary includes thresholdStatus ("pass" or "fail"), diff vs baseline, improvements, and regressions per evaluator. Numeric scores additionally carry passCount, failCount, and totalItems; categorical scores carry categoryCounts and passCategories.

Each metric summary includes metric (the aggregate value), unit (s, ms, tok, $), and diff vs baseline.

Comparison Verdict

getComparisonVerdict reads the AI Verdict for a comparison between two runs of the same experiment — which run is better, a confidence level (HIGH / MEDIUM / LOW), a reasoning sentence citing the scorer numbers, and the per-scorer drivers behind the call. The verdict is produced by a fully deterministic (no-LLM) engine and is saved on the platform.

This method is read-only: it returns the last verdict generated from the experiment compare page in the UI. If no verdict has been generated for the run pair yet, status is NOT_GENERATED and winner is null — generate one from the compare page first.

Read a verdict

const verdict = await testOps.experimentRuns.getComparisonVerdict(
  "experiment-id-abc", // experimentId
  "run-uuid-0",        // baseRunId
  "run-uuid-1",        // targetRunId
);

if (verdict.status === "NOT_GENERATED") {
  console.log("No verdict yet — generate it from the compare page in the UI.");
} else if (verdict.status === "DECLINED") {
  console.log("Not enough comparable data to call a winner.");
} else {
  // READY or STALE
  console.log(`Winner: ${verdict.winner} (${verdict.confidence})`);
  console.log(verdict.reasoning);
  if (verdict.status === "STALE") {
    console.warn("Scores changed since this verdict was generated — refresh it in the UI.");
  }
}

verdict = client.experiment_runs.get_comparison_verdict(
    "experiment-id-abc",  # experiment_id
    "run-uuid-0",         # base_run_id
    "run-uuid-1",         # target_run_id
)

if verdict["status"] == "NOT_GENERATED":
    print("No verdict yet — generate it from the compare page in the UI.")
elif verdict["status"] == "DECLINED":
    print("Not enough comparable data to call a winner.")
else:
    # READY or STALE
    print(f"Winner: {verdict['winner']} ({verdict['confidence']})")
    print(verdict["reasoning"])
    if verdict["status"] == "STALE":
        print("Scores changed since this verdict was generated — refresh it in the UI.")

This method is coming soon for Java. Use the public REST endpoint in the meantime.

This method is coming soon for Go. Use the public REST endpoint in the meantime.

Response shape

Field	Type	Description
`status`	`"READY" \| "STALE" \| "DECLINED" \| "NOT_GENERATED"`	Verdict state
`baseRunId`	`string`	Echo of the baseline run ID
`targetRunId`	`string`	Echo of the target run ID
`winner`	`"BASE" \| "TARGET" \| null`	Which run is better; `null` unless `READY` or `STALE`
`confidence`	`"HIGH" \| "MEDIUM" \| "LOW" \| null`	How decisive the result is; `null` when no winner
`reasoning`	`string \| null`	Plain-English explanation citing scorer numbers; `null` when no winner
`drivers`	`array \| null`	Per-scorer breakdown — each has `name`, `role` (`FOR`/`AGAINST`/`NEUTRAL`), `baseValue`, `targetValue`, `normalized`, `items`, `lowData`; `null` when no winner
`generatedAt`	`string \| null`	ISO timestamp of the saved verdict; `null` when `NOT_GENERATED`

Generation (and refreshing a STALE verdict) happens in the UI — see the AI Verdict platform guide. The SDK and REST API surface is read-only.

Run Status Values

enum ExperimentRunStatus {
  PENDING    = 'PENDING',
  RUNNING    = 'RUNNING',
  COMPLETED  = 'COMPLETED',
  FAILED     = 'FAILED',
  CANCELLED  = 'CANCELLED',
}

Complete End-to-End Example

import { AISDK, ExperimentRunStatus } from '@browserstack/ai-sdk';

const testOps = new AISDK({
  publicKey: process.env.AISDK_PUBLIC_KEY,
  secretKey: process.env.AISDK_SECRET_KEY,
});

async function runExperiment() {
  const experiments = testOps.experiments;
  const experimentRuns = testOps.experimentRuns;

  // 1. Create the experiment
  const experiment = await experiments.create({
    name: `accuracy-test-${Date.now()}`,
    description: 'Automated accuracy evaluation',
    promptId: process.env.PROMPT_ID!,
    datasetId: process.env.DATASET_ID!,
    evaluatorListId: process.env.EVALUATOR_LIST_ID!,
    concurrency: 5,
  });

  console.log('Created experiment:', experiment.id);

  // 2. Start a run
  const run = await experimentRuns.create(experiment.id, 'output', 5);
  console.log('Started run:', run.id, 'status:', run.status);

  // 3. Wait for completion
  const result = await experimentRuns.subscribe(run.id, 300_000, 10_000);

  if (result.finalStatus === 'COMPLETED') {
    console.log('Experiment completed successfully!');
    console.log('Run details:', result.experimentRunData);
  } else {
    console.error('Experiment did not complete:', result.finalStatus);
    process.exit(1);
  }

  await testOps.shutdown();
}

runExperiment();

import os
from browserstack_ai_sdk import AISDK

client = AISDK(
    public_key=os.environ["AISDK_PUBLIC_KEY"],
    secret_key=os.environ["AISDK_SECRET_KEY"],
)

# 1. Create experiment
experiment = client.experiments.create({
    "name": "support-qa-eval",
    "evaluatorListId": "<evaluator-list-id>",
    "datasetId": "<dataset-id>",
    "promptId": "<prompt-id>",
})

# 2. Create a run
run = client.experiment_runs.create({
    "experimentId": experiment["id"],
    "name": "initial-run",
})

print(f"Experiment run created: {run['id']}")

import com.browserstack.aisdk.AISDK;
import com.browserstack.aisdk.eval.model.*;
import java.util.List;

public class ExperimentExample {
    public static void main(String[] args) throws InterruptedException {
        AISDK sdk = AISDK.fromEnv();

        // 1. Create an experiment
        ExperimentResponse experiment = sdk.experiments().create(
            CreateExperimentRequest.builder()
                .name("aurora-qa-v1")
                .evaluatorListId("evl_abc123")
                .promptId("prm_xyz789")
                .datasetId("aurora-qa")
                .build()
        );

        // 2. Start a run
        ExperimentRunResponse run = sdk.experimentRuns().create(
            CreateExperimentRunRequest.builder()
                .experimentId(experiment.getId())
                .build()
        );

        // 3. Wait for completion
        ExperimentRunResponse result = sdk.experimentRuns().subscribe(
            run.getId(),
            r -> System.out.printf("[%s] %s%n", r.getStatus(), r.getId())
        );

        System.out.println("Done: " + result.getStatus());
        sdk.shutdown();
    }
}

Experiments

On this page