Experiments

Experiments let you systematically evaluate a prompt + dataset + evaluator combination. Each run executes every dataset item through the prompt and scores the output.

Setup

import { AISDK } from '@browserstack/ai-sdk';

const testOps = new AISDK({
  publicKey: process.env.AISDK_PUBLIC_KEY,
  secretKey: process.env.AISDK_SECRET_KEY,
});

const experiments = testOps.experiments;       // Experiments
const experimentRuns = testOps.experimentRuns; // ExperimentRuns

import os
from browserstack_ai_sdk import AISDK

client = AISDK(
    public_key=os.environ["AISDK_PUBLIC_KEY"],
    secret_key=os.environ["AISDK_SECRET_KEY"],
)

import com.browserstack.aisdk.TestOps;
import com.browserstack.aisdk.eval.ExperimentsClient;
import com.browserstack.aisdk.eval.ExperimentRunsClient;

TestOps sdk = TestOps.fromEnv();
ExperimentsClient experiments = sdk.experiments();
ExperimentRunsClient experimentRuns = sdk.experimentRuns();

Create an Experiment

An experiment requires a name, an evaluator list, and either a datasetRunTagId or both promptId + datasetId.

With a prompt and dataset

import { CreateExperimentWithPromptRequest } from '@browserstack/ai-sdk';

const experiment = await experiments.create({
  name: 'qa-accuracy-test',
  description: 'Evaluate QA accuracy on golden set',
  promptId: 'prompt-id-abc',
  datasetId: 'dataset-id-xyz',
  evaluatorListId: 'eval-list-id-123',
  concurrency: 5,
} satisfies CreateExperimentWithPromptRequest);

console.log(experiment.id);

experiment = client.experiments.create({
    "name": "rag-eval-v2",
    "evaluatorListId": "eval-list-id",
    "datasetId": "dataset-id",
    "promptId": "prompt-id",
})

import com.browserstack.aisdk.eval.model.CreateExperimentRequest;
import com.browserstack.aisdk.eval.model.ExperimentResponse;

ExperimentResponse experiment = experiments.create(
    CreateExperimentRequest.builder()
        .name("gpt-4o-faithfulness-v2")
        .evaluatorListId("evl_abc123")
        .promptId("prm_xyz789")
        .datasetId("ds_def456")
        .concurrency(5)
        .build()
);

System.out.println("Experiment ID: " + experiment.getId());

You must provide either datasetRunTagId alone, or both promptId and datasetId together. Mixing them throws IllegalArgumentException.

With a dataset run tag

import { CreateExperimentWithTagRequest } from '@browserstack/ai-sdk';

const experiment = await experiments.create({
  name: 'tag-based-experiment',
  datasetRunTagId: 'tag-id-abc',
  evaluatorListId: 'eval-list-id-123',
  concurrency: 3,
} satisfies CreateExperimentWithTagRequest);

experiment = client.experiments.create({
    "name": "rag-eval-v1",
    "evaluatorListId": "eval-list-id",
    "datasetRunTagId": "tag-id-from-dataset-run",
})
print(experiment)

List and Get Experiments

const result = await experiments.list(
  20, // limit (1-100)
  1   // page
);

for (const exp of result.experiments) {
  console.log(exp.id, exp.name, exp.createdAt);
}

const experiment = await experiments.find('experiment-id-abc');
console.log(experiment.name, experiment.status);

experiments = client.experiments.list(page=1, limit=50)

experiment = client.experiments.get("experiment-id")
print(experiment)

ListExperimentsResponse list = experiments.list(20, 1);
list.getData().forEach(e -> System.out.println(e.getId() + " " + e.getName()));

ExperimentResponse experiment = experiments.find("exp_abc123");
System.out.println("Status: " + experiment.getStatus());

Create and Monitor Runs

Create a Run

const run = await experimentRuns.create(
  'experiment-id-abc', // experimentId
  'output',            // llmColumnName
  5,                   // concurrency (1-100)
  { temperature: 0.0 } // optional runConfig
);

console.log(run.id, run.status); // 'PENDING'

run = client.experiment_runs.create({
    "experimentId": "experiment-id",
    "name": "run-2026-04-01",
})
print(run)

import com.browserstack.aisdk.eval.model.CreateExperimentRunRequest;
import com.browserstack.aisdk.eval.model.ExperimentRunResponse;

ExperimentRunResponse run = experimentRuns.create(
    CreateExperimentRunRequest.builder()
        .experimentId(experiment.getId())
        .build()
);

System.out.println("Run ID: " + run.getId());

Poll for Completion

const result = await experimentRuns.subscribe(
  run.id,
  120_000, // timeout in ms
  5_000    // poll interval in ms
);

if (result.finalStatus === 'COMPLETED') {
  console.log('Run completed:', result.experimentRunData);
} else {
  console.error('Run failed or timed out:', result.finalStatus);
}

// Default: polls every 5s, times out after 5 minutes
ExperimentRunResponse finalRun = experimentRuns.subscribe(
    run.getId(),
    update -> System.out.println("Status: " + update.getStatus())
);

System.out.println("Final status: " + finalRun.getStatus());

With a custom timeout:

long tenMinutes = 10 * 60 * 1_000L;

ExperimentRunResponse finalRun = experimentRuns.subscribe(
    run.getId(),
    update -> System.out.println("[" + update.getStatus() + "] " + update.getProgress()),
    tenMinutes
);

Summarize a Run

summarize() returns scores, metrics, diffs, and UI deep-links for a completed run in a single call. Use it in CI to gate on threshold status, post PR comments, or feed dashboards.

Parameters

Parameter	Default	Description
`compare_to_run_id` / `compareToRunId` / `compareToRunID`	UI-pinned baseline (or none)	Explicit baseline run ID. When omitted (or `""` in Go, `None` in Python, not passed in TypeScript/Java), the server uses the experiment's UI-pinned baseline (set via "Set as baseline" in the experiment view). If neither is set, no comparison is performed and every `diff` is `null`. The server does NOT auto-pick a baseline by recency.

Basic call

const summary = await experimentRuns.summarize("run-uuid-1");

console.log(summary.summary.verdict);       // "PASS" | "REGRESSION"
console.log(summary.urls.scoresTab);        // deep-link to Scores tab in the UI
const exactMatch = summary.scores.find(s => s.name === "exact_match");
console.log(exactMatch?.thresholdStatus); // "pass" | "fail"

Pass an options object to pin the baseline:

// With an explicit baseline
const summary = await experimentRuns.summarize("run-uuid-1", {
  compareToRunId: "run-uuid-0",
});

summary = client.experiment_runs.summarize("run-uuid-1")

print(summary.summary.verdict)                              # "PASS" or "REGRESSION"
print(summary.urls.scores_tab)                              # deep-link to Scores tab
exact_match = next((s for s in summary.scores if s.name == "exact_match"), None)
print(exact_match.threshold_status if exact_match else None)  # "pass" or "fail"

With an explicit baseline:

summary = client.experiment_runs.summarize(
    "run-uuid-1",
    compare_to_run_id="run-uuid-0",
)

ExperimentRunSummary summary = experimentRuns.summarize(runId);

System.out.println(summary.getSummary().getVerdict());                              // "PASS" or "REGRESSION"
System.out.println(summary.getUrls().getScoresTab());                               // deep-link to Scores tab
ExperimentRunScoreSummary exactMatch = summary.getScores().stream()
    .filter(s -> "exact_match".equals(s.getName()))
    .findFirst()
    .orElse(null);
System.out.println(exactMatch != null ? exactMatch.getThresholdStatus() : null);    // "pass" or "fail"

Two overloads are available, matching the list() / get() pattern used elsewhere in the Java SDK:

// Uses UI-pinned baseline if any, else no comparison
ExperimentRunSummary summary = experimentRuns.summarize(runId);

// Explicit baseline
ExperimentRunSummary summary = experimentRuns.summarize(runId, "run-uuid-0");

summary, err := client.ExperimentRuns.Summarize(ctx, "run-uuid-1", "")
if err != nil {
    log.Fatal(err)
}

fmt.Println(summary.Summary.Verdict)                              // "PASS" or "REGRESSION"
fmt.Println(summary.Urls.ScoresTab)                               // deep-link to Scores tab
var exactMatch *crud.ExperimentRunScoreSummary
for i := range summary.Scores {
    if summary.Scores[i].Name == "exact_match" {
        exactMatch = &summary.Scores[i]
        break
    }
}
if exactMatch != nil {
    fmt.Println(exactMatch.ThresholdStatus)                       // "pass" or "fail"
}

Pass an empty string for compareToRunID to use auto-resolution.

// Uses UI-pinned baseline if any, else no comparison
summary, err := client.ExperimentRuns.Summarize(ctx, "run-uuid-1", "")

// Explicit baseline
summary, err := client.ExperimentRuns.Summarize(ctx, "run-uuid-1", "run-uuid-0")

CI usage — post a PR comment

A common pattern is running summarize() at the end of a CI job and posting the result as a pull-request comment.

// GitHub Actions step — gate on regression and post a PR comment.
// The SDK returns structured `scores` and `metrics`; render the comment
// however you like (table, one-line summary, Slack blocks, etc.).
const summary = await experimentRuns.summarize(run.id);

if (summary.summary.verdict === "REGRESSION") {
  console.error(`Run regressed: ${summary.summary.regressedCount} scorer(s) degraded`);
  process.exit(1);
}

const scoreLines = summary.scores.map(
  (s) => `- **${s.name}** — ${s.score?.toFixed(3) ?? "—"} (${s.thresholdStatus ?? "—"})`,
);
const body = [
  `**Eval summary — ${summary.summary.verdict}**`,
  ...scoreLines,
  `\n[View full scores](${summary.urls.scoresTab})`,
].join("\n");

await octokit.issues.createComment({ body, /* owner, repo, issue_number */ });

# GitHub Actions step — gate on regression and post a PR comment.
# The SDK returns structured `scores` and `metrics`; render the comment
# however you like (table, one-line summary, Slack blocks, etc.).
summary = client.experiment_runs.summarize(run["id"])

if summary.summary.verdict == "REGRESSION":
    print(f"Run regressed: {summary.summary.regressed_count} scorer(s) degraded")
    sys.exit(1)

score_lines = [
    f"- **{s.name}** — {s.score:.3f} ({s.threshold_status or '—'})"
    for s in summary.scores
    if s.score is not None
]
body = "\n".join([
    f"**Eval summary — {summary.summary.verdict}**",
    *score_lines,
    f"\n[View full scores]({summary.urls.scores_tab})",
])

github.create_pull_request_comment(body=body)

// GitHub Actions step — gate on regression and post a PR comment.
// The SDK returns structured `scores` and `metrics`; render the comment
// however you like (table, one-line summary, Slack blocks, etc.).
ExperimentRunSummary summary = experimentRuns.summarize(run.getId());

if ("REGRESSION".equals(summary.getSummary().getVerdict())) {
    System.err.println("Run regressed: "
        + summary.getSummary().getRegressedCount() + " scorer(s) degraded");
    System.exit(1);
}

StringBuilder body = new StringBuilder("**Eval summary — ")
    .append(summary.getSummary().getVerdict()).append("**\n");
for (ExperimentRunScoreSummary s : summary.getScores()) {
    body.append("- **").append(s.getName()).append("** — ")
        .append(s.getScore() != null ? String.format("%.3f", s.getScore()) : "—")
        .append(" (").append(s.getThresholdStatus() != null ? s.getThresholdStatus() : "—").append(")\n");
}
body.append("\n[View full scores](").append(summary.getUrls().getScoresTab()).append(")");

githubClient.createPullRequestComment(body.toString());

// GitHub Actions step — gate on regression and post a PR comment.
// The SDK returns structured `scores` and `metrics`; render the comment
// however you like (table, one-line summary, Slack blocks, etc.).
summary, err := client.ExperimentRuns.Summarize(ctx, run.ID, "")
if err != nil {
    log.Fatal(err)
}

if summary.Summary.Verdict == "REGRESSION" {
    fmt.Fprintf(os.Stderr, "Run regressed: %d scorer(s) degraded\n",
        summary.Summary.RegressedCount)
    os.Exit(1)
}

var lines []string
lines = append(lines, fmt.Sprintf("**Eval summary — %s**", summary.Summary.Verdict))
for _, s := range summary.Scores {
    score := "—"
    if s.Score != nil {
        score = fmt.Sprintf("%.3f", *s.Score)
    }
    status := "—"
    if s.ThresholdStatus != nil {
        status = *s.ThresholdStatus
    }
    lines = append(lines, fmt.Sprintf("- **%s** — %s (%s)", s.Name, score, status))
}
lines = append(lines, fmt.Sprintf("\n[View full scores](%s)", summary.Urls.ScoresTab))
body := strings.Join(lines, "\n")
githubClient.CreatePullRequestComment(ctx, body)

Response shape

The returned object has these top-level fields:

Field	Type	Description
`projectId`	`string`	Project ID
`projectName`	`string`	Project name
`experimentId`	`string`	Experiment ID
`experimentName`	`string`	Experiment name
`experimentRunId`	`string`	This run's ID
`experimentRunName`	`string`	This run's name
`comparisonExperimentRunId`	`string \| null`	Baseline run ID (`null` when no baseline)
`comparisonExperimentRunName`	`string \| null`	Baseline run name (`null` when no baseline)
`urls.project`	`string`	UI link to the project
`urls.experiment`	`string`	UI link to the experiment
`urls.run`	`string`	UI link to the run page
`urls.scoresTab`	`string`	Direct link to the Scores tab
`urls.analyticsTab`	`string`	Direct link to the Analytics tab
`scores`	`array`	Ordered list of evaluator score summaries — worst-regressed first; look up by `name` (see below)
`metrics`	`object`	Map of metric name → metric summary (see below)
`summary.verdict`	`"PASS" \| "REGRESSION"`	Overall run verdict
`summary.failureCount`	`number`	Scorers below threshold
`summary.improvedCount`	`number`	Scorers that improved vs baseline
`summary.regressedCount`	`number`	Scorers that regressed vs baseline

Each score summary includes thresholdStatus ("pass" or "fail"), diff vs baseline, improvements, and regressions per evaluator. Numeric scores additionally carry passCount, failCount, and totalItems; categorical scores carry categoryCounts and passCategories.

Each metric summary includes metric (the aggregate value), unit (s, ms, tok, $), and diff vs baseline.

Run Status Values

enum ExperimentRunStatus {
  PENDING    = 'PENDING',
  RUNNING    = 'RUNNING',
  COMPLETED  = 'COMPLETED',
  FAILED     = 'FAILED',
  CANCELLED  = 'CANCELLED',
}

Complete End-to-End Example

import { AISDK, ExperimentRunStatus } from '@browserstack/ai-sdk';

const testOps = new AISDK({
  publicKey: process.env.AISDK_PUBLIC_KEY,
  secretKey: process.env.AISDK_SECRET_KEY,
});

async function runExperiment() {
  const experiments = testOps.experiments;
  const experimentRuns = testOps.experimentRuns;

  // 1. Create the experiment
  const experiment = await experiments.create({
    name: `accuracy-test-${Date.now()}`,
    description: 'Automated accuracy evaluation',
    promptId: process.env.PROMPT_ID!,
    datasetId: process.env.DATASET_ID!,
    evaluatorListId: process.env.EVALUATOR_LIST_ID!,
    concurrency: 5,
  });

  console.log('Created experiment:', experiment.id);

  // 2. Start a run
  const run = await experimentRuns.create(experiment.id, 'output', 5);
  console.log('Started run:', run.id, 'status:', run.status);

  // 3. Wait for completion
  const result = await experimentRuns.subscribe(run.id, 300_000, 10_000);

  if (result.finalStatus === 'COMPLETED') {
    console.log('Experiment completed successfully!');
    console.log('Run details:', result.experimentRunData);
  } else {
    console.error('Experiment did not complete:', result.finalStatus);
    process.exit(1);
  }

  await testOps.shutdown();
}

runExperiment();

import os
from browserstack_ai_sdk import AISDK

client = AISDK(
    public_key=os.environ["AISDK_PUBLIC_KEY"],
    secret_key=os.environ["AISDK_SECRET_KEY"],
)

# 1. Create experiment
experiment = client.experiments.create({
    "name": "support-qa-eval",
    "evaluatorListId": "<evaluator-list-id>",
    "datasetId": "<dataset-id>",
    "promptId": "<prompt-id>",
})

# 2. Create a run
run = client.experiment_runs.create({
    "experimentId": experiment["id"],
    "name": "initial-run",
})

print(f"Experiment run created: {run['id']}")

import com.browserstack.aisdk.TestOps;
import com.browserstack.aisdk.eval.model.*;
import java.util.List;

public class ExperimentExample {
    public static void main(String[] args) throws InterruptedException {
        TestOps sdk = TestOps.fromEnv();

        // 1. Create an experiment
        ExperimentResponse experiment = sdk.experiments().create(
            CreateExperimentRequest.builder()
                .name("aurora-qa-v1")
                .evaluatorListId("evl_abc123")
                .promptId("prm_xyz789")
                .datasetId("aurora-qa")
                .build()
        );

        // 2. Start a run
        ExperimentRunResponse run = sdk.experimentRuns().create(
            CreateExperimentRunRequest.builder()
                .experimentId(experiment.getId())
                .build()
        );

        // 3. Wait for completion
        ExperimentRunResponse result = sdk.experimentRuns().subscribe(
            run.getId(),
            r -> System.out.printf("[%s] %s%n", r.getStatus(), r.getId())
        );

        System.out.println("Done: " + result.getStatus());
        sdk.shutdown();
    }
}

Experiments

On this page