import { Experiments, ExperimentRuns } from "@browserstack/ai-sdk";
import OpenAI from "openai";

const openai = new OpenAI(); // reads OPENAI_API_KEY from env

const experiment = await Experiments.create({
  name: "qa-golden-set",
  dataset: () => [
    { input: "What is the capital of France?", expected: "Paris" },
    { input: "What is 2 + 2?", expected: "4" },
  ],
  evaluators: [
    ({ output, expected }) => ({
      name: "exact_match",
      score: String(output).trim() === String(expected).trim() ? 1 : 0,
    }),
  ],
});

const result = await ExperimentRuns.create(experiment, {
  task: async (input) => {
    const response = await openai.chat.completions.create({
      model: "gpt-4o-mini",
      messages: [{ role: "user", content: String(input) }],
    });
    return response.choices[0].message.content ?? "";
  },
});

console.log("Run URL:", result.uiUrl);
console.log("Comparison:", result.comparison?.summary.verdict);

import os
from browserstack_ai_sdk import Experiments, ExperimentRuns
import openai

_openai = openai.OpenAI()  # reads OPENAI_API_KEY from env

def exact_match(args):
    output = str(args.get("output", "")).strip()
    expected = str(args.get("expected", "")).strip()
    return {"name": "exact_match", "score": 1 if output == expected else 0}

def my_task(input_val):
    response = _openai.chat.completions.create(
        model="gpt-4o-mini",
        messages=[{"role": "user", "content": str(input_val)}],
    )
    return response.choices[0].message.content or ""

experiment = Experiments.create({
    "name": "qa-golden-set",
    "dataset": lambda: [
        {"input": "What is the capital of France?", "expected": "Paris"},
        {"input": "What is 2 + 2?", "expected": "4"},
    ],
    "evaluators": [exact_match],
})

result = ExperimentRuns.create(experiment, {"task": my_task})

print("Run URL:", result["uiUrl"])

import { Experiments, LocalExperimentCreateOptions } from "@browserstack/ai-sdk";

const experiment = await Experiments.create({
  name: "qa-golden-set",          // required — upserted by (project, name)
  dataset: () => [...],           // see Dataset sources below
  evaluators: [myEvaluator],      // see Evaluator sources below
  evaluatorListId: "el_abc123",   // optional — merge with inline evaluators
  description: "weekly eval",     // optional
  metadata: { owner: "ml-team" }, // optional — stored on the run
});
// Returns LocalExperimentHandle

from browserstack_ai_sdk import Experiments

experiment = Experiments.create({
    "name": "qa-golden-set",
    "dataset": lambda: [...],
    "evaluators": [my_evaluator],
    "evaluatorListId": "el_abc123",   # optional
    "description": "weekly eval",     # optional
    "metadata": {"owner": "ml-team"}, # optional
})
# Returns a dict handle used in Step 2

import { ExperimentRuns } from "@browserstack/ai-sdk";

const result = await ExperimentRuns.create(experiment, {
  task: async (input) => {
    // input is one dataset item's `input` field
    // return the model output (string, object, or any serializable value)
    return await callMyLLM(input);
  },
  maxConcurrency: 5,   // optional — default: 1
  trialCount: 1,       // optional — repeat each item N times
  timeout: 30_000,     // optional — overall run timeout in ms (whole-run deadline)
  hooks: {
    onTestCase: (item, result) => console.log(item.input, result.scores),
    onScore: (item, scores) => console.log(item.input, scores),
    onComplete: (summary) => console.log(summary.successCount, "passed"),
  },
});

from browserstack_ai_sdk import ExperimentRuns

result = ExperimentRuns.create(experiment, {
    "task": my_task,
    "maxConcurrency": 5,    # optional — default: 1
    "trialCount": 1,        # optional
    "timeout": 30000,       # optional — overall run timeout in ms
    "hooks": {              # optional — onTestCase / onScore / onComplete
        "onComplete": lambda summary: print(summary["successCount"], "passed"),
    },
})

task: async (input) => {
  const params = JSON.parse(process.env.AIEVALS_PARAMS_JSON || "{}");
  const response = await openai.chat.completions.create({
    model: params.model ?? "gpt-4o-mini",
    messages: [{ role: "user", content: String(input) }],
  });
  return response.choices[0].message.content ?? "";
},

import os, json

def my_task(input_val):
    params = json.loads(os.environ.get("AIEVALS_PARAMS_JSON", "{}"))
    return _openai.chat.completions.create(
        model=params.get("model", "gpt-4o-mini"),
        messages=[{"role": "user", "content": str(input_val)}],
    ).choices[0].message.content

if (result.comparison?.summary.verdict === "REGRESSION") {
  console.error("Regression detected. See:", result.uiUrl);
  process.exit(1);
}

comparison = result.get("comparison") or {}
if comparison.get("summary", {}).get("verdict") == "REGRESSION":
    import sys
    print("Regression detected. See:", result["uiUrl"])
    sys.exit(1)

const exactMatch = ({ output, expected }) => ({
  name: "exact_match",
  score: String(output).trim() === String(expected).trim() ? 1 : 0,
});

def exact_match(args):
    output = str(args.get("output", "")).strip()
    expected = str(args.get("expected", "")).strip()
    return {"name": "exact_match", "score": 1 if output == expected else 0}