bench

See what the best agents do differently.

One line of code. Live dashboard, public profile, README badge.

npm install @virajmishra1/bench-sdk
export BENCH_KEY="bk_..."

import { observe } from "@virajmishra1/bench-sdk";

const agent = observe({ apiKey: process.env.BENCH_KEY, agent: "my-agent" });

await agent.task("search", { query }, async (t) => {
  const result = await doSearch(query);
  t.log("found", result.length);
  t.cost(0.004);
  return result;
});

pip install bench-observe
export BENCH_KEY="bk_..."

import bench

agent = bench.observe(api_key=os.environ["BENCH_KEY"], agent="my-agent")

async with agent.task_ctx("search", {"query": query}) as task:
    result = await do_search(query)
    task.log("found", len(result))
    task.set_output(result)

export OTEL_EXPORTER_OTLP_ENDPOINT=https://bench.virajmishratakehome.workers.dev
export OTEL_EXPORTER_OTLP_PROTOCOL=http/json
export OTEL_EXPORTER_OTLP_HEADERS="X-Bench-Key=bka_...,X-Bench-Agent=my-agent"

npx @virajmishra1/bench-cli init --install
npx @virajmishra1/bench-cli login

// Anthropic — wraps every messages.create() call
import { wrapAnthropic } from "@virajmishra1/bench-anthropic";
const client = wrapAnthropic(new Anthropic(), bench);

// OpenAI — wraps chat completions, responses, and embeddings
import { wrapOpenAI } from "@virajmishra1/bench-openai";
const client = wrapOpenAI(new OpenAI(), bench);

// Vercel AI SDK — wraps generateText / streamText / generateObject
import { track } from "@virajmishra1/bench-vercel-ai";
const result = await track(bench, "summarize", () =>
  generateText({ model: anthropic("claude-sonnet-4-6"), prompt: "..." })
);

// Mastra
import { wrapMastra } from "@virajmishra1/bench-mastra";

claude mcp add --transport http bench https://bench.virajmishratakehome.workers.dev/mcp

codex mcp add bench --url https://bench.virajmishratakehome.workers.dev/mcp

SDK (npm: @virajmishra1/bench-sdk)
        |  batched events, X-Bench-Key
        v
POST /ingest                              <- Workers (Hono)
        |
        +-> D1 --- users, agents, tasks, events
        |
        +-> AgentDO --- one Durable Object per agent
        |           +- ring buffer (last 1k events, SQLite in DO storage)
        |           +- latency histogram (p50, p95)
        |           +- hibernating WebSocket -> live dashboards
        |
        +-> EvalWorkflow --- runs per task.end
        |           +- Workers AI (Llama 3.3 70B) -> score 0-1 + reasoning
        |              -> writes back to D1.tasks
        |              -> updates agents.avg_eval_score
        |
        +-> ClusterWorkflow --- on-demand + hourly cron
                    +- k-means on task embeddings -> cluster labels
                       -> Workers AI LLM describes each cluster
                       -> stored in agents.failure_clusters

Public surfaces:
  /u/:login/:slug          -> profile page (server-rendered, OG image)
  /badge/:login/:slug.svg  -> README badge (KV-cached 60s)
  /embed/:login/:slug      -> iframe widget (3 sizes, dark/light)
  /leaderboard             -> discovery (5 sort modes)
  /vs/:a/:b                -> compare two agents
  /try                     -> sandbox (no signup)
  /benchmarks              -> verified benchmark registry
  /mcp                     -> read-only MCP server
  /api/agents/:l/:s/insights -> failure pattern analysis (JSON)

const agent = observe({
  apiKey: string;           // bk_xxx — from your dashboard
  agent: string;            // slug, e.g. "my-agent"
  displayName?: string;
  endpoint?: string;        // default: bench.virajmishratakehome.workers.dev
  flushIntervalMs?: number; // default: 2000
  maxBatchSize?: number;    // default: 50
});

// Wrap a task — records start/end/duration/status/eval automatically
await agent.task("name", input, async (task) => {
  task.log("label", value);    // attach a log event
  task.cost(0.003);            // report LLM spend (owner-reported)
  return result;               // returned value becomes the task output
});

// Fire a custom event
agent.event("custom", { key: "value" });

// Flush immediately (auto-runs on batch full or interval)
await agent.flush();

git clone https://github.com/VirajMishra1/bench
cd bench && npm install

cd packages/worker
npx wrangler login

# Create infrastructure
npx wrangler d1 create bench-db
npx wrangler kv namespace create CACHE
npx wrangler kv namespace create SESSIONS

# Paste the returned IDs into wrangler.jsonc, then:
npx wrangler secret put SESSION_SECRET             # any random 32+ char string
npx wrangler secret put GITHUB_OAUTH_CLIENT_SECRET # from github.com/settings/developers

# Apply schema and deploy
npm run db:remote
npm run deploy

bench/
+-- packages/
|   +-- sdk/                   <- @virajmishra1/bench-sdk
|   +-- adapters/
|   |   +-- anthropic/         <- @virajmishra1/bench-anthropic
|   |   +-- openai/            <- @virajmishra1/bench-openai
|   |   +-- vercel-ai/         <- @virajmishra1/bench-vercel-ai
|   |   +-- mastra/            <- @virajmishra1/bench-mastra
|   |   +-- langchain/         <- bench-langchain
|   +-- worker/                <- Cloudflare Worker (all backend + frontend)
|       +-- src/
|       |   +-- index.ts       <- Hono routes
|       |   +-- ingest.ts      <- POST /ingest
|       |   +-- profile.ts     <- public profile page
|       |   +-- badge.ts       <- SVG README badge
|       |   +-- embed.ts       <- iframe widget
|       |   +-- leaderboard.ts <- discovery page
|       |   +-- compare.ts     <- /vs/:a/:b
|       |   +-- do/agent.ts    <- AgentDO (actor per agent)
|       |   +-- workflows/
|       |       +-- eval.ts    <- LLM judge per task
|       |       +-- cluster.ts <- failure clustering
|       +-- public/            <- landing, dashboard, styles
|       +-- migrations/        <- D1 schema history
+-- benchmarks/
|   +-- grounded-research-v1/  <- example benchmark suite + cases
|   +-- eval-prompts.md        <- open-source judge prompts
+-- examples/                  <- runnable example agents