AI-Native Eval for Agents.

1import { defineEval } from "niceeval";
2
3export default defineEval({
4  description: "Live weather must go through get_weather — no making it up",
5
6  async test(t) {
7    const turn = await t.send("What's the weather in Beijing today?");
8    turn.expectOk();
9
10    t.calledTool("get_weather", { input: { city: "Beijing" } });
11    t.notCalledTool("web_search");
12    t.eventOrder(["operation.started", "operation.finished", "message"]);
13    t.messageIncludes(/°C|sunny|cloudy|rain/i);
14
15    await t.group("stays within budget", () => {
16      t.maxToolCalls(2);
17      t.maxCost(0.05);
18    });
19
20    t.judge.autoevals
21      .closedQA("Did the assistant give concrete weather data instead of hedging?")
22      .atLeast(0.7);
23  },
24});

1import { defineEval } from "niceeval";
2import { includes } from "niceeval/expect";
3
4export default defineEval({
5  description: "Ask a coding agent to sum a CSV column, then verify the artifact",
6
7  async test(t) {
8    await t.sandbox.writeFiles({ "sales.csv": "id,amount\n1,100.5\n2,200.0\n3,50.5" });
9
10    const turn = await t.send("Write sum_sales.py that prints the total of the amount column.");
11    turn.expectOk();
12
13    t.fileChanged("sum_sales.py");
14
15    const run = await t.sandbox.runCommand("python3", ["sum_sales.py"]);
16    t.check(run.stdout.trim(), includes(/^351(\.0)?$/));
17
18    const code = await t.sandbox.readFile("sum_sales.py");
19    t.judge.autoevals
20      .closedQA("Does the code use the csv stdlib instead of pandas, and stay concise?", { on: code })
21      .atLeast(0.7);
22  },
23});