更适合 Agent 的评估。

1import { defineEval } from "niceeval";
2
3export default defineEval({
4  description: "实时天气必须走 get_weather 工具，不许编造",
5
6  async test(t) {
7    const turn = await t.send("北京今天天气怎么样？");
8    turn.expectOk();
9
10    t.calledTool("get_weather", { input: { city: "北京" } });
11    t.notCalledTool("web_search");
12    t.eventOrder(["operation.started", "operation.finished", "message"]);
13    t.messageIncludes(/°C|晴|多云|雨/);
14
15    await t.group("预算内完成", () => {
16      t.maxToolCalls(2);
17      t.maxCost(0.05);
18    });
19
20    t.judge.autoevals
21      .closedQA("助手是否给出了具体的天气数据，而不是含糊其辞？")
22      .atLeast(0.7);
23  },
24});

1import { defineEval } from "niceeval";
2import { includes } from "niceeval/expect";
3
4export default defineEval({
5  description: "让 coding agent 求和 CSV 列，然后验证产物与运行结果",
6
7  async test(t) {
8    await t.sandbox.writeFiles({ "sales.csv": "id,amount\n1,100.5\n2,200.0\n3,50.5" });
9
10    const turn = await t.send("写一个 sum_sales.py，打印 amount 列的总和。");
11    turn.expectOk();
12
13    t.fileChanged("sum_sales.py");
14
15    const run = await t.sandbox.runCommand("python3", ["sum_sales.py"]);
16    t.check(run.stdout.trim(), includes(/^351(\.0)?$/));
17
18    const code = await t.sandbox.readFile("sum_sales.py");
19    t.judge.autoevals
20      .closedQA("代码是否用 csv 标准库而非 pandas，并保持简洁？", { on: code })
21      .atLeast(0.7);
22  },
23});