AI-Native Eval for Agents.

1import { defineEval, defineJudge } from "niceeval";
2import { pattern } from "niceeval/expect";
3
4const imageContext = defineJudge({ name: "image-context", rubric: "Does the last reply stay grounded in the earlier image?" });
5
6export default defineEval({
7  judge: imageContext,
8  description: "Evaluate an agent's multimodal ability across a multi-turn conversation",
9
10  async test(t) {
    const first = await t.sendFile("evals/sample.png", "What is in this image?");
    await first.succeeded().orStop();
    first.usedNoTools();
    const second = await t.send("What color is the background?");
    t.check(second.message, pattern(/blue|white|square/i));
    const third = await t.send("What color is the shape in the middle?");
17
18    await t.group("follow-ups stay grounded in the image context", () => {
      t.check(third.message, pattern(/white/i));
20    });
21
22    t.check({ earlierImageFacts: ["blue background", "white square"], question: third.input, answer: third.message }, imageContext.atLeast(0.7)).gate();
  },1/0.7
24});