{
  "kind": "contextfile.benchmarks/v1",
  "goldenSet": {
    "id": "contextfile-golden",
    "version": "0.1.0",
    "digest": "sha256:f4c8c07b8e1f68a90dfb35859350377d74159aa78ac97d89576dbe9fbe86d955",
    "works": 30,
    "targetWorks": 220,
    "claims": 180,
    "byVerdict": {
      "supported": 90,
      "wrong": 46,
      "misgrounded": 44
    },
    "agreement": {
      "doubleLabeled": 0,
      "observed": null,
      "kappa": null
    }
  },
  "latest": null,
  "history": [],
  "unpublished": 0,
  "liveJudge": {
    "version": "v0-dev",
    "hash": "sha256:1e64388843ffb61194ac8007b4992da06d5edd7ac2671ec35b1e82d8c2a9d04d"
  },
  "publicRepo": null,
  "reverify": {
    "mcp": "claims_verify",
    "rest": "POST /v1/reverify"
  },
  "definitions": {
    "precision": "Of claims the verifier passed, the share a person labeled supported.",
    "recall": "Of claims a person labeled supported, the share the verifier passed.",
    "misgroundedRate": "Of claims whose cited passage does not support them, the share the verifier passed.",
    "wrongPassRate": "Of false claims the cited passage contradicts, the share the verifier passed."
  },
  "toolSelection": {
    "harnessVersion": "selection-eval/1",
    "tasks": {
      "match": 12,
      "matchWaiting": 12,
      "control": 6
    },
    "latest": [],
    "unpublished": 0,
    "definitions": {
      "callRate": "Of tasks where a matching verified artifact exists, the share where the model's first tool call was a ContextFile tool.",
      "falseCallRate": "Of control tasks with no matching artifact, the share where the model called ContextFile anyway.",
      "validArgsRate": "Of ContextFile calls, the share whose arguments the tool's own schema accepts."
    }
  },
  "uplift": {
    "latest": null,
    "history": [],
    "cadenceDays": 30,
    "stale": false,
    "unpublished": 0,
    "taskSet": {
      "id": "contextfile-uplift",
      "version": "0.1.0",
      "digest": "sha256:918cc83dc239b45250e290adf50b4de1b7f021dbe2d3c3389f9c2d74e5484f15",
      "tasks": 6
    },
    "definitions": {
      "accuracy": "Of tasks, the share whose answer passed the task's checks.",
      "callRate": "Of tasks, the share where the model called a ContextFile tool at least once. Only the with arm has a server to call.",
      "tokensPerAnswer": "Mean prompt plus completion tokens per answered task.",
      "misgroundedRate": "Of ContextFile citations in answers, the share not backed by a source the model read in that run."
    }
  }
}