[
  {
    "id": "hle",
    "name": "Humanity's Last Exam",
    "short": "HLE",
    "emoji_skill": "🧠 Reasoning",
    "measures": "Expert-written questions across dozens of subjects, built to be hard enough that today's best models still fail most of them.",
    "n_items": 2500,
    "meaningful_gap": 2.0,
    "trust_grade": "B",
    "lifecycle": "current",
    "variants": [
      "no_tools",
      "with_tools"
    ],
    "sample_question": "",
    "source_url": "https://arxiv.org/abs/2501.14249",
    "notes": "2,500 questions from ~1,000 experts across 500+ institutions. Not saturated — top model as of Aug 2026 scores well under the ~88-90% range where benchmarks stop discriminating."
  },
  {
    "id": "terminal-bench-2-1",
    "name": "Terminal-Bench 2.1",
    "short": "Terminal-Bench 2.1",
    "emoji_skill": "💻 Terminal ops",
    "measures": "Whether an AI agent can actually operate a real command line to debug code, administer systems, and fix security issues.",
    "n_items": 89,
    "meaningful_gap": 10.6,
    "trust_grade": "B",
    "lifecycle": "current",
    "variants": [
      "default"
    ],
    "sample_question": "",
    "source_url": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench-2-1/6",
    "notes": "89 tasks; a hardened revision of 2.0 with 26 tasks fixed for bugs and reward-hacking. Small n means single-digit-point gaps are noise."
  },
  {
    "id": "deepswe",
    "name": "DeepSWE",
    "short": "DeepSWE",
    "emoji_skill": "🛠️ Long-horizon coding",
    "measures": "Original, never-before-public software engineering tasks, built so agents can't just recall memorized GitHub fixes.",
    "n_items": 113,
    "meaningful_gap": 9.4,
    "trust_grade": "B",
    "lifecycle": "current",
    "variants": [
      "default"
    ],
    "sample_question": "",
    "source_url": "https://arxiv.org/abs/2607.07946",
    "notes": "113 tasks across 91 repos, 5 languages. Distinct 2026 benchmark — not the same thing as the earlier 'DeepSWE-Preview' model. Hand-written functional verifiers, low disagreement with independent judges."
  },
  {
    "id": "toolathlon-verified",
    "name": "Toolathlon-Verified",
    "short": "Toolathlon-Verified",
    "emoji_skill": "🧰 Multi-tool chores",
    "measures": "Realistic multi-step chores that require an agent to juggle many real software tools over a long session, not just answer questions.",
    "n_items": 108,
    "meaningful_gap": 9.6,
    "trust_grade": "B",
    "lifecycle": "current",
    "variants": [
      "default"
    ],
    "sample_question": "",
    "source_url": "https://github.com/hkust-nlp/Toolathlon",
    "notes": "108 tasks across 32 MCP servers / 600+ tools. 'Verified' revision hardened the grading and isolated task state after the original Toolathlon shipped some scoring bugs."
  },
  {
    "id": "agents-last-exam",
    "name": "Agents' Last Exam",
    "short": "Agents' Last Exam",
    "emoji_skill": "💼 Professional work",
    "measures": "Real, economically valuable professional work — video editing, CAD, manufacturing simulation — done end-to-end.",
    "n_items": 1000,
    "meaningful_gap": 3.2,
    "trust_grade": "A",
    "lifecycle": "current",
    "variants": [
      "default"
    ],
    "sample_question": "",
    "source_url": "https://arxiv.org/abs/2606.05405",
    "notes": "1,000+ tasks across 55 sub-fields from 300+ industry experts. Explicitly refreshed every ~6 months as a living benchmark to resist contamination — the best trust grade in this set for that reason. Metric pinned 2026-08-17: our column stores the board's OVERALL-split pass_rate_pct for each model's best harness config (the board also publishes a partial-credit score_pct and an ALE-CLI split — do not mix; an earlier version of this dataset did)."
  },
  {
    "id": "swe-bench-verified",
    "name": "SWE-bench Verified",
    "short": "SWE-bench Verified",
    "emoji_skill": "🐛 Bug fixing",
    "measures": "Gives an agent a real closed GitHub issue and checks whether its patch fixes it and passes the project's real test suite.",
    "n_items": 500,
    "meaningful_gap": 4.5,
    "trust_grade": "D",
    "lifecycle": "saturated",
    "variants": [
      "default"
    ],
    "sample_question": "",
    "source_url": "https://www.swebench.com/verified.html",
    "notes": "500 tasks, human-filtered from the original SWE-bench with OpenAI. Frontier models now cluster in the mid-90s% — widely regarded as saturated, with independent audits raising contamination concerns at the top of the leaderboard. Treat close rankings here as unreliable; the field is shifting to SWE-bench Pro."
  },
  {
    "id": "gpqa-diamond",
    "name": "GPQA Diamond",
    "short": "GPQA Diamond",
    "emoji_skill": "🔬 Expert science Q&A",
    "measures": "PhD-level science questions written to be 'Google-proof' — hard even for skilled non-experts with internet access.",
    "n_items": 198,
    "meaningful_gap": 7.1,
    "trust_grade": "C",
    "lifecycle": "aging",
    "variants": [
      "default"
    ],
    "sample_question": "",
    "source_url": "https://arxiv.org/abs/2311.12022",
    "notes": "198 questions, the hardest cut of an original 448-question set. Top models now cluster within 1-3 points of each other near the ceiling — differences here are increasingly noise, not signal."
  },
  {
    "id": "livecodebench",
    "name": "LiveCodeBench",
    "short": "LiveCodeBench",
    "emoji_skill": "⌨️ Contest coding",
    "measures": "Coding contest problems dated after each model's training cutoff, so scores can't be inflated by memorization.",
    "n_items": 1055,
    "meaningful_gap": 3.1,
    "trust_grade": "B",
    "lifecycle": "current",
    "variants": [
      "default"
    ],
    "sample_question": "",
    "source_url": "https://github.com/LiveCodeBench/LiveCodeBench",
    "notes": "1,055 problems in the current release (release_v6). Grows over time by design; only compare scores from the same release version."
  }
]