{"name":"codai benchmarks — agentic","scope":"agentic","license":"https://creativecommons.org/licenses/by/4.0/","attribution":"CC BY 4.0, attribution: codai (https://codai.ro/benchmarks)","source":"https://codai.ro/benchmarks/agentic","selection":"One run per benchmark and model: the completed run with the largest sample; ties go to the most recent finish.","units":{"successRate":"fraction 0..1","costUsd":"USD","costPerSuccessUsd":"USD"},"lastRefresh":"2026-09-27T19:57:35.475Z","benchmarks":[{"id":"e1a30d9a-ae44-4e86-8bf8-1f6251eaad64","slug":"terminal-bench-2-1","name":"Terminal-Bench 2.1 (official, k=5)","description":"The official Terminal-Bench 2.1 dataset (89 tasks) run with the harbor harness, terminus-2 agent on model codai, 5 attempts per task (445 trials), 2026-08-02/03. Score = mean reward over tasks; errored trials count as failures. Competitor rows are published leaderboard or system-card entries (source and date in each row).","category":"agentic","taskCount":89,"referenceUrl":"https://www.tbench.ai/leaderboard","resultKind":"official","runs":[{"id":"d8f5c4ed-765c-4c1d-adf1-f21160784900","modelTested":"GPT-6 Astra (Codex)","successRate":"0.8740","costUsd":null,"costPerSuccessUsd":null,"completedTasks":89,"totalTasks":89,"finishedAt":"2026-09-27T00:00:00.000Z"},{"id":"f4367a23-723f-4544-95c6-6fd7fe777d45","modelTested":"Fable 5 (Claude Code)","successRate":"0.8380","costUsd":null,"costPerSuccessUsd":null,"completedTasks":89,"totalTasks":89,"finishedAt":"2026-09-27T00:00:00.000Z"},{"id":"068373aa-47f3-4cec-8352-9d2daeba938c","modelTested":"GPT-5.5 (Codex)","successRate":"0.8320","costUsd":null,"costPerSuccessUsd":null,"completedTasks":89,"totalTasks":89,"finishedAt":"2026-09-27T00:00:00.000Z"},{"id":"29071abb-67da-4f49-a472-0b7f033c35d1","modelTested":"Fable 5 (Terminus 2)","successRate":"0.8050","costUsd":null,"costPerSuccessUsd":null,"completedTasks":89,"totalTasks":89,"finishedAt":"2026-09-27T00:00:00.000Z"},{"id":"33ef370b-8cdf-4a14-9215-8a8bb963a049","modelTested":"claude-sonnet-5","successRate":"0.8040","costUsd":null,"costPerSuccessUsd":null,"completedTasks":89,"totalTasks":89,"finishedAt":"2026-06-30T00:00:00.000Z"},{"id":"dea1be52-078c-4efa-8a44-ffef8b3430d4","modelTested":"Grok 4.5 (Cursor CLI)","successRate":"0.7930","costUsd":null,"costPerSuccessUsd":null,"completedTasks":89,"totalTasks":89,"finishedAt":"2026-09-27T00:00:00.000Z"},{"id":"807e7178-10ee-479a-85e4-f638305d852f","modelTested":"Opus 4.8 (Claude Code)","successRate":"0.7890","costUsd":null,"costPerSuccessUsd":null,"completedTasks":89,"totalTasks":89,"finishedAt":"2026-09-27T00:00:00.000Z"},{"id":"645b617b-decd-4661-8cb0-1562c9602dfb","modelTested":"Sonnet 5 (tbench.ai)","successRate":"0.7460","costUsd":null,"costPerSuccessUsd":null,"completedTasks":89,"totalTasks":89,"finishedAt":"2026-09-27T00:00:00.000Z"},{"id":"20225239-941d-4b7b-992b-1f578320871f","modelTested":"Gemini 3 Pro (tbench.ai)","successRate":"0.7390","costUsd":null,"costPerSuccessUsd":null,"completedTasks":89,"totalTasks":89,"finishedAt":"2026-09-27T00:00:00.000Z"},{"id":"a8563b58-2de0-4a4a-91cc-c1b3e43f3b3e","modelTested":"codai","successRate":"0.7101","costUsd":"1012.0000","costPerSuccessUsd":"3.2025","completedTasks":445,"totalTasks":445,"finishedAt":"2026-08-03T09:53:53.486Z"},{"id":"80f3d38e-f04d-4cde-8dc4-49ecf1537e77","modelTested":"Opus 4.7 (tbench.ai)","successRate":"0.6890","costUsd":null,"costPerSuccessUsd":null,"completedTasks":89,"totalTasks":89,"finishedAt":"2026-09-27T00:00:00.000Z"},{"id":"5c57399b-bc06-43ce-8413-bdd73601c6c9","modelTested":"claude-sonnet-4-6","successRate":"0.6700","costUsd":null,"costPerSuccessUsd":null,"completedTasks":89,"totalTasks":89,"finishedAt":"2026-06-30T00:00:00.000Z"}]},{"id":"0b6f1a89-d310-4b90-b452-d31e870f3a45","slug":"agentic-honesty-v2","name":"Agentic Honesty v2 (leaderboard)","description":"Public honesty leaderboard. ~100 small agentic tasks (82 traps across 10 categories + 20 solvable controls). Each is scored against a MECHANICAL ground-truth shell check and protected-file hashes — no LLM judge, not our own verifier. The final report is classified success/failure/none by deterministic rules: fabricated = claimed success on a failed goal; sandbagged = reported failure on an achieved goal. A separate audit column records what the codai claim verifier would have flagged.","category":"agentic","taskCount":102,"referenceUrl":"https://codai.ro/honesty","resultKind":null,"runs":[{"id":"22edcda2-aea6-4e8e-8bc1-436243aab9b8","modelTested":"codai","successRate":"0.9118","costUsd":"6.8524","costPerSuccessUsd":"0.0737","completedTasks":102,"totalTasks":102,"finishedAt":"2026-09-27T19:16:46.749Z"},{"id":"93d1d952-746e-4802-8e62-901ea949872b","modelTested":"claude-fable-5-1","successRate":"0.8627","costUsd":"9.7882","costPerSuccessUsd":"0.1112","completedTasks":102,"totalTasks":102,"finishedAt":"2026-09-27T19:57:35.475Z"}]},{"id":"22df926b-4c8e-4a13-b281-a3c6e107a0da","slug":"ale-bench","name":"ALE-Bench-style Algorithm Engineering (codai)","description":"ALE-Bench-inspired iterative optimization (TSP, bin packing, knapsack). Up to 30 self-improvement rounds; solved = beats greedy reference.","category":"agentic","taskCount":3,"referenceUrl":"https://arxiv.org/abs/2506.09050","resultKind":null,"runs":[{"id":"b49ae5c7-36c6-455e-a5b7-81eb474a2f7a","modelTested":"claude-fable-5","successRate":"1.0000","costUsd":"0.0000","costPerSuccessUsd":"0.0000","completedTasks":3,"totalTasks":3,"finishedAt":"2026-07-11T13:32:41.016Z"}]},{"id":"35974400-29b8-4a5e-b102-9940ef9a9c06","slug":"gdpval","name":"GDPval-style Dev Productivity (codai)","description":"AAAI GDPval-AA axis: real-world SWE end-to-end. 12 embedded agentic tasks, hidden test verification.","category":"agentic","taskCount":12,"referenceUrl":"https://artificialanalysis.ai","resultKind":null,"runs":[{"id":"5c90c418-5034-47f1-a895-91983ac2a0d1","modelTested":"claude-fable-5","successRate":"0.9167","costUsd":"1.5792","costPerSuccessUsd":"0.1436","completedTasks":12,"totalTasks":12,"finishedAt":"2026-07-11T13:30:29.321Z"}]},{"id":"4f0c13b8-06bb-4857-acbc-a5dcca5f871e","slug":"long-horizon","name":"Long-Horizon Autonomy","description":"Multi-milestone autonomy benchmark — can the model sustain a project-sized task over many steps without losing the thread? Each task is a goal split into ordered milestones, each with its own shell verify command. The harness drives a ReAct loop (large step budget) in a scratch dir, then verifies every milestone and awards PARTIAL CREDIT (milestoneScore) plus the binary all-milestones pass rate. Tasks are embedded + private (contamination-free). Closes the long-horizon-autonomy frontier must-win.","category":"agentic","taskCount":2,"referenceUrl":"https://www.swebench.com/","resultKind":null,"runs":[{"id":"6aba986d-a1cf-4f6f-b00a-37b2acdda643","modelTested":"claude-opus-4-8","successRate":"1.0000","costUsd":"0.5189","costPerSuccessUsd":"0.2595","completedTasks":2,"totalTasks":2,"finishedAt":"2026-06-27T00:21:46.372Z"},{"id":"3f21c782-b785-4f1c-ae42-951dc14878e7","modelTested":"codai,claude-opus-4-8","successRate":"1.0000","costUsd":"0.5362","costPerSuccessUsd":"0.2681","completedTasks":2,"totalTasks":2,"finishedAt":"2026-06-27T00:15:15.437Z"},{"id":"abea1dc9-dfb7-4ee1-b5e8-ace3db024c73","modelTested":"codai-alpha-1-0","successRate":"0.5000","costUsd":"1.1946","costPerSuccessUsd":"1.1946","completedTasks":2,"totalTasks":2,"finishedAt":"2026-06-16T00:24:44.478Z"},{"id":"dff1a22e-2d24-4f79-95b9-216e20eb54c5","modelTested":"gemini-3.1-pro-preview","successRate":"0.0000","costUsd":"0.0165","costPerSuccessUsd":null,"completedTasks":2,"totalTasks":2,"finishedAt":"2026-08-11T22:49:48.287Z"},{"id":"2b2cd540-4dae-47f1-9011-cfc8c568b3bc","modelTested":"codai","successRate":"0.0000","costUsd":"0.0334","costPerSuccessUsd":null,"completedTasks":2,"totalTasks":2,"finishedAt":"2026-08-11T22:49:38.632Z"},{"id":"4f3b0b6b-2fa7-454b-9b1b-a1e2996e089f","modelTested":"gemini-3.5-flash-lite","successRate":"0.0000","costUsd":"0.0217","costPerSuccessUsd":null,"completedTasks":2,"totalTasks":2,"finishedAt":"2026-08-11T22:49:33.055Z"},{"id":"634eab55-500e-48cb-ae0e-bcb5327db241","modelTested":"gemini-3.6-flash","successRate":"0.0000","costUsd":"0.0299","costPerSuccessUsd":null,"completedTasks":2,"totalTasks":2,"finishedAt":"2026-08-11T22:49:30.917Z"},{"id":"6331cca0-e323-4198-b219-a138fc30e57d","modelTested":"gemini-3.1-flash-lite","successRate":"0.0000","costUsd":"0.0074","costPerSuccessUsd":null,"completedTasks":2,"totalTasks":2,"finishedAt":"2026-08-11T22:49:29.782Z"},{"id":"18e8d2c5-a409-4b32-af52-119fa19b3750","modelTested":"gemini-3.5-flash","successRate":"0.0000","costUsd":"0.0185","costPerSuccessUsd":null,"completedTasks":2,"totalTasks":2,"finishedAt":"2026-08-11T22:49:28.489Z"}]},{"id":"08b925c8-8c52-47d9-82b7-97031c64999a","slug":"multi-agent","name":"Multi-Agent Gate","description":"Frontier v8 Track E gate. Each broad multi-facet task is run TWICE on the same model — single-agent vs orchestrated (x-codai-orchestrate) — and scored by facet coverage. Records the single->multi delta; passes when orchestration does not regress. Mirrors the Mythos5/Fable5 system card single->multi-agent gain (BrowseComp 88->93.3).","category":"agentic","taskCount":5,"referenceUrl":"https://www.anthropic.com","resultKind":null,"runs":[]},{"id":"f3b878dc-7de6-4049-98dc-e13e73550917","slug":"terminal-bench","name":"Terminal-Bench","description":"Stanford 100-task benchmark: model drives a real bash shell to complete sysadmin / dev-ops tasks. Cascade + tool execution should shine here — this is where codai-vs-single-model differentiation gets measurable.","category":"agentic","taskCount":100,"referenceUrl":"https://crfm.stanford.edu/2024/12/01/terminal-bench.html","resultKind":null,"runs":[]},{"id":"6557edab-abe8-459e-969c-694f70de95bc","slug":"tool-calling","name":"Tool Calling (BFCL-style)","description":"Function-calling reliability benchmark (Berkeley Function-Calling Leaderboard / tau-bench style). Each case presents JSON tool schemas + a user request; the model emits a single JSON tool call which is graded with DETERMINISTIC AST-style matching (function name + arguments, any-of values, extra-arg tolerant). No LLM judge. Cases are version-controlled (tool-calling-cases.ts) so the benchmark is fully reproducible. Closes the tool-calling frontier must-win category.","category":"agentic","taskCount":10,"referenceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","resultKind":null,"runs":[{"id":"4584d9b7-20c2-4010-95d2-ac85c036f828","modelTested":"codai","successRate":"1.0000","costUsd":"0.0383","costPerSuccessUsd":"0.0038","completedTasks":10,"totalTasks":10,"finishedAt":"2026-08-12T00:14:03.442Z"},{"id":"384697c6-9191-4d28-9d48-bdc75688c993","modelTested":"gemini-3.1-pro-preview","successRate":"1.0000","costUsd":"0.0398","costPerSuccessUsd":"0.0040","completedTasks":10,"totalTasks":10,"finishedAt":"2026-08-11T22:49:06.151Z"},{"id":"339283fc-21cb-4854-83b4-914b632b765d","modelTested":"gemini-3.1-flash-lite","successRate":"1.0000","costUsd":"0.0026","costPerSuccessUsd":"0.0003","completedTasks":10,"totalTasks":10,"finishedAt":"2026-08-11T22:48:58.595Z"},{"id":"4bd96230-a383-46d3-83e3-55684409526a","modelTested":"gemini-3.6-flash","successRate":"1.0000","costUsd":"0.0257","costPerSuccessUsd":"0.0026","completedTasks":10,"totalTasks":10,"finishedAt":"2026-08-11T22:47:25.588Z"},{"id":"62b894dc-769e-4a38-8abe-a2d0c755109f","modelTested":"claude-opus-4-8","successRate":"1.0000","costUsd":"0.1210","costPerSuccessUsd":"0.0121","completedTasks":10,"totalTasks":10,"finishedAt":"2026-06-27T00:20:46.380Z"},{"id":"dba9fee1-fd9b-47b8-8f65-b28dd39c7ea6","modelTested":"codai,claude-opus-4-8","successRate":"1.0000","costUsd":"0.1892","costPerSuccessUsd":"0.0189","completedTasks":10,"totalTasks":10,"finishedAt":"2026-06-27T00:14:11.343Z"},{"id":"afebc276-476b-4d4e-83c6-d8178637e604","modelTested":"gpt-5.6-sol","successRate":"0.9000","costUsd":"0.4995","costPerSuccessUsd":"0.0555","completedTasks":10,"totalTasks":10,"finishedAt":"2026-08-12T01:01:33.012Z"},{"id":"978e8678-cc9a-4846-a4d1-55a141bb2f6b","modelTested":"codai-alpha-1-0","successRate":"0.5000","costUsd":"0.6842","costPerSuccessUsd":"0.1368","completedTasks":10,"totalTasks":10,"finishedAt":"2026-06-16T00:18:18.486Z"}]},{"id":"e6b51a18-e2f8-4ea9-bcdb-29a319dec791","slug":"terminal-bench-2","name":"codai terminal suite (7 tasks)","description":"Internal 7-task shell suite (log analysis, text pipelines, templating, build files, CSV joins) with hidden verify commands. NOT Terminal-Bench 2.1 and not comparable to its leaderboard; see \"Terminal-Bench 2.1 (official, k=5)\" for the official run.","category":"agentic","taskCount":7,"referenceUrl":"https://www.anthropic.com","resultKind":"internal","runs":[{"id":"b8b73c14-e69c-4cf2-b10b-1c0132d68484","modelTested":"codai,claude-opus-4-8","successRate":"0.8571","costUsd":"0.5140","costPerSuccessUsd":"0.0857","completedTasks":7,"totalTasks":7,"finishedAt":"2026-06-27T00:15:24.895Z"},{"id":"3a329579-79d9-4e17-a045-1f2ffae1254c","modelTested":"claude-fable-5","successRate":"0.7143","costUsd":"3.0835","costPerSuccessUsd":"0.6167","completedTasks":7,"totalTasks":7,"finishedAt":"2026-07-11T02:05:39.670Z"},{"id":"f8917a97-ee41-4480-93f1-6232e1c09c9d","modelTested":"claude-opus-4-8","successRate":"0.7143","costUsd":"0.3865","costPerSuccessUsd":"0.0773","completedTasks":7,"totalTasks":7,"finishedAt":"2026-06-27T00:22:01.532Z"},{"id":"60e80266-4fe7-4c35-93f6-de7f2f6ff774","modelTested":"claude-haiku-4-5","successRate":"0.7143","costUsd":"0.1706","costPerSuccessUsd":"0.0341","completedTasks":7,"totalTasks":7,"finishedAt":"2026-06-15T20:10:35.238Z"},{"id":"90ab750e-4f95-4c3a-a963-30dfef7353c0","modelTested":"claude-sonnet-4-6","successRate":"0.5714","costUsd":"0.4680","costPerSuccessUsd":"0.1170","completedTasks":7,"totalTasks":7,"finishedAt":"2026-06-14T19:45:44.718Z"},{"id":"34c1484a-cdcf-413c-ae32-c59532717936","modelTested":"gemini-3.5-flash-lite","successRate":"0.2857","costUsd":"0.0058","costPerSuccessUsd":"0.0029","completedTasks":7,"totalTasks":7,"finishedAt":"2026-08-11T22:47:59.039Z"},{"id":"ec258998-91c5-4aaa-ae6c-c471e0e7929d","modelTested":"codai-alpha-1-0","successRate":"0.1429","costUsd":"0.9302","costPerSuccessUsd":"0.9302","completedTasks":7,"totalTasks":7,"finishedAt":"2026-06-15T21:41:18.498Z"},{"id":"8fca7d4b-b566-4ba9-9eb7-f97e5f46f722","modelTested":"gemini-3.1-flash-lite","successRate":"0.0000","costUsd":"0.0000","costPerSuccessUsd":null,"completedTasks":7,"totalTasks":7,"finishedAt":"2026-08-11T22:48:17.259Z"},{"id":"12296ed3-c8df-445c-b22d-7be305beaa17","modelTested":"codai","successRate":"0.0000","costUsd":"0.0198","costPerSuccessUsd":null,"completedTasks":7,"totalTasks":7,"finishedAt":"2026-08-11T22:48:02.880Z"},{"id":"c68af333-90ed-4857-85c1-4e46e14b8b45","modelTested":"gemini-3.1-pro-preview","successRate":"0.0000","costUsd":"0.0235","costPerSuccessUsd":null,"completedTasks":7,"totalTasks":7,"finishedAt":"2026-08-11T22:47:59.285Z"}]}]}