{ "schema_version": "0.13", "generation_sampled": true, "skill": { "name": "hyperframes-cli", "version": "0.0.0", "content_hash": "93bc719ae762f5f4ac4eaa3be58529d9f48dbbcbe90ac07ba20b40a0965ca3b6", "tokens": 6491 }, "suite": { "format": "agentskills.io/evals", "suite_hash": "a189a1665384ef24c2ea9d2aac5bb8762cad4d107c47f7c5612dcd2c8f5ea707", "case_count": 3, "canary": "6a82ae2b-5d88-7987-8468-0ede008b3d69" }, "run": { "model_id": "claude-opus-5-5", "model_release_date": null, "provider": "anthropic", "surface": "claude-cli", "runner_version": "0.22.2", "date_utc": "2026-10-18T07:34:02.114Z", "generated_at": "2026-10-18T07:18:29.465Z", "registry": "registered", "transcripts": "retained-local", "judge": { "samples": 5, "temperature": null, "sampling": "surface-controlled", "surface": "claude-cli", "model_id": "claude-haiku-5-5", "prompt_template_hash": "82586d1e44f84df2b8ce5f18ac777866465cb5ad9686503c93d9776c93c344ed", "harness": { "name": "claude-code", "version": "2.1.287" }, "isolation": "eval-user", "endpoint": { "custom": false, "sha256": null, "resolved_by": "cli", "routing": { "ANTHROPIC_BEDROCK_BASE_URL": { "set": false, "sha256": null }, "ANTHROPIC_VERTEX_BASE_URL": { "set": false, "sha256": null }, "CLAUDE_CODE_USE_BEDROCK": { "set": false, "sha256": null }, "CLAUDE_CODE_USE_VERTEX": { "set": false, "sha256": null }, "HTTPS_PROXY": { "set": false, "sha256": null } } }, "code_revision": "judge-code-bb552899a1df887e", "reported_models": [ "claude-haiku-5-5" ] }, "answered_by": { "kind": "model", "attested": true, "reported_model": "claude-opus-5-5", "reported_models": [ "claude-haiku-5-5", "claude-opus-5-5" ], "isolation": "eval-user" }, "capture": { "mode": "text" }, "harness": { "name": "claude-code", "version": "2.1.287" }, "pricing_snapshot": { "frozen_at": "2026-10-18T07:34:02.114Z", "source": "config/models.json", "currency": "USD", "models": { "claude-opus-5-5": { "input_per_mtok": 4, "output_per_mtok": 20, "registered": true }, "claude-haiku-5-5": { "input_per_mtok": 0.1, "output_per_mtok": 0.5, "registered": true } }, "note": "Prices frozen at run time. Every derived cost in this receipt is computed from THIS snapshot, never from the live registry, so the receipt keeps its meaning when registry prices later change." }, "counts": { "judge_samples_per_generation": 5 } }, "results": { "cases": [ { "id": "loop", "mode": "with_skill", "outcome": "pass", "score": 0.906, "mean": 0.906, "stddev": 0.068411, "samples": [ 1, 1, 0.9, 1, 1 ], "generation_hash": "257c080a37212b6a9c11d580a016d9c2d7fab71f1c9e2f588b4a6bcd58098987", "judge_sample_hashes": [ "014f6b250896fb531ae897abb46e1c3a55dfe7fdc1cdf35472b07da78aa6488e", "22230cd0a3828171a6c666e3e04882a2e2ade0a67a6350ed92313e623d0bc374", "9cc2305fa545f96730666c229a47b33f4c8c378109e58df38434a6a352a25df0", "6ebea2bd6c5ac7664c76c73bfb12d408a7bf6d68af9974f8f58f59fbe66b4ce1", "1243e5b93a409d01389c07d5328ec357b6df79f850592201dbe68e407f470a88" ], "threshold": 0.75, "reason": "Criterion 1 met: catalog searched by the desired reveal effect. Criterion 2 met: add/author workflow (hyperframes-registry, add, hyperframes-core) precedes validation. Criterion 3 met: check is the final gate and the response explicitly avoids a redundant final lint. Criterion 4 met: preview is run ", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "29f195679e4668e033cd35142324e5fd864b4c6fbb8e4b003660e8fe439e6715" }, "judge_usage": { "input_tokens": 35016, "output_tokens": 2393, "cached_tokens": 31962, "wall_ms": 23380 }, "generation": { "n_planned": 3, "n_drawn": 5, "n_measured": 5, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.906, "sd": 0.068411, "judge_sd_mean": 0.108438, "variance_ratio": 0.630877, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "464e895b79d0448f63e093c771380750ca8128b92230463b1e2ede6121a8bfb6", "status": "measured", "samples": [ 1, 0.75, 1, 1, 1 ], "judge_sample_hashes": [ "5e5716d840a97bc3210241673be751773b6b9504ada2cfcaee5c7401e272ddea", "5697a756846261debf593c6830759fe5c1e56339be08f89903bed92d03098ac1", "b73f02c2fda2b66af0b8a3febc31967e8e4e26727f0cf13f91026baa976fb011", "148cf359742eeb519a74a940e8fe889cc73d44312e4c977736e615deace20cb7", "ebdd5bfa82c1e4aa31f5ce048b9d6dc297edae894419d56c7b0a73e4e3ce6ec1" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 10851, "output_tokens": 1105, "cached_tokens": 544, "wall_ms": 12447 }, "judge_usage": { "input_tokens": 33535, "output_tokens": 2253, "cached_tokens": 30782, "wall_ms": 22612 } }, { "draw_index": 1, "generation_hash": "1c13b5da9ade3d11eb0d68f14b588f2c54e1bc9ef8bc48415e7e2b9b14198b9a", "status": "measured", "samples": [ 1, 1, 0.75, 1, 0.75 ], "judge_sample_hashes": [ "c91454433033e45362276037c28be1b5f8d23af4f7be828aa5ba23dc4366f82b", "0a0a6691081aaa7bc6a2f748d4160761c007383ea50b4c90e008d0e0cec0b875", "2a383f3150d91b251da49a58183c73d2bf6944cb4bec1cb13154cd551b991f42", "d283a91ad3e13900b4f5420b763b77dd3dbf1edcef3fb085920035235c1ed751", "2f72ce2bfa0ee0e5fd1bf42ddbae58cfff757d874769a3d31b32e33713d0e5ae" ], "mean": 0.9, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 10851, "output_tokens": 943, "cached_tokens": 10647, "wall_ms": 10635 }, "judge_usage": { "input_tokens": 32570, "output_tokens": 1673, "cached_tokens": 30006, "wall_ms": 20281 } }, { "draw_index": 2, "generation_hash": "5e47cb7799d28fde1d39192e48c1c9f1c870fe817127384fdfb72b4e43013522", "status": "measured", "samples": [ 0.75, 0.75, 1, 0.75, 0.75 ], "judge_sample_hashes": [ "d17af7703987fe5e6d6fecdcf3322e135287eae265b6415e4373b4cebbb5f053", "8eccdbc84893a225679d95d7c7ca030327314d8a07a50b5f5b728612a7d5ab33", "0ad31a3ffa122a6e6011464128e552faa51167613189f79da11d1be26c55f8fa", "041a9984e770b355f74cbbe93ada3427960ba0b977b5929426fb40f4b9ec3798", "37a442856334c37294193a23edc421705a83e7e768238539913506f6e52e3888" ], "mean": 0.8, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 10852, "output_tokens": 1102, "cached_tokens": 10647, "wall_ms": 11886 }, "judge_usage": { "input_tokens": 33322, "output_tokens": 2041, "cached_tokens": 30610, "wall_ms": 20851 } }, { "draw_index": 3, "generation_hash": "b38f9303c565461dfbe34805776fdf21bb1eb8f2fae77f263ef1aaf8de36e7c0", "status": "measured", "samples": [ 1, 1, 0.75, 1, 0.75 ], "judge_sample_hashes": [ "9e379f6d95f7dbc896ede84e088d6bd1e57d5603177918aace78f7cf09996182", "39a9cd1331a750d126313b7cf9abd8d6b9b1c33a2c2c708ea9881f8b528f10e8", "6c04ab5b79ceb39e0603bdb58391628c3860ac9aa4e312e874ede7f4a745d0c7", "16f7642a9c8276c850d73f18e09702967cb1a5f772a7648713ffcf52454e56b3", "2fcd5bbad27978fd78448ce87136258a4ff28b2f5a0e3ef9d7f7e7440050fdab" ], "mean": 0.9, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 10852, "output_tokens": 1048, "cached_tokens": 10647, "wall_ms": 11886 }, "judge_usage": { "input_tokens": 32976, "output_tokens": 3549, "cached_tokens": 30334, "wall_ms": 27374 } }, { "draw_index": 4, "generation_hash": "257c080a37212b6a9c11d580a016d9c2d7fab71f1c9e2f588b4a6bcd58098987", "status": "measured", "samples": [ 1, 1, 0.9, 1, 1 ], "judge_sample_hashes": [ "014f6b250896fb531ae897abb46e1c3a55dfe7fdc1cdf35472b07da78aa6488e", "22230cd0a3828171a6c666e3e04882a2e2ade0a67a6350ed92313e623d0bc374", "9cc2305fa545f96730666c229a47b33f4c8c378109e58df38434a6a352a25df0", "6ebea2bd6c5ac7664c76c73bfb12d408a7bf6d68af9974f8f58f59fbe66b4ce1", "1243e5b93a409d01389c07d5328ec357b6df79f850592201dbe68e407f470a88" ], "mean": 0.98, "stddev": 0.044721, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 10853, "output_tokens": 1560, "cached_tokens": 10647, "wall_ms": 17333 }, "judge_usage": { "input_tokens": 35016, "output_tokens": 2393, "cached_tokens": 31962, "wall_ms": 23380 } } ], "issue408": { "attempt_id": "e6b9880e7eb4ce1f137a5a328b94b5ef93122183a344201bdc1c309eea082c3e", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "heygen-com/hyperframes", "skill": "hyperframes-cli", "commit": "c4440dfb200def5e0c0afee938778f05b5985039", "skill_path": "skills/hyperframes-cli/SKILL.md", "declared_name": "hyperframes-cli", "skill_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/skills/hyperframes-cli/SKILL.md", "skill_blob_sha1": "711734fd394dc9d22d4dd21a9baaf5e3504f9aa8", "skill_sha256": "be481984f8d3c871c353a4ba82bb295a416634e11938de2081b950e9e488a3ff", "readme_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/README.md", "readme_sha256": "3e8f26d5e35c6a8c32c661bf1439443d8eee332eedc9bf1a32bfe3dc417c218e", "suite": "suites/heygen-com--hyperframes--hyperframes-cli.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:18:29.356Z", "configuration_unchanged": true }, "budget_cost_usd": 2.068524, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 5 } }, { "id": "loop", "mode": "baseline", "outcome": "fail", "score": 0.6875, "mean": 0.6875, "stddev": 0.047871, "samples": [ 0.75, 0.75, 0.5, 0.75, 0.75 ], "generation_hash": "7eb929615054fd8c1e7e7fd7d934b80605e9c3a926071ecc57834108709a7c93", "judge_sample_hashes": [ "27134e70c8f48e69a08c32e3beda675af5ae02c5a6141e924b97c3e9b17abc90", "377722e34adcdfcc26ddedd2874620e2991ae8bb71e9ac1edfdcf9187efa4121", "2a1135e8d47785a3a334952620e6594c07caf334dfd44896b58fb33e713d5b80", "27515283356c7889d70f13ca42cdb39c9292941f330d279ec912b48b2b04a36a", "06525565e7e3a875107de0f88a9d210f0fdf67afad5bfb6e1cdea0f72bc827e1" ], "threshold": 0.75, "reason": "Met criteria 1 (catalog search by headline/reveal style), 2 (add before validation), and 4 (preview, render held for approval); missed 3 because it uses `lint` as the final gate instead of `check`.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "29f195679e4668e033cd35142324e5fd864b4c6fbb8e4b003660e8fe439e6715" }, "judge_usage": { "input_tokens": 32410, "output_tokens": 2555, "cached_tokens": 29878, "wall_ms": 23717 }, "generation": { "n_planned": 3, "n_drawn": 4, "n_measured": 4, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.6875, "sd": 0.047871, "judge_sd_mean": 0.139754, "variance_ratio": 0.342538, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "eb724f599f7fe5f2048a2bcb0b42e6d9e5c7c022f11744db9a9570279dd1250c", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.25 ], "judge_sample_hashes": [ "16d1e62ecdbe6eea486c00117708bed766f7904e403a41ecfcab8a617334b759", "14aacb2cb3218d208ff5e01d8169f01b0a3b5bb889e135607321b838407579cb", "7aab218f2a053d1bce2904bef30064ae3ee5875a5f55b084f2f53b28aa64e413", "a07ed0436d10374cd4651d663b2379d217153f93a2905efe4a3ad025a66b8a01", "09803b379f51556c270934a702eba7e441d3f831e63777a083ad4bc5be3366ed" ], "mean": 0.65, "stddev": 0.223607, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2029, "output_tokens": 1194, "cached_tokens": 1467, "wall_ms": 13822 }, "judge_usage": { "input_tokens": 31920, "output_tokens": 1775, "cached_tokens": 29486, "wall_ms": 20456 } }, { "draw_index": 1, "generation_hash": "9d6ac6c2c89d9014b38adb4c1198a8ed05ea36390c4eda53e047a695ce1d4390", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.25 ], "judge_sample_hashes": [ "842d75501dfe3ef86cbd383da92861664a280872fbe907d0cce8499cff442ec0", "6c416ab13e68f10d5822951c182c4214b93be59af8d1a14e05053b4fef02e110", "514eca512a629bc62ae141e2bf839fe608c59b6bd6c7d2da0f14885c6b4dc02b", "728499c491415c8de635a3c5d1b9ad6e3555d13bc148279146d289f6df45091a", "06baedb478d33019b4e11025820f3bad34cdba7718d2d5efb47833df0bb51a84" ], "mean": 0.65, "stddev": 0.223607, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2029, "output_tokens": 1239, "cached_tokens": 1824, "wall_ms": 16491 }, "judge_usage": { "input_tokens": 32928, "output_tokens": 1707, "cached_tokens": 30294, "wall_ms": 19579 } }, { "draw_index": 2, "generation_hash": "2becab901efdc5e79eb4fe08b9661341c995b499177f314921c083cc80d1d6b9", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "5247edaed638c8ab499e8b2f3bd0e09bd8d65e1ab5518891c8b3ceeb4a79a9bc", "e80a0ca7f93c03ee1b260352a00a9b9c54cbbacb9fdcd659294c83495681bf0d", "86cfb0f2960f49a1f66c4eabf84cd4f38e58c158a34bb57ec6d79026385f8ae2", "69438b3e9e5aed75c07ae7e4c846383e7d33cabc77ae1e044c41fbe1a937591b", "10df66266f6051dadb6147dcf6205924156574ee63c01fc649ae4d3c1da6e755" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2028, "output_tokens": 1060, "cached_tokens": 1824, "wall_ms": 13797 }, "judge_usage": { "input_tokens": 31858, "output_tokens": 2313, "cached_tokens": 29438, "wall_ms": 23955 } }, { "draw_index": 3, "generation_hash": "7eb929615054fd8c1e7e7fd7d934b80605e9c3a926071ecc57834108709a7c93", "status": "measured", "samples": [ 0.75, 0.75, 0.5, 0.75, 0.75 ], "judge_sample_hashes": [ "27134e70c8f48e69a08c32e3beda675af5ae02c5a6141e924b97c3e9b17abc90", "377722e34adcdfcc26ddedd2874620e2991ae8bb71e9ac1edfdcf9187efa4121", "2a1135e8d47785a3a334952620e6594c07caf334dfd44896b58fb33e713d5b80", "27515283356c7889d70f13ca42cdb39c9292941f330d279ec912b48b2b04a36a", "06525565e7e3a875107de0f88a9d210f0fdf67afad5bfb6e1cdea0f72bc827e1" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2029, "output_tokens": 1362, "cached_tokens": 1824, "wall_ms": 16092 }, "judge_usage": { "input_tokens": 32410, "output_tokens": 2555, "cached_tokens": 29878, "wall_ms": 23717 } } ], "issue408": { "attempt_id": "e6b9880e7eb4ce1f137a5a328b94b5ef93122183a344201bdc1c309eea082c3e", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "heygen-com/hyperframes", "skill": "hyperframes-cli", "commit": "c4440dfb200def5e0c0afee938778f05b5985039", "skill_path": "skills/hyperframes-cli/SKILL.md", "declared_name": "hyperframes-cli", "skill_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/skills/hyperframes-cli/SKILL.md", "skill_blob_sha1": "711734fd394dc9d22d4dd21a9baaf5e3504f9aa8", "skill_sha256": "be481984f8d3c871c353a4ba82bb295a416634e11938de2081b950e9e488a3ff", "readme_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/README.md", "readme_sha256": "3e8f26d5e35c6a8c32c661bf1439443d8eee332eedc9bf1a32bfe3dc417c218e", "suite": "suites/heygen-com--hyperframes--hyperframes-cli.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:18:29.356Z", "configuration_unchanged": true }, "budget_cost_usd": 2.068524, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 4 } }, { "id": "timeline", "mode": "with_skill", "outcome": "pass", "score": 0.983333, "mean": 0.983333, "stddev": 0.028868, "samples": [ 1, 1, 0.75, 1, 1 ], "generation_hash": "8aac7733014537156ffe42278d8bfbb9e89a63b80a14820262d2a1a86082d208", "judge_sample_hashes": [ "3fe2c3fa48e8ed993badf5f239b0a1d3ad18a3d85dd37be5ba1cb817ae255385", "f032066069900c0894df39f6cb26c14d55f4232e97c5742634707efa93cf8143", "ba554aff881c445ecaa72cefe1b2e07311f615a94e27ff867cbf39aac667fc40", "5a241e32a9eaa7463a83735c20e91807987460056871821873c045ca97b5a727", "a622a1b8c6eb882afe6afb65fce2b3854d23f7e7f9ae9393145fdc4acd7c369a" ], "threshold": 0.75, "reason": "Uses timeline --json, distinguishes absolute (absStart/absEnd with mount offset) from local timing, uses the file field to identify the owner, and does not claim to have inspected the whole timeline.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "96e3bdb7297d45877585bad2bdfbaae8ed2748453396e4d1613f1ca2b6e52df4" }, "judge_usage": { "input_tokens": 32344, "output_tokens": 1372, "cached_tokens": 29826, "wall_ms": 18424 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.983333, "sd": 0.028868, "judge_sd_mean": 0.037268, "variance_ratio": 0.774606, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "7ee563e76ba5ed6e9d1dbc30affb8155b30bc83a24b945b6835eae047bed250b", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "60b7a2851f9e98dc4d7dd40e5b4c14a473d45d76d55bb32a9419ff1b581c99e0", "0d222fed756b470d1f300a1dba8049811638d121a05a70fdea96326f6f29e292", "74e65cad9783b8d429414432c1d85b3ed567392489821a7836408d4f6d0df644", "20dbc4f90ce4ebe77e1f5b46ea827181468cddfe8418442f113bc8cc1b93c2ab", "ff23e964a3f9e3733d4d70df39ea0080397e7eae8f9039a3c34ee010a9a30ae5" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 10861, "output_tokens": 1222, "cached_tokens": 10290, "wall_ms": 15776 }, "judge_usage": { "input_tokens": 32459, "output_tokens": 1638, "cached_tokens": 29918, "wall_ms": 20344 } }, { "draw_index": 1, "generation_hash": "69f7e2d166d4f97fef2e8cfca59f77aa66f9f183f38dcf5da93f7aea60e457fd", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "a694768b4b694cfd526bd47d2065d4f24df3e7f89e07e46549ad31f16b2e2dab", "e0ce28d3598ffbf8310d83d072007a7a9e36c7e1b130aa72809be222e3438498", "6366839f039cb899c2d8c87c81ba34696cea330ae8c9d929a7dc63f3cd25be36", "40859bb98948223b3186a0fa2ce38c20bcb77188c1116276b7c0498fef4d61fd", "fd99867f7fce99e6d66c06bafb1ad0426658e60e48066d5394299b5cfcfcba2a" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 10861, "output_tokens": 1132, "cached_tokens": 10656, "wall_ms": 13532 }, "judge_usage": { "input_tokens": 32353, "output_tokens": 1829, "cached_tokens": 29834, "wall_ms": 19976 } }, { "draw_index": 2, "generation_hash": "8aac7733014537156ffe42278d8bfbb9e89a63b80a14820262d2a1a86082d208", "status": "measured", "samples": [ 1, 1, 0.75, 1, 1 ], "judge_sample_hashes": [ "3fe2c3fa48e8ed993badf5f239b0a1d3ad18a3d85dd37be5ba1cb817ae255385", "f032066069900c0894df39f6cb26c14d55f4232e97c5742634707efa93cf8143", "ba554aff881c445ecaa72cefe1b2e07311f615a94e27ff867cbf39aac667fc40", "5a241e32a9eaa7463a83735c20e91807987460056871821873c045ca97b5a727", "a622a1b8c6eb882afe6afb65fce2b3854d23f7e7f9ae9393145fdc4acd7c369a" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 10861, "output_tokens": 1109, "cached_tokens": 10656, "wall_ms": 15966 }, "judge_usage": { "input_tokens": 32344, "output_tokens": 1372, "cached_tokens": 29826, "wall_ms": 18424 } } ], "issue408": { "attempt_id": "e6b9880e7eb4ce1f137a5a328b94b5ef93122183a344201bdc1c309eea082c3e", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "heygen-com/hyperframes", "skill": "hyperframes-cli", "commit": "c4440dfb200def5e0c0afee938778f05b5985039", "skill_path": "skills/hyperframes-cli/SKILL.md", "declared_name": "hyperframes-cli", "skill_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/skills/hyperframes-cli/SKILL.md", "skill_blob_sha1": "711734fd394dc9d22d4dd21a9baaf5e3504f9aa8", "skill_sha256": "be481984f8d3c871c353a4ba82bb295a416634e11938de2081b950e9e488a3ff", "readme_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/README.md", "readme_sha256": "3e8f26d5e35c6a8c32c661bf1439443d8eee332eedc9bf1a32bfe3dc417c218e", "suite": "suites/heygen-com--hyperframes--hyperframes-cli.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:18:29.356Z", "configuration_unchanged": true }, "budget_cost_usd": 2.068524, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } }, { "id": "timeline", "mode": "baseline", "outcome": "fail", "score": 0.5875, "mean": 0.5875, "stddev": 0.094648, "samples": [ 0.75, 0.75, 0.75, 0.25, 0.75 ], "generation_hash": "7bf24247c9a8e94b4a53cade9eace8c82a7af36aaaf878536b4d0a4e50e57648", "judge_sample_hashes": [ "d3b016f71e72ac2aedf6409e37c66a59d3f7344da4992d2ebb9fd15c82908c91", "b3d255b82f9a88f1d2756649719df8ba62188b62405cc4abb3754837941cd403", "61a080caf4f84167de6e29e905177e63f8dadba3d5ef3689da033d4fdc2a441e", "75de5dd5ac912529d327a5b9a7ea53d22f4ffe67661f82968bf782e01db26ca5", "cb08d6ede41b9df460d9d581e7872531f83b338c03f501449d083865079db0e1" ], "threshold": 0.75, "reason": "Meets the absolute-versus-local timing distinction, owning-file field, and no-whole-timeline-claim criteria; misses the required 'timeline --json' command, offering only guessed generic command names.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "96e3bdb7297d45877585bad2bdfbaae8ed2748453396e4d1613f1ca2b6e52df4" }, "judge_usage": { "input_tokens": 32031, "output_tokens": 2053, "cached_tokens": 29574, "wall_ms": 21105 }, "generation": { "n_planned": 3, "n_drawn": 4, "n_measured": 4, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.5875, "sd": 0.094648, "judge_sd_mean": 0.236171, "variance_ratio": 0.40076, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "97b1357cdc62203c2200b607db8e2833c3e35d2a088087f802eea572cb672c48", "status": "measured", "samples": [ 0.25, 0.25, 0.25, 0.75, 0.75 ], "judge_sample_hashes": [ "c70171dfa59bf81e0af03aa952a1687c50b37d578de6809a8d0cfd8507dd8cda", "7b18c46cd7dd443c1f8d206e145f8c0e5cbb0312df9dbbdc9ffd9e0469891f53", "a76601e6e010f83ee8ce42e833defa2a48046e16788e1c31d945f33c3730fc66", "831350ee86dba3135b2b8ec103f33f3cd80c36f596dca1300bf92f1f349feb25", "62504ae3c14642b0183357848f4bb36aafa322385e6f2d7ed22f166df50127e3" ], "mean": 0.45, "stddev": 0.273861, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2038, "output_tokens": 845, "cached_tokens": 1467, "wall_ms": 11108 }, "judge_usage": { "input_tokens": 31674, "output_tokens": 1342, "cached_tokens": 29290, "wall_ms": 18851 } }, { "draw_index": 1, "generation_hash": "a99459da93b9abfb071746258c607acd1256c40b1a85d794dbd8dfaf74a266f4", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.25 ], "judge_sample_hashes": [ "acc2440945701b7f0c6dbd0154c1635de3e3b19c22c891fb500d36c568bf533f", "9b141ed396b446f1b0fc2f1faa8886bcee9064bc5ec7c6375ed90e98a441f307", "3abe3b54568ab10776177935d2faeb71161b89d97e6f6fe567334f7207c08922", "43dff6ed681833e7119d14ed0b07d447096d651b3b8cbff3befea18087d9a542", "9c47cd754c2708499069777f2d5f15522d4d56b98d9cf850d5a98d99251c1371" ], "mean": 0.65, "stddev": 0.223607, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2038, "output_tokens": 998, "cached_tokens": 1833, "wall_ms": 15255 }, "judge_usage": { "input_tokens": 31873, "output_tokens": 2021, "cached_tokens": 29450, "wall_ms": 20595 } }, { "draw_index": 2, "generation_hash": "f76143b6d6f07bb6eb0d5dba124df8c0c02ea66523ee8f47bd7d38c32b7b7af2", "status": "measured", "samples": [ 0.5, 0.75, 0.25, 0.75, 0.75 ], "judge_sample_hashes": [ "553255702133c5fb4ee8764db0524b57e6498471c230d9551593a25a6875463b", "ba2466a02c2dd5bf99206dcb76454ee9dd23e1728a37f9387fada01b529456cf", "d98e9a6c1eb1a58bae03ad9606a88777b912ba1bfdf1c3658247761bce843ed1", "e38b691e3acdfd607a5d5db0677219824c815b2679f82022417917031364c64c", "a8b101be6a8cdc8cd013eb48428011ad978cdcf462235ff67e1e2fae8029cb92" ], "mean": 0.6, "stddev": 0.223607, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2039, "output_tokens": 850, "cached_tokens": 1833, "wall_ms": 11707 }, "judge_usage": { "input_tokens": 31778, "output_tokens": 1589, "cached_tokens": 29374, "wall_ms": 19283 } }, { "draw_index": 3, "generation_hash": "7bf24247c9a8e94b4a53cade9eace8c82a7af36aaaf878536b4d0a4e50e57648", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.25, 0.75 ], "judge_sample_hashes": [ "d3b016f71e72ac2aedf6409e37c66a59d3f7344da4992d2ebb9fd15c82908c91", "b3d255b82f9a88f1d2756649719df8ba62188b62405cc4abb3754837941cd403", "61a080caf4f84167de6e29e905177e63f8dadba3d5ef3689da033d4fdc2a441e", "75de5dd5ac912529d327a5b9a7ea53d22f4ffe67661f82968bf782e01db26ca5", "cb08d6ede41b9df460d9d581e7872531f83b338c03f501449d083865079db0e1" ], "mean": 0.65, "stddev": 0.223607, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2038, "output_tokens": 935, "cached_tokens": 1833, "wall_ms": 12508 }, "judge_usage": { "input_tokens": 32031, "output_tokens": 2053, "cached_tokens": 29574, "wall_ms": 21105 } } ], "issue408": { "attempt_id": "e6b9880e7eb4ce1f137a5a328b94b5ef93122183a344201bdc1c309eea082c3e", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "heygen-com/hyperframes", "skill": "hyperframes-cli", "commit": "c4440dfb200def5e0c0afee938778f05b5985039", "skill_path": "skills/hyperframes-cli/SKILL.md", "declared_name": "hyperframes-cli", "skill_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/skills/hyperframes-cli/SKILL.md", "skill_blob_sha1": "711734fd394dc9d22d4dd21a9baaf5e3504f9aa8", "skill_sha256": "be481984f8d3c871c353a4ba82bb295a416634e11938de2081b950e9e488a3ff", "readme_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/README.md", "readme_sha256": "3e8f26d5e35c6a8c32c661bf1439443d8eee332eedc9bf1a32bfe3dc417c218e", "suite": "suites/heygen-com--hyperframes--hyperframes-cli.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:18:29.356Z", "configuration_unchanged": true }, "budget_cost_usd": 2.068524, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 4 } }, { "id": "render", "mode": "with_skill", "outcome": "pass", "score": 0.975, "mean": 0.975, "stddev": 0.05, "samples": [ 1, 1, 1, 1, 1 ], "generation_hash": "64b0ba1ac294621106065d7fc6ab644dd6604a97443563af15cb1107f783881a", "judge_sample_hashes": [ "b3e38a6b91796f7ac289b8c562451e7da54503611231246fe1823db84e2a13d5", "3b9367c12a72c3d120295e3aef6d0dc6406b34386453442f1eb583939c4615c5", "4c92744f5b36f6c012efe1e5619a71bbc5808a2943b0ffadfb445eeccab06e4b", "bfc1bbdad027114e6b72dd29612f57d9e5479837b0eed614c27e15ce0e5d77d6", "9c6d32168edf2588fd7cb717378f130b50e140c970da2ecbd105664d83fffcf5" ], "threshold": 0.75, "reason": "Meets all four criteria: identifies screenshot/software-GPU as a slow path, not a failure; checks duration and fps against the composition; inspects decoded frames including zoomed detail regions; and states that successful encoding does not certify pixels.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "546a92f047d04fe8e9343b93fbf080ef88f201b443e711bedf549dfce2a3ee8e" }, "judge_usage": { "input_tokens": 33481, "output_tokens": 1730, "cached_tokens": 30734, "wall_ms": 18785 }, "generation": { "n_planned": 3, "n_drawn": 4, "n_measured": 4, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.975, "sd": 0.05, "judge_sd_mean": 0.034233, "variance_ratio": 1.460579, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "6d47e900cb9159cf3bc30b2d0b61a86f3c35d9c0ae5dd9f721dc4c36ca163c2e", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "3ed7b02bb200e7e9a15bd55d14aa8bcad3ff3c16fb7d2914a43a001c0e5d071d", "b48633e5bb14bc0bf5ca84cdcb8f84404e135e20c91e9d8ae12cbf3fde291bf3", "3d0d25d085056b8c531c04ff4dae763af777f39570cb452ad43654aa1c65658f", "4ebf72ce731064acbe3a17c832a50eb225086d7cc1b2017c51b4a4da068643e1", "c109c28101cee00a3d86a827584c6453aaf9bb052d53f845655cafae36c99821" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 10843, "output_tokens": 1168, "cached_tokens": 10290, "wall_ms": 14513 }, "judge_usage": { "input_tokens": 33186, "output_tokens": 1182, "cached_tokens": 30502, "wall_ms": 17346 } }, { "draw_index": 1, "generation_hash": "af3de5e900d3727d088e5739b35c166159b4f129e446f4b40aa49e6ee22d1891", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "9a9106eda35ccd74fad3f3d9cce172dd014417170bb6949ae24c4c3808715fe9", "e962aeb00940c05e3de0030eb845adc092e2c53915dffb2cce67c03e2693c341", "b24ec8955c1537e85ccf8404f9be962e67b5ceb643abe4b01736e17c2eed9808", "cf569ed8991c5aba3ec6eaf7632638140557b46eacbd052b9314496f15aa74a0", "cde7ee72904149969438e3d4b7b2690693673e469adb172e0c8565ccc40e4023" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 10843, "output_tokens": 1252, "cached_tokens": 10638, "wall_ms": 14064 }, "judge_usage": { "input_tokens": 33389, "output_tokens": 791, "cached_tokens": 30662, "wall_ms": 14654 } }, { "draw_index": 2, "generation_hash": "d1128de015043fcb0bacd33e6e49500fa78c16a0880776d1f1a59b6ddc5bae5f", "status": "measured", "samples": [ 1, 0.75, 0.75, 1, 1 ], "judge_sample_hashes": [ "b0c3e91ae56ec41531fb0f2af2ed155c08d57050600fb2de1e5cf57e6a4e841f", "559723c8db075655f0510434510a4ba8a52c8c02590359677f2cf41178f55bcc", "0913c3edd6e3dede368425af9a647d53a08beadc37c44340135a9970031b1ac7", "840974d2ad61c6ad2609e60e23b4b2684093c4a8c9706c6a5b2314a14793b55e", "d0baff4ee425f309aaa4d69e7f3bc9874048b5ae9f91370695143bf62701c6c3" ], "mean": 0.9, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 10843, "output_tokens": 1347, "cached_tokens": 10638, "wall_ms": 16346 }, "judge_usage": { "input_tokens": 33729, "output_tokens": 997, "cached_tokens": 30938, "wall_ms": 17277 } }, { "draw_index": 3, "generation_hash": "64b0ba1ac294621106065d7fc6ab644dd6604a97443563af15cb1107f783881a", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "b3e38a6b91796f7ac289b8c562451e7da54503611231246fe1823db84e2a13d5", "3b9367c12a72c3d120295e3aef6d0dc6406b34386453442f1eb583939c4615c5", "4c92744f5b36f6c012efe1e5619a71bbc5808a2943b0ffadfb445eeccab06e4b", "bfc1bbdad027114e6b72dd29612f57d9e5479837b0eed614c27e15ce0e5d77d6", "9c6d32168edf2588fd7cb717378f130b50e140c970da2ecbd105664d83fffcf5" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 10843, "output_tokens": 1267, "cached_tokens": 10638, "wall_ms": 14838 }, "judge_usage": { "input_tokens": 33481, "output_tokens": 1730, "cached_tokens": 30734, "wall_ms": 18785 } } ], "issue408": { "attempt_id": "e6b9880e7eb4ce1f137a5a328b94b5ef93122183a344201bdc1c309eea082c3e", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "heygen-com/hyperframes", "skill": "hyperframes-cli", "commit": "c4440dfb200def5e0c0afee938778f05b5985039", "skill_path": "skills/hyperframes-cli/SKILL.md", "declared_name": "hyperframes-cli", "skill_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/skills/hyperframes-cli/SKILL.md", "skill_blob_sha1": "711734fd394dc9d22d4dd21a9baaf5e3504f9aa8", "skill_sha256": "be481984f8d3c871c353a4ba82bb295a416634e11938de2081b950e9e488a3ff", "readme_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/README.md", "readme_sha256": "3e8f26d5e35c6a8c32c661bf1439443d8eee332eedc9bf1a32bfe3dc417c218e", "suite": "suites/heygen-com--hyperframes--hyperframes-cli.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:18:29.356Z", "configuration_unchanged": true }, "budget_cost_usd": 2.068524, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 4 } }, { "id": "render", "mode": "baseline", "outcome": "fail", "score": 0.658333, "mean": 0.658333, "stddev": 0.073598, "samples": [ 0.75, 0.75, 0.5, 0.75, 0.75 ], "generation_hash": "61a2dff6ef788106dbf5cc26c799de450fb916d60cb29b7825b8e06665c34468", "judge_sample_hashes": [ "faaefd33877a2eab87bdf481697a22138b0a1f07e2fa519cea3c4c82050f3591", "30ba3f032b9391b6a3b42ebfb61023c9f4dc3aac396dfb3c31a130b3edaae07d", "dba7ea939a7deaa6eb7df6442ccc0ad8dd488ed3c0bc7794edcc4f73ec11d413", "bdae169b2a57a5112bb4880ec8477983c39ba18e073bc82f4a62200f4e5c5b25", "a7694d69642ca927220db28dac76f14e2634cfaccf3624ed5698f486564ce55f" ], "threshold": 0.75, "reason": "Meets criteria 2 (duration/frame rate checks), 3 (decoded frames plus detailed WebGL/gradient/text regions) and 4 (explicitly says a successful render is not verified pixels); criterion 1 is only partly met since it never names the slow path.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "546a92f047d04fe8e9343b93fbf080ef88f201b443e711bedf549dfce2a3ee8e" }, "judge_usage": { "input_tokens": 32159, "output_tokens": 3134, "cached_tokens": 29678, "wall_ms": 24786 }, "generation": { "n_planned": 3, "n_drawn": 6, "n_measured": 6, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.658333, "sd": 0.073598, "judge_sd_mean": 0.147235, "variance_ratio": 0.499868, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "d1a0186d0a36132a58bf91be0a3690977a91193890928fa8f09ad298586720b7", "status": "measured", "samples": [ 0.75, 0.5, 0.5, 0.5, 0.5 ], "judge_sample_hashes": [ "c2751b1752f5c4442aca042fb59d06089d7a12c3b1d9de1ebc797334db85215c", "813f1927dd99c06a8263caac9fadb8e7d388d816c1bf82835de5de6d3f0ffd00", "59aec7571930742b89f3453f2b1fb9dbb57d8b566a9ae4dd266af52b7186a1aa", "5f134938925e219808e65c46570c1bc15fba9f6cc1d4b43af303fd5ac1d5e623", "f1fc0a72da4a3fcfd62ed33284c59af4f9b29665a3afec48d54f4e046625a9b7" ], "mean": 0.55, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2020, "output_tokens": 969, "cached_tokens": 1467, "wall_ms": 16094 }, "judge_usage": { "input_tokens": 32802, "output_tokens": 1593, "cached_tokens": 30194, "wall_ms": 19810 } }, { "draw_index": 1, "generation_hash": "715977e04c314241a9be59ae56cf12b3c6c0b542c74e55e8f5b92752b5618eab", "status": "measured", "samples": [ 1, 0.75, 0.5, 0.5, 0.75 ], "judge_sample_hashes": [ "6495bf183714d96712318407d8da172035e95320490df6935b9f4f77f3f9adfc", "e0a56f483e72fcc29ca9c5eb96bcde4ec732e2e98f6bc4ae4f5142f153a2d758", "109e3b9a29abad8015bafafe63c45f483f3b437e39269c877de6c9a440a8ab92", "b7bdef724b9ced0388b944fc85387ef24986bc880745b7d3580c894985257e34", "e20aab8f6e66fc5dd418c0bf6c0748ad66caf0b33a7b8aae66f7b330108fa132" ], "mean": 0.7, "stddev": 0.209165, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2020, "output_tokens": 1763, "cached_tokens": 1815, "wall_ms": 20875 }, "judge_usage": { "input_tokens": 34442, "output_tokens": 2892, "cached_tokens": 31506, "wall_ms": 25309 } }, { "draw_index": 2, "generation_hash": "e2c4ba1d5196b46f49e6f8c1396977658fa07e73215ea899b9fdde024df098d4", "status": "measured", "samples": [ 0.5, 0.5, 0.75, 0.5, 0.75 ], "judge_sample_hashes": [ "f46e9cfab5c45fa7b35f574512a80c0c52f1e54a6d9516110ec441b77ea4002b", "541e3c79e623dad3178671370fa9e39ab69559caf9ee8149eb1f1bef7a68bf14", "0c0a32fdbd7b3d87ee7549b1b132319f88015f46a2df4f1115dac54f745f617c", "6055f72b6fe9911905f1700181c27c7dcf14c547d42597609dd736dda9ec5583", "65499e98e3755d531be012dbb00ad9df82099a603d0727d123dea44085637f7c" ], "mean": 0.6, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2020, "output_tokens": 1021, "cached_tokens": 1815, "wall_ms": 13137 }, "judge_usage": { "input_tokens": 31948, "output_tokens": 1906, "cached_tokens": 29510, "wall_ms": 21110 } }, { "draw_index": 3, "generation_hash": "a33597602b3c495e884fbd41d7e504cfc8e5ed07347d06973da49f6f4fbd73d8", "status": "measured", "samples": [ 0.5, 0.75, 0.75, 0.5, 0.75 ], "judge_sample_hashes": [ "e063e5e83fa0fd9843eb5401248decccdb113562af6a776cabf8bf1af94101a4", "4bf0f7ec29d670cf6f5b4b3510184ac8b322962d7af8a7fd627dc784f41efe9e", "51c0ad6f92f888efd5f40ac0a19f98058db0dcfb9c8386b56b34b17999dc87c8", "0f35f50af7c06aa4fcbb59418e5cc1ddacf3580b2be43def4331328a60a2de7c", "a77742a2dc28c3e639f5ebc83485ca575026f9ec5d8214c196cf792a960b05a1" ], "mean": 0.65, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2019, "output_tokens": 1141, "cached_tokens": 1815, "wall_ms": 14945 }, "judge_usage": { "input_tokens": 32384, "output_tokens": 2660, "cached_tokens": 29858, "wall_ms": 28493 } }, { "draw_index": 4, "generation_hash": "e1f5598184edcd6630e67f7f69acbb8277082e1ae150d0214c87576597c7c937", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 1, 0.5 ], "judge_sample_hashes": [ "d21d644c0a2b59cbfb5341b58b416ad945bf5dc38b92a738ae0912b09e078b5e", "010f46e55303713e27d6e416c7be27484ecc9646533a10f2af3deed8aca4ae78", "85491184a4c84bb315544e697b045dabf0fd1ae3bda0183e44d9c2e476a8f0c0", "f3ce804b26141138902658906319328decb19be80c4caa75248d2ab283866405", "3831a8255ed0248bfd033628cb703ff5d26c07d37f400c49ccdf7ce53635515f" ], "mean": 0.75, "stddev": 0.176777, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2020, "output_tokens": 1449, "cached_tokens": 1815, "wall_ms": 17713 }, "judge_usage": { "input_tokens": 32568, "output_tokens": 3526, "cached_tokens": 30006, "wall_ms": 26776 } }, { "draw_index": 5, "generation_hash": "61a2dff6ef788106dbf5cc26c799de450fb916d60cb29b7825b8e06665c34468", "status": "measured", "samples": [ 0.75, 0.75, 0.5, 0.75, 0.75 ], "judge_sample_hashes": [ "faaefd33877a2eab87bdf481697a22138b0a1f07e2fa519cea3c4c82050f3591", "30ba3f032b9391b6a3b42ebfb61023c9f4dc3aac396dfb3c31a130b3edaae07d", "dba7ea939a7deaa6eb7df6442ccc0ad8dd488ed3c0bc7794edcc4f73ec11d413", "bdae169b2a57a5112bb4880ec8477983c39ba18e073bc82f4a62200f4e5c5b25", "a7694d69642ca927220db28dac76f14e2634cfaccf3624ed5698f486564ce55f" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2019, "output_tokens": 1081, "cached_tokens": 1815, "wall_ms": 14340 }, "judge_usage": { "input_tokens": 32159, "output_tokens": 3134, "cached_tokens": 29678, "wall_ms": 24786 } } ], "issue408": { "attempt_id": "e6b9880e7eb4ce1f137a5a328b94b5ef93122183a344201bdc1c309eea082c3e", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "heygen-com/hyperframes", "skill": "hyperframes-cli", "commit": "c4440dfb200def5e0c0afee938778f05b5985039", "skill_path": "skills/hyperframes-cli/SKILL.md", "declared_name": "hyperframes-cli", "skill_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/skills/hyperframes-cli/SKILL.md", "skill_blob_sha1": "711734fd394dc9d22d4dd21a9baaf5e3504f9aa8", "skill_sha256": "be481984f8d3c871c353a4ba82bb295a416634e11938de2081b950e9e488a3ff", "readme_url": "https://raw.githubusercontent.com/heygen-com/hyperframes/c4440dfb200def5e0c0afee938778f05b5985039/README.md", "readme_sha256": "3e8f26d5e35c6a8c32c661bf1439443d8eee332eedc9bf1a32bfe3dc417c218e", "suite": "suites/heygen-com--hyperframes--hyperframes-cli.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:18:29.356Z", "configuration_unchanged": true }, "budget_cost_usd": 2.068524, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "loop", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "loop", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "timeline", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "timeline", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "render", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "render", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 6 } } ], "aggregates": { "with_skill": { "case_count": 3, "pass_count": 3, "borderline_count": 0, "mean_score": 0.954778, "stddev": 0.042448 }, "baseline": { "case_count": 3, "pass_count": 0, "borderline_count": 0, "mean_score": 0.644444, "stddev": 0.051426 }, "band_rule": "per arm: the sample standard deviation (n-1) of the per-case means across the included cases; the comparison band is the quadrature sum of the two arms; null where an arm has fewer than two included cases" } }, "comparison": { "with_skill_score": 0.954778, "baseline_score": 0.644444, "delta": 0.310334, "delta_uncertainty": 0.066682 }, "verification_level": "TESTED", "receipt_hash": "3939420c0ea1f9e5b2bffbff5b7600ac53126a761fa20ca460d68f961727d13d", "economics": { "basis": "metered-equivalent", "surface": "claude-cli", "with_skill": { "call_count": 12, "mean_input_tokens": 10851.17, "mean_output_tokens": 1187.92, "mean_cost_usd_per_call": 0.067163, "median_wall_ms": 14288.5, "wall_ms_p25": 12166.5, "wall_ms_p75": 15871, "wall_ms_iqr": 3704.5 }, "baseline": { "call_count": 14, "mean_input_tokens": 2027.57, "mean_output_tokens": 1136.21, "mean_cost_usd_per_call": 0.030835, "median_wall_ms": 14642.5, "wall_ms_p25": 13137, "wall_ms_p75": 16094, "wall_ms_iqr": 2957 }, "skill_incremental_cost_usd_per_call": 0.036328, "skill_incremental_cost_usd_per_1k_calls": 36.328, "output_tokens_delta": 51.71, "median_wall_ms_delta": -354, "judge_excluded": true, "judge_overhead": { "note": "Measurement overhead imposed by Driftproof, NOT a cost of running the skill. Excluded from every field above. Counted over every draw sent to the judge, each once: the usage its answered judge samples reported, at the frozen judge price. Retried attempts and samples that reported no usage are not in it.", "total_cost_usd": 0.111371, "case_rows_measured": 6, "judged_draws_counted": 26, "judged_draws_missing_usage": 0 }, "notes": { "absolute_cost": "Absolute per-call cost on a CLI surface includes a fixed harness preamble (observed ~25k input tokens on claude-cli, ~11k on codex) that we do not control. It is identical in the with-skill and baseline arms, so it cancels in the Δ figures — read the incremental columns, not the absolute ones.", "cache_pricing": "Cached input tokens are costed at the list input rate (an over-estimate where caching is heavy); cached_tokens is recorded per call so a reader can recompute.", "latency": "observed on subscription CLI surface, indicative" } } }