{ "schema_version": "0.13", "generation_sampled": true, "skill": { "name": "prototype", "version": "0.0.0", "content_hash": "6bfd51e2eab7f88f0f30d9e86329becf028f7bd067ea8573fb912ff39bc1cbbf", "tokens": 731 }, "suite": { "format": "agentskills.io/evals", "suite_hash": "37ce2341bdc63a127f8e5bc8d7546b8ec7f3c15a7d392d04df31e57b10b6b209", "case_count": 3, "canary": "1d061a8b-0fc4-afde-0a87-af3453dc3ac3" }, "run": { "model_id": "claude-sonnet-5-5", "model_release_date": null, "provider": "anthropic", "surface": "claude-cli", "runner_version": "0.22.2", "date_utc": "2026-10-18T06:49:25.297Z", "generated_at": "2026-10-18T06:31:35.318Z", "registry": "registered", "transcripts": "retained-local", "judge": { "samples": 5, "temperature": null, "sampling": "surface-controlled", "surface": "claude-cli", "model_id": "claude-haiku-5-5", "prompt_template_hash": "82586d1e44f84df2b8ce5f18ac777866465cb5ad9686503c93d9776c93c344ed", "harness": { "name": "claude-code", "version": "2.1.287" }, "isolation": "eval-user", "endpoint": { "custom": false, "sha256": null, "resolved_by": "cli", "routing": { "ANTHROPIC_BEDROCK_BASE_URL": { "set": false, "sha256": null }, "ANTHROPIC_VERTEX_BASE_URL": { "set": false, "sha256": null }, "CLAUDE_CODE_USE_BEDROCK": { "set": false, "sha256": null }, "CLAUDE_CODE_USE_VERTEX": { "set": false, "sha256": null }, "HTTPS_PROXY": { "set": false, "sha256": null } } }, "code_revision": "judge-code-bb552899a1df887e", "reported_models": [ "claude-haiku-5-5" ] }, "answered_by": { "kind": "model", "attested": true, "reported_model": "claude-sonnet-5-5", "reported_models": [ "claude-haiku-5-5", "claude-sonnet-5-5" ], "isolation": "eval-user" }, "capture": { "mode": "text" }, "harness": { "name": "claude-code", "version": "2.1.287" }, "pricing_snapshot": { "frozen_at": "2026-10-18T06:49:25.297Z", "source": "config/models.json", "currency": "USD", "models": { "claude-sonnet-5-5": { "input_per_mtok": 2, "output_per_mtok": 10, "registered": true }, "claude-haiku-5-5": { "input_per_mtok": 0.1, "output_per_mtok": 0.5, "registered": true } }, "note": "Prices frozen at run time. Every derived cost in this receipt is computed from THIS snapshot, never from the live registry, so the receipt keeps its meaning when registry prices later change." }, "counts": { "judge_samples_per_generation": 5 } }, "results": { "cases": [ { "id": "logic", "mode": "with_skill", "outcome": "pass", "score": 0.983333, "mean": 0.983333, "stddev": 0.028868, "samples": [ 1, 1, 1, 1, 1 ], "generation_hash": "b05a3bbae570bae396ca2fbba902180c61d4140f105945c3b76b45e0cf7ebf3b", "judge_sample_hashes": [ "a19ccfe74795e76e1b3a29c7a3401b905b506694c0b893f084f9751bd3b15e17", "9d8b20b4fab4d6c3766813474f5baa844b57883ccbd42bdc928a85b2f173cbc3", "9643cf914ea009bdc5fa049fa3f1b3630d0f81c6f93f98d6ea90230f37e53bbb", "a4dd28e1574018d2ddad9174df6e4a342f1d7a4e4e242bfb569ad82f80b6e70e", "02aef6b0d33518ce966652d53a9c89708e40a750cdb7978221f870f05d69ed4d" ], "threshold": 0.75, "reason": "Meets all four criteria: a state-machine demo, state and log shown after every action via render(), a single double-clickable HTML file, and in-memory-only throwaway design with no persistence.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "a8afcecf3740c46b62f144480eb24012410586e9791d58d138f3d27b85ebcad7" }, "judge_usage": { "input_tokens": 39504, "output_tokens": 1976, "cached_tokens": 35554, "wall_ms": 21004 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.983333, "sd": 0.028868, "judge_sd_mean": 0.037268, "variance_ratio": 0.774606, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "e58581cfa1e8c1ba5f18f53e52618f12ffdb00165d003aadbbfdbc07d5852e6e", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "37c7107de9392a6fa6f75653bb511027e99365c9d32ac29e18774b606159840f", "407b2685bafa5cff21e1c09a437f8c8c4d5604fa6f16b1635a36809a8d1d8ab0", "3d5ecbb4ab952dce3143cfd722261df86bd8edcb81d5c897b372cbef2ab7d13d", "e1edc3892810ad9f6fe16483fd2fa55bc01da083277299df3706a48e3fd8878e", "a3aa3cde46f679b9ccb080ec657b8ec83312b43b5ee0fee77834a95d790d9266" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2966, "output_tokens": 1863, "cached_tokens": 544, "wall_ms": 17676 }, "judge_usage": { "input_tokens": 35534, "output_tokens": 1698, "cached_tokens": 32378, "wall_ms": 20028 } }, { "draw_index": 1, "generation_hash": "0ac1dc106bce4333f7e484ee438e5cfceb390788262701299b2ba5fb6530a4c3", "status": "measured", "samples": [ 1, 0.75, 1, 1, 1 ], "judge_sample_hashes": [ "2ed0614038bc4c55503e3a908366aec950ff08d350e24d4ddaca2a87453f62e1", "3a76f5d66073b375e3a6c8ee824be59badd720958577895be0c56c28667068f9", "56be7c8cf510a1f3b5687ab3881eb5b220b14a631f6c0c4ebc98f73f2308598a", "eeb78fd4ed7ee69ac99356ea18dd2eaa1e35730682f65a57569005e85a3683ed", "15e199fdee6a5b30cbf16287aa18d8ce973a88d92bec27a594dc2e8d0fecbef2" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2969, "output_tokens": 2084, "cached_tokens": 2759, "wall_ms": 16241 }, "judge_usage": { "input_tokens": 36800, "output_tokens": 1258, "cached_tokens": 33390, "wall_ms": 17461 } }, { "draw_index": 2, "generation_hash": "b05a3bbae570bae396ca2fbba902180c61d4140f105945c3b76b45e0cf7ebf3b", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "a19ccfe74795e76e1b3a29c7a3401b905b506694c0b893f084f9751bd3b15e17", "9d8b20b4fab4d6c3766813474f5baa844b57883ccbd42bdc928a85b2f173cbc3", "9643cf914ea009bdc5fa049fa3f1b3630d0f81c6f93f98d6ea90230f37e53bbb", "a4dd28e1574018d2ddad9174df6e4a342f1d7a4e4e242bfb569ad82f80b6e70e", "02aef6b0d33518ce966652d53a9c89708e40a750cdb7978221f870f05d69ed4d" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2968, "output_tokens": 2915, "cached_tokens": 2759, "wall_ms": 21511 }, "judge_usage": { "input_tokens": 39504, "output_tokens": 1976, "cached_tokens": 35554, "wall_ms": 21004 } } ], "issue408": { "attempt_id": "47f9da585f12d468bcdb3b19e38fa79abda7b7aefa872eb897bbf3e41a627a48", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "prototype", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/prototype/SKILL.md", "declared_name": "prototype", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/prototype/SKILL.md", "skill_blob_sha1": "a0044501fe0d385b4d8575b610188ede9b236ccf", "skill_sha256": "714de632d116bb73f65cdb5a882db15b9369a6713b9a47c0fad827848f0bfbe3", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--prototype.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:31:35.229Z", "configuration_unchanged": true }, "budget_cost_usd": 0.84447, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "logic", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 6, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 7, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 8, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 9, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } }, { "id": "logic", "mode": "baseline", "outcome": "fail", "score": 0.5, "mean": 0.5, "stddev": 0, "samples": [ 0.5, 0.5, 0.5, 0.5, 0.5 ], "generation_hash": "113c75109850526dbc83660cd859a4a90936b83b0f8b375d394b3389606fd217", "judge_sample_hashes": [ "1f07cc7939dd110ab9525dc2dea0b6e1d7392ca2113faf506336602e307789cd", "28c669992f8ccad17e2963338c245cb7dd35f66b66257d470649445180106629", "4163c31af7c41051f0c28c2f0e732a65b2ac1b471bdaf64cbaabde40a9836684", "7783d46da89f07075be69750d620b618d3204988b9b97ed6c47bc9c489e9e395", "1c2808441773f6eb5a2c59d88d58f5b3f732d540e080c78ae51a4f25f4143cb6" ], "threshold": 0.75, "reason": "Meets criterion 1 (state-machine demo) and criterion 4 (throwaway, in-memory, no persistence). Misses criterion 2 (state is printed only at the end, not after every action) and criterion 3 (a Python script, not a single HTML demo).", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "a8afcecf3740c46b62f144480eb24012410586e9791d58d138f3d27b85ebcad7" }, "judge_usage": { "input_tokens": 34632, "output_tokens": 1595, "cached_tokens": 31658, "wall_ms": 18825 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.5, "sd": 0, "judge_sd_mean": 0, "variance_ratio": null, "variance_ratio_unavailable": "judge_sd_zero", "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "97df2dac47785a6a46b579c12d2add1438e75b4c741ccb77b4477b8b75d90e42", "status": "measured", "samples": [ 0.5, 0.5, 0.5, 0.5, 0.5 ], "judge_sample_hashes": [ "3e775e8010ffa098470fc99a3378a52654013cdd256f3cb41f48b3800e9c9159", "9fc81782d7d54ba5e39af63c65555e9c0befaa2b03fb266781285ebf2e7f905c", "13c849eee48a610b1dd5a26d1d6aaa39345e84492f0822e49a1517f5ae2157b7", "e02e824dfbaa8f2bcd3935bbeb17487faaf4edbc405ca2b7cac99f23f49da69f", "668c4c650002dea3137610453b0db44f05532a86108ef331b43b4e432e353846" ], "mean": 0.5, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2021, "output_tokens": 2532, "cached_tokens": 1467, "wall_ms": 20500 }, "judge_usage": { "input_tokens": 35807, "output_tokens": 1741, "cached_tokens": 32594, "wall_ms": 19488 } }, { "draw_index": 1, "generation_hash": "afa77c5e259664181108f925bd58addc13d6385c138b57cb64fc739530146a09", "status": "measured", "samples": [ 0.5, 0.5, 0.5, 0.5, 0.5 ], "judge_sample_hashes": [ "afb81214c2458b740150028f27641383addccaa9cd45eba388c51d248160d432", "b04f623fc1201ffa612fe494be437c6dbc91b238e347311ed53b744aae9f8d67", "2b264319493385f873e9d3c22039b9e836e58fbd5d229ffb22899ac7015d7552", "3e464e0c6b5c6b53d9676ad127a23456436cf7b816941b4ae2e941c5fcc208f0", "c28178f99919a3c39b3d180549e1acb31ba329cf5daadffd572069988505f21d" ], "mean": 0.5, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2019, "output_tokens": 1529, "cached_tokens": 1812, "wall_ms": 13117 }, "judge_usage": { "input_tokens": 35099, "output_tokens": 1530, "cached_tokens": 32030, "wall_ms": 18640 } }, { "draw_index": 2, "generation_hash": "113c75109850526dbc83660cd859a4a90936b83b0f8b375d394b3389606fd217", "status": "measured", "samples": [ 0.5, 0.5, 0.5, 0.5, 0.5 ], "judge_sample_hashes": [ "1f07cc7939dd110ab9525dc2dea0b6e1d7392ca2113faf506336602e307789cd", "28c669992f8ccad17e2963338c245cb7dd35f66b66257d470649445180106629", "4163c31af7c41051f0c28c2f0e732a65b2ac1b471bdaf64cbaabde40a9836684", "7783d46da89f07075be69750d620b618d3204988b9b97ed6c47bc9c489e9e395", "1c2808441773f6eb5a2c59d88d58f5b3f732d540e080c78ae51a4f25f4143cb6" ], "mean": 0.5, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2019, "output_tokens": 1550, "cached_tokens": 1812, "wall_ms": 13944 }, "judge_usage": { "input_tokens": 34632, "output_tokens": 1595, "cached_tokens": 31658, "wall_ms": 18825 } } ], "issue408": { "attempt_id": "47f9da585f12d468bcdb3b19e38fa79abda7b7aefa872eb897bbf3e41a627a48", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "prototype", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/prototype/SKILL.md", "declared_name": "prototype", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/prototype/SKILL.md", "skill_blob_sha1": "a0044501fe0d385b4d8575b610188ede9b236ccf", "skill_sha256": "714de632d116bb73f65cdb5a882db15b9369a6713b9a47c0fad827848f0bfbe3", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--prototype.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:31:35.229Z", "configuration_unchanged": true }, "budget_cost_usd": 0.84447, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "logic", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 6, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 7, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 8, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 9, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } }, { "id": "ui", "mode": "with_skill", "outcome": "pass", "score": 0.89, "mean": 0.89, "stddev": 0.089443, "samples": [ 1, 0.75, 1, 1, 1 ], "generation_hash": "cd1a0e4508881e3c0cf4583f3b979f758024ea611e4fb4e1a9d4916672b38af2", "judge_sample_hashes": [ "823296991fcbaccda4d26c7434485c98fe594e74a61287b540c74a52717f6982", "5ebfd7399aa65ee0e137f2d78c73f0dddcf452bd69bee2bea46641ec70a1487b", "efed1d4604bae65aef0996100da20dacc2412904c530ca3ca63e3fd066d998d8", "8b10cd991b2ab21e1b5d76351bd106024a0fb3e5885e5f4472d56968320519f9", "e8c2334b19185ea72265d6d8f29fc03f70705389ee3ba7ef79f9cfe9d5d4f569" ], "threshold": 0.75, "reason": "Chooses the UI branch, gives three structurally different variants on one route, switches via a ?variant= URL param, and defers to the existing route convention while labeling the work as a prototype on a throwaway branch.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "8a0a980b20ecd0b369c0a573edc6dae34355e45e180dfa772845753c0b6c8a5d" }, "judge_usage": { "input_tokens": 32866, "output_tokens": 3457, "cached_tokens": 30242, "wall_ms": 27639 }, "generation": { "n_planned": 3, "n_drawn": 5, "n_measured": 5, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.89, "sd": 0.089443, "judge_sd_mean": 0.094468, "variance_ratio": 0.946807, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "f0f73b4e47311939350f7476fa0e5aac11a50e546fa5a2f79196de9c1d41a7b2", "status": "measured", "samples": [ 1, 1, 1, 0.75, 1 ], "judge_sample_hashes": [ "48323f938f4dc7074bfbfb67f2283b289371e126f09352f57df91f67f8dafec9", "bc1ab84bd01c681c0f74beb4762fb5368357438f863a512c31842d3ec01aa214", "8ef099c81c03f5b2c6914eeb4db0aa401ea260e04a8939de980995c06994d027", "5eb5f716cfe189b9fd85309a051fecee0b97692fa7da2db5c2277d8d699e8a47", "5002c4195ab0b789a4c85ea0fe6c5a6e050ed7c716b8624b7c550c94d1c70e7b" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2964, "output_tokens": 1028, "cached_tokens": 2414, "wall_ms": 11147 }, "judge_usage": { "input_tokens": 32855, "output_tokens": 2869, "cached_tokens": 30234, "wall_ms": 23906 } }, { "draw_index": 1, "generation_hash": "c45cfa5b597929bcd4c07eecba2596a79d91b44da598def382955173cd2cfbd6", "status": "measured", "samples": [ 0.75, 1, 1, 1, 1 ], "judge_sample_hashes": [ "c5a3eaf71340a1ebd1ccb5b056e156789fde546000bad56c299902854017098f", "785fe7eda450a58bf0149e94034602ef9322e09468548521225ca85de45415c9", "4bc195aab801551e089d58b053b84be1a9ac79791bdb16b1a5e5be1867a627a9", "8767a1e19b4cabb31a351460423b1366612105f027164da64b8e0e348a8d21c8", "3e9f051d09f8dcdbd50c0c7f133249a31337a3608cae4e29970d3659a6d13b0a" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2965, "output_tokens": 1141, "cached_tokens": 2756, "wall_ms": 11987 }, "judge_usage": { "input_tokens": 32967, "output_tokens": 2368, "cached_tokens": 30326, "wall_ms": 22767 } }, { "draw_index": 2, "generation_hash": "8bc6dfdd4d8a04b1dc455a39ecbe12ce164a4340765821f88bb1f272e1c15e21", "status": "measured", "samples": [ 0.75, 1, 0.75, 1, 0.75 ], "judge_sample_hashes": [ "5abbcadd4e93952e08a04233ee8c9e0dc985650d9fa154216e60fce36cb0a73c", "00adee924c7328f0455eaa5594784cd7cf46c002e2ef21a01f0f5589440321e9", "861dc63115c3bdf482923315fee34500fc3abac32739c4b4e46921218844c769", "14749fb4a38b45a1307571e8fb1987d28d4e130fdb5a34022cb9eb0ec3edd2d6", "6d3a3e6ca43abde203a385a75240127890e2092ba49183548b491fb39d5ad4ea" ], "mean": 0.85, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2964, "output_tokens": 1300, "cached_tokens": 2756, "wall_ms": 13434 }, "judge_usage": { "input_tokens": 32575, "output_tokens": 3215, "cached_tokens": 30006, "wall_ms": 25802 } }, { "draw_index": 3, "generation_hash": "395f389e5969545419e2af681e349a696c7acd6cd56301a298d3d92ada3ec560", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "671b1b5f45b31c00efb9362bfca24de2bd255e222c219a199804a7cf1f794018", "9866f8736cdb7c08969e492714e4b33d04e4732f88e29d6308b5f2730d2307ca", "149a575118f876de8487cbbcad4c76800df22bb77e32b223d123071e3e79a30d", "c7adae5854777fbd66af164965b37324aa0f7c25f21bc53ab80af3cee3dd2bc2", "7dff937a9c5ee11b38d86d01a6c7e5e57f1f19481341b841d99988714ad93918" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2965, "output_tokens": 912, "cached_tokens": 2756, "wall_ms": 10184 }, "judge_usage": { "input_tokens": 31929, "output_tokens": 2800, "cached_tokens": 29494, "wall_ms": 27949 } }, { "draw_index": 4, "generation_hash": "cd1a0e4508881e3c0cf4583f3b979f758024ea611e4fb4e1a9d4916672b38af2", "status": "measured", "samples": [ 1, 0.75, 1, 1, 1 ], "judge_sample_hashes": [ "823296991fcbaccda4d26c7434485c98fe594e74a61287b540c74a52717f6982", "5ebfd7399aa65ee0e137f2d78c73f0dddcf452bd69bee2bea46641ec70a1487b", "efed1d4604bae65aef0996100da20dacc2412904c530ca3ca63e3fd066d998d8", "8b10cd991b2ab21e1b5d76351bd106024a0fb3e5885e5f4472d56968320519f9", "e8c2334b19185ea72265d6d8f29fc03f70705389ee3ba7ef79f9cfe9d5d4f569" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2965, "output_tokens": 1077, "cached_tokens": 2756, "wall_ms": 11215 }, "judge_usage": { "input_tokens": 32866, "output_tokens": 3457, "cached_tokens": 30242, "wall_ms": 27639 } } ], "issue408": { "attempt_id": "47f9da585f12d468bcdb3b19e38fa79abda7b7aefa872eb897bbf3e41a627a48", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "prototype", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/prototype/SKILL.md", "declared_name": "prototype", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/prototype/SKILL.md", "skill_blob_sha1": "a0044501fe0d385b4d8575b610188ede9b236ccf", "skill_sha256": "714de632d116bb73f65cdb5a882db15b9369a6713b9a47c0fad827848f0bfbe3", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--prototype.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:31:35.229Z", "configuration_unchanged": true }, "budget_cost_usd": 0.84447, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "logic", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 6, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 7, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 8, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 9, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 5 } }, { "id": "ui", "mode": "baseline", "outcome": "borderline", "score": 0.63, "mean": 0.63, "stddev": 0.175119, "samples": [ 0.5, 0.25, 0.5, 0.5, 0.5 ], "generation_hash": "bf64fff295fd9d7b750b23550c7148ac5cd8a38496946d14803871982765bb60", "judge_sample_hashes": [ "79e6469737d904ee25248e8aac8c9586f0dc9a31480d4ee03e32a23be6683e83", "4786a5eff2ae25cc495d2a2191791939c2c70013ca825fcb349a7faac886aab2", "dbd0a99a92b3d13107bc9729f58c6b5dc0b94c9c68935825c6bd8fb9fc76e84f", "ee322965abfd61fb10fc87c82ebf46c016097eab7169b6300e08cddf6d2fce14", "4ada00ff40a77f3319875afa6bbd13c6c8c3da85d3bc4be6479033fece2b0ae6" ], "threshold": 0.75, "reason": "Chooses a UI prototype and gives a 'Disposable' throwaway label, but puts variants on separate routes (/a, /b, /c) instead of one route, and does not respect the existing route convention.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "8a0a980b20ecd0b369c0a573edc6dae34355e45e180dfa772845753c0b6c8a5d" }, "judge_usage": { "input_tokens": 32129, "output_tokens": 4562, "cached_tokens": 29654, "wall_ms": 31803 }, "generation": { "n_planned": 3, "n_drawn": 10, "n_measured": 10, "n_unmeasured": 0, "stopping_reason": "max_reached", "mean": 0.63, "sd": 0.175119, "judge_sd_mean": 0.110702, "variance_ratio": 1.581896, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "3179abf6d3e83f641cc3628ec121af0d98904802004402779caf1f64f92246e5", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "bcd610123b21a05894de124177d8d952fdb5410739826c1aa484a7cccb7a8b1d", "248c04f9ca85d6e14edbea599cb87dc476d89ce606e298fc4c000dd3ed5f3dd0", "6870273995e14e616d05958065e67b92048f478f441c4ef28b129c73e41bd090", "2df41f09322b9c7030505efa03f298db5f3d921b51a554645ea5b3f7341efedb", "3026210c3b694ea6a0b458f62e42346193920ad99fa132d5362a67827172a331" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2017, "output_tokens": 833, "cached_tokens": 1467, "wall_ms": 10328 }, "judge_usage": { "input_tokens": 31881, "output_tokens": 3906, "cached_tokens": 29458, "wall_ms": 29775 } }, { "draw_index": 1, "generation_hash": "9d0783d247d9c7ab8f309e4b82931177755132329e6c18d64112741c6e708a30", "status": "measured", "samples": [ 0.5, 0.75, 0.75, 0.5, 0.75 ], "judge_sample_hashes": [ "f56c4725825cd18cd8ee46d9b5a9cacb9aa4fc7cbba74d978399e4d6e9be3667", "9a0aff7df0b947ec5efe7ef2b12d49b0a8ff346c19e2371dde4cbb6469a2542a", "1febafd6753b212e6030f3c47d12ebd21b65486db34139483e617e13704ec06a", "be893291212088886184c75879c66c026dd9ff887c21952979224daabf985395", "cae62ee28635e926e23ea772cea8799dee88a053ec7d34e514c264bf989d5c09" ], "mean": 0.65, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2017, "output_tokens": 826, "cached_tokens": 1809, "wall_ms": 9772 }, "judge_usage": { "input_tokens": 31794, "output_tokens": 3911, "cached_tokens": 29386, "wall_ms": 29405 } }, { "draw_index": 2, "generation_hash": "7781eb5b3f200774829352e96a2de6951b6c866aa38c384ec8058ab6a0de5d61", "status": "measured", "samples": [ 0, 0, 0.5, 0.25, 0.25 ], "judge_sample_hashes": [ "a80276d1a418ef70de79a263c13d7cac03eebdf5dbe49104b84d181fb8372b5c", "7d43cd8e17d9b4f0417cd76f61ddd76453dfd415ac292765f6cf553ea6ec243d", "60b4557bc01db590b35da55c4228c4bba6e81c00b3d1b0fbe2d0f240e85e0300", "695aa686a919051771f75a3b8b09a87408c54be9c3a6b5f0007180bf5fd6859a", "543d9b55101d1327c0e1ea091e4ad41075b3cd64f14ef77a92bd36c161d38943" ], "mean": 0.2, "stddev": 0.209165, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2016, "output_tokens": 923, "cached_tokens": 1809, "wall_ms": 10263 }, "judge_usage": { "input_tokens": 31884, "output_tokens": 3587, "cached_tokens": 29458, "wall_ms": 27983 } }, { "draw_index": 3, "generation_hash": "4c768e762f1b79e7dfe271216b9e884f1e6a52db6f9c4d571659e9c626c0bd32", "status": "measured", "samples": [ 0.5, 0.75, 0.75, 0.5, 0.75 ], "judge_sample_hashes": [ "45b70015446d8904d184f003b75c290710291fc3c65e23473cd916877cbe3eab", "d203df03bd0cc58274856dd191f2e020fc4ae5781c3f4f081208a6c804ece3d3", "5fb74063ee5bd29fd9726cf67ef5969b8e7f61fe56d1480f0aac421d1473b6ac", "6078533d1bb13108cfd75c5dfb5ae294a96a70d582e672d9deaa2fd0cbb6087d", "908e682c40c37916b1b9d87bd44c6b1c0d644caa0b7898c7dbedb4eaa15d2cdd" ], "mean": 0.65, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2016, "output_tokens": 885, "cached_tokens": 1809, "wall_ms": 10259 }, "judge_usage": { "input_tokens": 32154, "output_tokens": 3277, "cached_tokens": 29674, "wall_ms": 27621 } }, { "draw_index": 4, "generation_hash": "3a5b375133014a9f9ae2a6404651115c56c3f8b63e3c0332169ac058677f5b0f", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "b54154a0288662adca8388e48cebfaa64f1eb372c6baccf49776c2ae627e4b70", "8383b5e5ad7ad38bc39d20cd993c9077da91a64f8d0ea49f45eabf488d1d25ac", "9ae69c5e8fdeb6f8ff64785d25d111dae7e27d107050f79c8ded5921ca1e2bfd", "9457324bfe30beed05bd33d129ee57d4e9e6e050693b4cd8ccccccf2acd241be", "1220aba4305e2d41c6a2415f176cfa2b8c6ed0e63882a6bb4329e5bd25fbd830" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2018, "output_tokens": 792, "cached_tokens": 1809, "wall_ms": 9368 }, "judge_usage": { "input_tokens": 31361, "output_tokens": 3729, "cached_tokens": 29038, "wall_ms": 29084 } }, { "draw_index": 5, "generation_hash": "77df9e869e528fa759346bffad5395926819561c9fa620dabc62c1a897e52929", "status": "measured", "samples": [ 0.75, 1, 0.75, 0.75, 0.5 ], "judge_sample_hashes": [ "aa86ca0780503b05d6c93e56674195ac2311578ddf82c251dd11a1b0b8d6e064", "e702294134407754805b41c0e08be7d317b0c56f889dcc3746a83e76629f96cf", "d4880fd4e97726b3cb5d3d72d749970713d0511f1d28e8e59d85b46d013ce960", "45ba9b8df252ec9b5a27829afd1fc39e6b29f2b01e5174ab30a7d9aaec387b55", "09ae0003f0c33331af91e45e11936121c56413d596b67582aa0638ab63ce8e78" ], "mean": 0.75, "stddev": 0.176777, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2018, "output_tokens": 796, "cached_tokens": 1809, "wall_ms": 9567 }, "judge_usage": { "input_tokens": 31629, "output_tokens": 3798, "cached_tokens": 29254, "wall_ms": 28221 } }, { "draw_index": 6, "generation_hash": "0f8c23c7d61c144818acbedfc960cba322c29834a103994cc6bde7bdfa133eca", "status": "measured", "samples": [ 0.5, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "d601f460cb51b45bdbc698a47817bfa7d0ab062680546dfa05d478eb2aacb838", "8c215b1d097f0878dd53b00ca89e13f2327f9badd2572afbee2be2d4ee8ca69a", "fcea669dbcb9661c27739bf5b41ade5c1a8f4ae7c31ab7f32cff4c431de35755", "409f67c7d2783438a28c73a79374e28dc6e8080b931035f703129e40a752ee71", "df4f467942c2c1e4d92965c3909874024235ac9fd27c1e3a8dd41cb08e571cbc" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2018, "output_tokens": 766, "cached_tokens": 1809, "wall_ms": 9280 }, "judge_usage": { "input_tokens": 31473, "output_tokens": 3121, "cached_tokens": 29130, "wall_ms": 25523 } }, { "draw_index": 7, "generation_hash": "4252c61c554c08586ef90e99ffb0c7517711cc9bf9e7ff9f4f48adec5bdb52cc", "status": "measured", "samples": [ 0.5, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "f68979657fe2c4779d5a3d21892a7b22360512f23b2aea1bfa502b3229bb692e", "c798189665059be6835b0bda11fa2730427f7863cdde19bc87ff11cd3f997312", "0ca0ae93408913a87b259e9e4d0eba6ee460d48ad952be260571049eeee4cf39", "02278447ab24a88b5c83bac3457c0d1668fdaa4291f00cc73d874afe427a1549", "89ddb990b6fecbaa204c51a74194a9de912f7c1e1c7061994e31ee8795661ee1" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2018, "output_tokens": 818, "cached_tokens": 1809, "wall_ms": 9633 }, "judge_usage": { "input_tokens": 31944, "output_tokens": 3536, "cached_tokens": 29506, "wall_ms": 33410 } }, { "draw_index": 8, "generation_hash": "d186fd5e948ba1a8660fc64527c1c8816092224ca52259a39ed14594c2e3193f", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.5 ], "judge_sample_hashes": [ "baadd79c1f2a8734a69d135d3c1d3d50852a8cf8530637b9d82bdc96a32ee72d", "131370febf0d041461da5c275c33100173c024fc633ef6aca94d086484c3147e", "325497116dccfd78803c92192b7859e16682954043f15511b5cfde9c1b7fce22", "6f60afbd026533812c3fc3808f721ccd886415bb8d613ea8eb3501126e53f2a4", "25da1cd54092a0ae47da93463daca740024d8887cbf163b47ffa4234857afacd" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2018, "output_tokens": 770, "cached_tokens": 1809, "wall_ms": 9428 }, "judge_usage": { "input_tokens": 31506, "output_tokens": 3830, "cached_tokens": 29158, "wall_ms": 28717 } }, { "draw_index": 9, "generation_hash": "bf64fff295fd9d7b750b23550c7148ac5cd8a38496946d14803871982765bb60", "status": "measured", "samples": [ 0.5, 0.25, 0.5, 0.5, 0.5 ], "judge_sample_hashes": [ "79e6469737d904ee25248e8aac8c9586f0dc9a31480d4ee03e32a23be6683e83", "4786a5eff2ae25cc495d2a2191791939c2c70013ca825fcb349a7faac886aab2", "dbd0a99a92b3d13107bc9729f58c6b5dc0b94c9c68935825c6bd8fb9fc76e84f", "ee322965abfd61fb10fc87c82ebf46c016097eab7169b6300e08cddf6d2fce14", "4ada00ff40a77f3319875afa6bbd13c6c8c3da85d3bc4be6479033fece2b0ae6" ], "mean": 0.45, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2017, "output_tokens": 884, "cached_tokens": 1809, "wall_ms": 10167 }, "judge_usage": { "input_tokens": 32129, "output_tokens": 4562, "cached_tokens": 29654, "wall_ms": 31803 } } ], "issue408": { "attempt_id": "47f9da585f12d468bcdb3b19e38fa79abda7b7aefa872eb897bbf3e41a627a48", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "prototype", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/prototype/SKILL.md", "declared_name": "prototype", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/prototype/SKILL.md", "skill_blob_sha1": "a0044501fe0d385b4d8575b610188ede9b236ccf", "skill_sha256": "714de632d116bb73f65cdb5a882db15b9369a6713b9a47c0fad827848f0bfbe3", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--prototype.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:31:35.229Z", "configuration_unchanged": true }, "budget_cost_usd": 0.84447, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "logic", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 6, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 7, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 8, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 9, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 10 } }, { "id": "unclear", "mode": "with_skill", "outcome": "borderline", "score": 0.733333, "mean": 0.733333, "stddev": 0.028868, "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "generation_hash": "6cf33802e60c9d1a556c0820255398e0323c4d584054d31617bafcd2b2e40155", "judge_sample_hashes": [ "fdb8d942f2ad8237673692642cc946679ebfc6eeb0bb2a729adda379f9530146", "020d936096adf49b38cb3b57c922616c3302c2788833f55cb0ae98a21ccdac05", "4b84cb25205be9719ab4bb7eaa2c27d5bf2afefe33d961e02dccf1513b70f7cc", "3ce05758085b54c73e7ae44498e8a0a95784eb26f460986d41b2c72d3f095375", "065aed1bdc404e5a29b67f154d9f76c681e1ca5148e0098a56ce7a676501de49" ], "threshold": 0.75, "reason": "Meets criteria 1, 2, and 4: it states a logic-branch assumption, names concrete retry-state questions, and flags the assumption as unconfirmed. It misses criterion 3 because it lists states and asks about transitions without actually showing any transitions.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "5818f13d15039ac93b1e641011647b7ff9304733a199a48cfa7f9fe5881a4f68" }, "judge_usage": { "input_tokens": 32141, "output_tokens": 1874, "cached_tokens": 29662, "wall_ms": 19663 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.733333, "sd": 0.028868, "judge_sd_mean": 0.037268, "variance_ratio": 0.774606, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "0a0d9d26ae5fe85f8cf886a0c78aa0d0d1a5e45e8e671bb19d9444193bdc21f0", "status": "measured", "samples": [ 0.75, 0.5, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "ccdd6f82737be5e6b4a89517541bf5a76331b9598fd96fc31f47f0b1aa6c005c", "8f9f8326789c1b9127138dcd99477815c062aba9094a21f593af169b3e11680b", "3a70b76cef4acc95bf4d363434f036cb4c1eff0a166d11fc61e2dd329bdffa44", "86e37136c100a94d051ff66e5a72d4c9ad01b859a878a6141e8ad725e4d831de", "9192fdbd5c8c97af858c60767f19f3b084c172d9e91520141460e60a77560c1e" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2975, "output_tokens": 715, "cached_tokens": 2414, "wall_ms": 7852 }, "judge_usage": { "input_tokens": 31821, "output_tokens": 1524, "cached_tokens": 29406, "wall_ms": 19634 } }, { "draw_index": 1, "generation_hash": "a446b3024539699d82b5d6065a0aa729affcc16aabd8de3871c2967a7e8e074f", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "6cdc5e487b35ed6cfc4208bda19e4973f1a2816a813b50869d0357912831a7e4", "0875907134da636090d53f6342c29b8e36b74fd0e0589fd8bc88d81626f1dd63", "6761a3eb03ea31b74f408170025c62e5d41ace2ed91c0c1e205b85b8090bb63b", "409ccbb65716c8f58547499093fd18fa6b2ce603ca41a2903597a6c84aae1d1f", "36c0775a18f6ea7d775ea7e36f860ef822fcb9ccd5596b7518bd57b4aab54d3d" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2975, "output_tokens": 664, "cached_tokens": 2766, "wall_ms": 7378 }, "judge_usage": { "input_tokens": 31565, "output_tokens": 2119, "cached_tokens": 29202, "wall_ms": 21671 } }, { "draw_index": 2, "generation_hash": "6cf33802e60c9d1a556c0820255398e0323c4d584054d31617bafcd2b2e40155", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "fdb8d942f2ad8237673692642cc946679ebfc6eeb0bb2a729adda379f9530146", "020d936096adf49b38cb3b57c922616c3302c2788833f55cb0ae98a21ccdac05", "4b84cb25205be9719ab4bb7eaa2c27d5bf2afefe33d961e02dccf1513b70f7cc", "3ce05758085b54c73e7ae44498e8a0a95784eb26f460986d41b2c72d3f095375", "065aed1bdc404e5a29b67f154d9f76c681e1ca5148e0098a56ce7a676501de49" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2975, "output_tokens": 779, "cached_tokens": 2766, "wall_ms": 8490 }, "judge_usage": { "input_tokens": 32141, "output_tokens": 1874, "cached_tokens": 29662, "wall_ms": 19663 } } ], "issue408": { "attempt_id": "47f9da585f12d468bcdb3b19e38fa79abda7b7aefa872eb897bbf3e41a627a48", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "prototype", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/prototype/SKILL.md", "declared_name": "prototype", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/prototype/SKILL.md", "skill_blob_sha1": "a0044501fe0d385b4d8575b610188ede9b236ccf", "skill_sha256": "714de632d116bb73f65cdb5a882db15b9369a6713b9a47c0fad827848f0bfbe3", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--prototype.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:31:35.229Z", "configuration_unchanged": true }, "budget_cost_usd": 0.84447, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "logic", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 6, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 7, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 8, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 9, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } }, { "id": "unclear", "mode": "baseline", "outcome": "fail", "score": 0.466667, "mean": 0.466667, "stddev": 0.08165, "samples": [ 0.5, 0.75, 0.5, 0.5, 0.5 ], "generation_hash": "11f745da87927688482c9c827620322ced491a7ad280cfb68d629b80207d2730", "judge_sample_hashes": [ "cc58b5699d553f628a30e0121a6d6e08f48705dcc7a11a91d41c30ebfb4ff3d4", "70d808918a26afeb7e3f6aad6d19433fa52ca6cabae9672f8adc6498dd3812bf", "8dcc79cc5d4955d51df6c191a12fca9c803619be2db3f8e3bbe4f6f111af5c5f", "d125d808cef706a5d31b0a1e44ee6fd17b32972f64cf2ecfbf9fdad61498c9ac", "5a31efd1f256e93a364ab3807c65c1442696caa40f6db33fae55802752af1a9a" ], "threshold": 0.75, "reason": "Criterion 1 is met (clear logic-oriented assumption that the suggestion is the working assumption) and criterion 4 is met (explicitly provisional, not confirmed). Criterion 2 is partly met via retryable-error question, but criterion 3 (state transitions) is missed entirely.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "5818f13d15039ac93b1e641011647b7ff9304733a199a48cfa7f9fe5881a4f68" }, "judge_usage": { "input_tokens": 31089, "output_tokens": 2428, "cached_tokens": 28822, "wall_ms": 22370 }, "generation": { "n_planned": 3, "n_drawn": 6, "n_measured": 6, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.466667, "sd": 0.08165, "judge_sd_mean": 0.101545, "variance_ratio": 0.804077, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "2e5cf29fbb0db64b471d4ff9009c21ccb48525bc0ab38e0d1146c72c8cc9f693", "status": "measured", "samples": [ 0.5, 0.5, 0.5, 0.25, 0.25 ], "judge_sample_hashes": [ "ee95dc2115e7b006c7e5d339bf2d789342b31e9f7719e22e30d972fb494aad7e", "079748e88b7ce26480ba6a49120abb60c985ac4df89f0c513e8eef079d6fdece", "bc8fb8716745cf4a50fc8c2dc306441dc5b03d136c6cbc5387b67c56f26c74da", "8c45e278d690299f7273571675c7a8054d061612c68f68196a5d90b2a966e0cf", "b206c344921b41d72081d4af9432bb1bc39089fce35db3ad13030df03ca140c9" ], "mean": 0.4, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2029, "output_tokens": 615, "cached_tokens": 1467, "wall_ms": 11273 }, "judge_usage": { "input_tokens": 31319, "output_tokens": 1306, "cached_tokens": 29006, "wall_ms": 18096 } }, { "draw_index": 1, "generation_hash": "d59c23eb4c5c4e9005e5bef5e9f5c8f8c31abaf4e414eac470f70ea30206a4f1", "status": "measured", "samples": [ 0.5, 0.5, 0.5, 0.5, 0.5 ], "judge_sample_hashes": [ "c80a8dc57eb6c823321bed7691fa0e289fef8157ad4fbc082a0bc9b8a1d1d7ef", "707f7b96533c4d9dbbcee10eb67f5727be4576b083d00340455b3927d81c0383", "7969fb633910151cfc8026968d94ed58fcc41120b32cc4a468f5512b84ab2cbd", "ff9ed22babe6b13380c579a9fd0b2e964b82a92ccf25d2c52cb366a9d4db151e", "0ef4f5921409bd99459100fa48164183e57171ab6445225e4b67612248bd96e8" ], "mean": 0.5, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2028, "output_tokens": 676, "cached_tokens": 1819, "wall_ms": 7886 }, "judge_usage": { "input_tokens": 31625, "output_tokens": 2866, "cached_tokens": 29250, "wall_ms": 25377 } }, { "draw_index": 2, "generation_hash": "b368597f61c9d4de148e1ec690d24daebe7fb435ba45716de2032f29e7428bc3", "status": "measured", "samples": [ 0.25, 0.25, 0.25, 0.5, 0.5 ], "judge_sample_hashes": [ "4c778f17fab20b38f932ef153a4724ab969ae2c6fba76709dd4674530d2a7c30", "7402b24807bc16c3cd3817a5c53bb0d104216fb3752210f869242297f086d57b", "a1698520fc69d1562437ec13d558ae38c07b7ee8c0db40593d657c46f29ff40c", "4615051f22c9efcc0b4c54c5b865417bcbb49e9cbe4c1a09f5a476fc9d1fd6e9", "15d7d56b094f526dbbc3124702467692acfaad6adbab07f917c2ec3de223b74c" ], "mean": 0.35, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2028, "output_tokens": 720, "cached_tokens": 1819, "wall_ms": 8336 }, "judge_usage": { "input_tokens": 31841, "output_tokens": 2838, "cached_tokens": 29426, "wall_ms": 25033 } }, { "draw_index": 3, "generation_hash": "0a687b8ed86b78f35ac94524f3c39e659447441ddaefff3d9828e58af87ff8d9", "status": "measured", "samples": [ 0.5, 0.5, 0.5, 0.5, 0.25 ], "judge_sample_hashes": [ "94a9b571bcfe0782441f9fda93811e6d00ab8aa74d336ec2ee10aa2accc08c32", "2d6ce355797b050dabef009686da660e6cd39ed96dc792d6712f77138b18a366", "00028d06daf9705a67c71879c3a6250be01722602eb736224da6fad9ec5a34f1", "4a3072c81e5dac4a882965f42430ae1be760ad220980fdeac944e9d686a0fe96", "4cedb7770f56b32c9820720e770393459b47efda1f7ec461c96f2048b4510d0f" ], "mean": 0.45, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2028, "output_tokens": 705, "cached_tokens": 1819, "wall_ms": 8480 }, "judge_usage": { "input_tokens": 31772, "output_tokens": 3046, "cached_tokens": 29366, "wall_ms": 25494 } }, { "draw_index": 4, "generation_hash": "91dbcc7bab42f7d791d9d2067f74a42ef7d61d0ad3298c3903f344f1671db012", "status": "measured", "samples": [ 0.5, 0.75, 0.5, 0.5, 0.5 ], "judge_sample_hashes": [ "8c4b5edee91407b9c35f8a09377a8c2dba58db842a50bc29441b235c7487494b", "fa7b6d8d14dbcf0cca84bcbc2d4fe18499b27df8a0f12753704df7fdc8a7cd27", "e6b906ddde170a2aa5769b33ef88d65b1dc102aae83b9eb65ca9b75f40c63553", "49ed250d2c0358bd7f76fc8c1c55d6c7cfa23a6498389eb67ddc771d03225e5f", "969f9b651410dfcce2420f297acb38326bb6e7b5abb46640a2771e9ad04b5697" ], "mean": 0.55, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2029, "output_tokens": 587, "cached_tokens": 1819, "wall_ms": 7268 }, "judge_usage": { "input_tokens": 31177, "output_tokens": 2600, "cached_tokens": 28894, "wall_ms": 24215 } }, { "draw_index": 5, "generation_hash": "11f745da87927688482c9c827620322ced491a7ad280cfb68d629b80207d2730", "status": "measured", "samples": [ 0.5, 0.75, 0.5, 0.5, 0.5 ], "judge_sample_hashes": [ "cc58b5699d553f628a30e0121a6d6e08f48705dcc7a11a91d41c30ebfb4ff3d4", "70d808918a26afeb7e3f6aad6d19433fa52ca6cabae9672f8adc6498dd3812bf", "8dcc79cc5d4955d51df6c191a12fca9c803619be2db3f8e3bbe4f6f111af5c5f", "d125d808cef706a5d31b0a1e44ee6fd17b32972f64cf2ecfbf9fdad61498c9ac", "5a31efd1f256e93a364ab3807c65c1442696caa40f6db33fae55802752af1a9a" ], "mean": 0.55, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2028, "output_tokens": 569, "cached_tokens": 1819, "wall_ms": 6903 }, "judge_usage": { "input_tokens": 31089, "output_tokens": 2428, "cached_tokens": 28822, "wall_ms": 22370 } } ], "issue408": { "attempt_id": "47f9da585f12d468bcdb3b19e38fa79abda7b7aefa872eb897bbf3e41a627a48", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "prototype", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/prototype/SKILL.md", "declared_name": "prototype", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/prototype/SKILL.md", "skill_blob_sha1": "a0044501fe0d385b4d8575b610188ede9b236ccf", "skill_sha256": "714de632d116bb73f65cdb5a882db15b9369a6713b9a47c0fad827848f0bfbe3", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--prototype.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:31:35.229Z", "configuration_unchanged": true }, "budget_cost_usd": 0.84447, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "logic", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "logic", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "logic", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 6, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 7, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 7, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 8, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 8, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "ui", "mode": "baseline", "draw": 9, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "ui", "mode": "baseline", "draw": 9, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "unclear", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "unclear", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 6 } } ], "aggregates": { "with_skill": { "case_count": 3, "pass_count": 2, "borderline_count": 1, "mean_score": 0.868889, "stddev": 0.12633 }, "baseline": { "case_count": 3, "pass_count": 0, "borderline_count": 1, "mean_score": 0.532222, "stddev": 0.086303 }, "band_rule": "per arm: the sample standard deviation (n-1) of the per-case means across the included cases; the comparison band is the quadrature sum of the two arms; null where an arm has fewer than two included cases" } }, "comparison": { "with_skill_score": 0.868889, "baseline_score": 0.532222, "delta": 0.336667, "delta_uncertainty": 0.152995 }, "verification_level": "TESTED", "receipt_hash": "cb33a80a2e7db6d3d612b4ea17dcd7648b245fe68eb6adc08989983b6e2a98db", "economics": { "basis": "metered-equivalent", "surface": "claude-cli", "with_skill": { "call_count": 11, "mean_input_tokens": 2968.27, "mean_output_tokens": 1316.18, "mean_cost_usd_per_call": 0.019098, "median_wall_ms": 11215, "wall_ms_p25": 8490, "wall_ms_p75": 16241, "wall_ms_iqr": 7751 }, "baseline": { "call_count": 19, "mean_input_tokens": 2021.16, "mean_output_tokens": 935.58, "mean_cost_usd_per_call": 0.013398, "median_wall_ms": 9633, "wall_ms_p25": 8480, "wall_ms_p75": 10328, "wall_ms_iqr": 1848 }, "skill_incremental_cost_usd_per_call": 0.0057, "skill_incremental_cost_usd_per_1k_calls": 5.7, "output_tokens_delta": 380.6, "median_wall_ms_delta": 1582, "judge_excluded": true, "judge_overhead": { "note": "Measurement overhead imposed by Driftproof, NOT a cost of running the skill. Excluded from every field above. Counted over every draw sent to the judge, each once: the usage its answered judge samples reported, at the frozen judge price. Retried attempts and samples that reported no usage are not in it.", "total_cost_usd": 0.139451, "case_rows_measured": 6, "judged_draws_counted": 30, "judged_draws_missing_usage": 0 }, "notes": { "absolute_cost": "Absolute per-call cost on a CLI surface includes a fixed harness preamble (observed ~25k input tokens on claude-cli, ~11k on codex) that we do not control. It is identical in the with-skill and baseline arms, so it cancels in the Δ figures — read the incremental columns, not the absolute ones.", "cache_pricing": "Cached input tokens are costed at the list input rate (an over-estimate where caching is heavy); cached_tokens is recorded per call so a reader can recompute.", "latency": "observed on subscription CLI surface, indicative" } } }