{ "schema_version": "0.13", "generation_sampled": true, "skill": { "name": "setup-matt-pocock-skills", "version": "0.0.0", "content_hash": "f11021a04851b50f14fb1584bb974c9eb44fe0c81027df4f7d04c25deff52771", "tokens": 1745 }, "suite": { "format": "agentskills.io/evals", "suite_hash": "a11911f9dcca438ccbffe323bcb319823ffe09aab693961e8cd07c48779c3cdd", "case_count": 3, "canary": "2eaaef5c-0ec2-72d5-72fd-bae7e65d80bd" }, "run": { "model_id": "claude-opus-5-5", "model_release_date": null, "provider": "anthropic", "surface": "claude-cli", "runner_version": "0.22.2", "date_utc": "2026-10-18T07:44:32.920Z", "generated_at": "2026-10-18T07:27:44.916Z", "registry": "registered", "transcripts": "retained-local", "judge": { "samples": 5, "temperature": null, "sampling": "surface-controlled", "surface": "claude-cli", "model_id": "claude-haiku-5-5", "prompt_template_hash": "82586d1e44f84df2b8ce5f18ac777866465cb5ad9686503c93d9776c93c344ed", "harness": { "name": "claude-code", "version": "2.1.287" }, "isolation": "eval-user", "endpoint": { "custom": false, "sha256": null, "resolved_by": "cli", "routing": { "ANTHROPIC_BEDROCK_BASE_URL": { "set": false, "sha256": null }, "ANTHROPIC_VERTEX_BASE_URL": { "set": false, "sha256": null }, "CLAUDE_CODE_USE_BEDROCK": { "set": false, "sha256": null }, "CLAUDE_CODE_USE_VERTEX": { "set": false, "sha256": null }, "HTTPS_PROXY": { "set": false, "sha256": null } } }, "code_revision": "judge-code-bb552899a1df887e", "reported_models": [ "claude-haiku-5-5" ] }, "answered_by": { "kind": "model", "attested": true, "reported_model": "claude-opus-5-5", "reported_models": [ "claude-haiku-5-5", "claude-opus-5-5" ], "isolation": "eval-user" }, "capture": { "mode": "text" }, "harness": { "name": "claude-code", "version": "2.1.287" }, "pricing_snapshot": { "frozen_at": "2026-10-18T07:44:32.920Z", "source": "config/models.json", "currency": "USD", "models": { "claude-opus-5-5": { "input_per_mtok": 4, "output_per_mtok": 20, "registered": true }, "claude-haiku-5-5": { "input_per_mtok": 0.1, "output_per_mtok": 0.5, "registered": true } }, "note": "Prices frozen at run time. Every derived cost in this receipt is computed from THIS snapshot, never from the live registry, so the receipt keeps its meaning when registry prices later change." }, "counts": { "judge_samples_per_generation": 5 } }, "results": { "cases": [ { "id": "local", "mode": "with_skill", "outcome": "pass", "score": 1, "mean": 1, "stddev": 0, "samples": [ 1, 1, 1, 1, 1 ], "generation_hash": "e78e4dd0410895dfba79bcb3c41fc87ab2f23ded3e168f92a9f7f76b1933383f", "judge_sample_hashes": [ "f8f601c9f8877181b721693438e01b7685817c8521d4f1e7d3ded55900e9de46", "d168283111580f889eabadb63171a3488622610f7a447544e12d51da4d36896a", "13070157580a5e5d7258563d10cfe061698b8edd3f0844e20656241d62177079", "a7cea7fd34e7681b8c99ccd94559841be14e68f0503cba3d7078cf26f3510c84", "af38024eb2566a450148a197d6e128807ab264fe4d0fd6080da1142d23a4b309" ], "threshold": 0.75, "reason": "Proposes the existing local markdown tracker, uses the single-context GLOSSARY.md and docs/adr layout, skips triage labels since triage is absent, and presents drafts before any files are written.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "6c3320a480a3acfb99cb36901c93a2947141c55ce52d5de9403eebb0745700e9" }, "judge_usage": { "input_tokens": 33713, "output_tokens": 1180, "cached_tokens": 30922, "wall_ms": 16488 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 1, "sd": 0, "judge_sd_mean": 0, "variance_ratio": null, "variance_ratio_unavailable": "judge_sd_zero", "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "5acf7ec43af7df9dcc2d75a014293b7209a00c3bf29df1ab79f0b80665b0c3a7", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "4bcacd2d15dbcdbc45270e91af51c4183e0733cd9bf6ac853124256aa908b046", "0195e9300499447e45ab42ec268f22fce4b9179f5d10d3c6f5dc41598cc649aa", "57fbd2e27dc71f881951e053d99f2be25ff54fecff661d738acf5c87b9173464", "d1ea166706de235a31f68a42a02ee78052a8f648d48756417a61ac04b32facf3", "96af9199707e187a1f97872f658a0173f2eef32b4ef6595379391357b74354d6" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4620, "output_tokens": 1210, "cached_tokens": 544, "wall_ms": 14102 }, "judge_usage": { "input_tokens": 33509, "output_tokens": 1178, "cached_tokens": 30758, "wall_ms": 18590 } }, { "draw_index": 1, "generation_hash": "c18b6b4ca7395f7a345a7de6f1236ec51636d9801981d87460e3d6bfc46dd64f", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "ef8fd33892291c79e3ee05551b2cede4ad9b903fa22c0692f2f3f2f9473550a7", "59ffa94c7a7ff6b7c6dc42715b4d95c6318b4a6c2d3c4bb63f0ea9183d863949", "c01789da3e61f971eeffcd5879ca1fd964a58c31c0de79e8b98050d91f8c4504", "c01789da3e61f971eeffcd5879ca1fd964a58c31c0de79e8b98050d91f8c4504", "d2986c8e840717c8905d8f1655055f6bbd30bd63653271fc082526cdb5b62944" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4620, "output_tokens": 1053, "cached_tokens": 4416, "wall_ms": 15767 }, "judge_usage": { "input_tokens": 32833, "output_tokens": 1468, "cached_tokens": 30218, "wall_ms": 18391 } }, { "draw_index": 2, "generation_hash": "e78e4dd0410895dfba79bcb3c41fc87ab2f23ded3e168f92a9f7f76b1933383f", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "f8f601c9f8877181b721693438e01b7685817c8521d4f1e7d3ded55900e9de46", "d168283111580f889eabadb63171a3488622610f7a447544e12d51da4d36896a", "13070157580a5e5d7258563d10cfe061698b8edd3f0844e20656241d62177079", "a7cea7fd34e7681b8c99ccd94559841be14e68f0503cba3d7078cf26f3510c84", "af38024eb2566a450148a197d6e128807ab264fe4d0fd6080da1142d23a4b309" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4621, "output_tokens": 1251, "cached_tokens": 4416, "wall_ms": 13831 }, "judge_usage": { "input_tokens": 33713, "output_tokens": 1180, "cached_tokens": 30922, "wall_ms": 16488 } } ], "issue408": { "attempt_id": "178358a690b1878459d930c59acd931b5841521d7d88bb02ae11611c56e9ea03", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "setup-matt-pocock-skills", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/setup-matt-pocock-skills/SKILL.md", "declared_name": "setup-matt-pocock-skills", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/setup-matt-pocock-skills/SKILL.md", "skill_blob_sha1": "293c25b75590119794af03b037ba4288054a32b3", "skill_sha256": "f148858ffb7f63c03c06fca06e6fdb1d91a35c6b293244b8fdfacd5f4cc028c4", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--setup-matt-pocock-skills.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:27:44.803Z", "configuration_unchanged": true }, "budget_cost_usd": 1.456865, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 6, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } }, { "id": "local", "mode": "baseline", "outcome": "borderline", "score": 0.807143, "mean": 0.807143, "stddev": 0.120515, "samples": [ 1, 1, 1, 0.75, 0.75 ], "generation_hash": "3196d9b14c0c1d5efcd6956309fbfdf921295ab3992977db17b4d93269404e37", "judge_sample_hashes": [ "8883bf056d13658fcb2f3aa6621ff5b7ae01e7f690ccdc413ce8c55d8ddbfbaa", "eaccb3e89f8fd9baa882eb2f514fcdf04acba3bd45df8674217e0775bbcabb78", "4bfdb75b9d6df35cf6f5a15c36755c72203c1b298f83c4570ad888d67c3cc994", "3518dc41a41ea840278982e5eaf00bcec7470c0aa51d24dc373ef2dc0385335c", "e1558f44ecaba9d1f2fa6809045da08e3ef0aef92458c0ccd20fbe3064fc65cc" ], "threshold": 0.75, "reason": "Keeps the existing local markdown tracker, uses the single-context domain layout, skips label mapping since triage is absent, and states nothing was changed while offering to write config later.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "6c3320a480a3acfb99cb36901c93a2947141c55ce52d5de9403eebb0745700e9" }, "judge_usage": { "input_tokens": 31531, "output_tokens": 1822, "cached_tokens": 29178, "wall_ms": 20728 }, "generation": { "n_planned": 3, "n_drawn": 7, "n_measured": 7, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.807143, "sd": 0.120515, "judge_sd_mean": 0.090628, "variance_ratio": 1.329777, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "9f7752a396f1941fcec8feaddf05917f5b19698747867a4021716663600e0d13", "status": "measured", "samples": [ 0.5, 0.5, 0.75, 0.5, 0.75 ], "judge_sample_hashes": [ "cab3495f6a916b45f17249caff1b3fa1fec770fed0e1cfc6fd19006dcf0f6879", "c6aa34e554133e310397f8d095aebd7ae319d19cdcfa9e219b23bc42de174c5f", "4f6d2a88f0249209e814ac87bfbaa608f94f00820658be05970888e7ffbe96b6", "f7ca04f92008092efd88e366fd49a14c2e1853bfe6dc18ad1c1d173135e141fd", "0223009fb67c4c6b5f6183339bd2463714a642740cf2e90cf10f7dfca9704b80" ], "mean": 0.6, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2021, "output_tokens": 1151, "cached_tokens": 1467, "wall_ms": 13895 }, "judge_usage": { "input_tokens": 32327, "output_tokens": 2569, "cached_tokens": 29814, "wall_ms": 26332 } }, { "draw_index": 1, "generation_hash": "d0a13dfa4b1bd452479a81aa006bba33f8978f1d670fb0b3f8ebf08961a7f5ae", "status": "measured", "samples": [ 1, 0.75, 1, 0.75, 1 ], "judge_sample_hashes": [ "5d699d2219e29f0309cfaf2ea0f4896ba908adb7a85c9bb14bbb75234a1709d9", "ca0b97dd20fe2dcc8001a5e5e09134c4f8fd32e485eb760cf30432052ec96e8a", "fc90209ce3454c88ce2f8506191f064c025a7f718c3ec64679ff5f908f7b4028", "34314d203a10c8b07c9b390cc47f7f3256ab491b9daf62c9ed48d86f5552e2a0", "88801218e65ec99ab7ee54b8f1510911ef7a5f191409b6d8380e03871a392f56" ], "mean": 0.9, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2022, "output_tokens": 1088, "cached_tokens": 1817, "wall_ms": 13577 }, "judge_usage": { "input_tokens": 31859, "output_tokens": 3202, "cached_tokens": 29438, "wall_ms": 26000 } }, { "draw_index": 2, "generation_hash": "fe5d3ea8756fca0425f4bf759e4bdba959c4c3be313e2a9ac8a504b0c5f6ef41", "status": "measured", "samples": [ 0.75, 1, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "35ddd595c331a6114f2a2dda5600b7c28faa4ea81bef3bc2eaa322979c0efc40", "df35018e05424f97d3a2db842de5c0efeae7b20f10dbf4ae9d976b45c87a6a65", "65bfb0b0cfb9fa162acc04df8fce1f518889d97d187930d0dd159ceb630b1b28", "a1bee4b349ed95402c498be138db893541f3d7e7ce5e61ce30b14734eebb0cfb", "75febb467efbd13d937665056aec2e72770460eb521e8fe7de5ce5cde15f9239" ], "mean": 0.8, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2022, "output_tokens": 1015, "cached_tokens": 1817, "wall_ms": 13040 }, "judge_usage": { "input_tokens": 32293, "output_tokens": 1178, "cached_tokens": 29786, "wall_ms": 17873 } }, { "draw_index": 3, "generation_hash": "71315ae6d172b7488a4372dbd2b368254caf3b337cb748f452ada0a75141cc2d", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "ea1dd3969cba2b5ab7b091735d408621fc05576ea52a7715f2827e3e400d09f4", "81365bae9287225c8550a95d32456d0aab05f2f295ada1a894904fea351c8601", "6e98d84e26cc6bf067bf6ba77ecda99a24dd7f64018bd7615134f65aac256207", "bffbaaad1cbddb58ffd556649f7c2033b8320afa104e7c0d03496c46a36234f2", "faccf551eca06cf18e75bb35f63bf68fc0cee66a1c8a3c1c4adba5320ff10758" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2021, "output_tokens": 968, "cached_tokens": 1817, "wall_ms": 14458 }, "judge_usage": { "input_tokens": 31966, "output_tokens": 2324, "cached_tokens": 29522, "wall_ms": 22447 } }, { "draw_index": 4, "generation_hash": "3e45253cbfabbe612942db9a76723f6c6c3b0df62fe07d468d944f2b98a6da87", "status": "measured", "samples": [ 0.75, 1, 1, 1, 1 ], "judge_sample_hashes": [ "8177884f2770932db3a59bdf407decd078ca459b05978cd39e6be43f57af9c18", "7cc8c8f9dc7c6a7325030243097c50cc611f885e814c89d121b718390ac498b7", "2d1010556916ff8a7b9731a12a1732baf2a2c43bf2dafc7ccf99ae05a9d29691", "81d2c94a3f3934d149dacc4efa7485b3ed86f01e5462842244163acd1e814d13", "0d2dffa1d59b9ad7858a99f5aa3b2b4c0f71519f92a6eb968ffb229153cbddea" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2021, "output_tokens": 869, "cached_tokens": 1817, "wall_ms": 11376 }, "judge_usage": { "input_tokens": 31379, "output_tokens": 976, "cached_tokens": 29054, "wall_ms": 16393 } }, { "draw_index": 5, "generation_hash": "f6ae93ddbb2339538f4f04f8d0a5d3cf13eaca24bb9142fd63840ab72f6dc135", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "bc8fdf5941e1744b0a0924f31f056c8eca2f3a66508e89c861a8ea9f83cacdf3", "770ff2c916ac69e16333d2d810bbb6ffd0b6143f92d8c130a5b8b172dab51267", "cc966bcb8bd75aadfa337743333bfb39eb2fb792efd24d9701f08e6e6a1d4e6e", "232385a604e192d23ddf9921a19fcc4de47c94b5c6c4acbafd79c889c2f6c53f", "1d57379ecfafca0241ba3d0c1d241a9fbd3412e472bacb520546570f6f9006cb" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2022, "output_tokens": 950, "cached_tokens": 1817, "wall_ms": 29557 }, "judge_usage": { "input_tokens": 32081, "output_tokens": 2562, "cached_tokens": 29614, "wall_ms": 24227 } }, { "draw_index": 6, "generation_hash": "3196d9b14c0c1d5efcd6956309fbfdf921295ab3992977db17b4d93269404e37", "status": "measured", "samples": [ 1, 1, 1, 0.75, 0.75 ], "judge_sample_hashes": [ "8883bf056d13658fcb2f3aa6621ff5b7ae01e7f690ccdc413ce8c55d8ddbfbaa", "eaccb3e89f8fd9baa882eb2f514fcdf04acba3bd45df8674217e0775bbcabb78", "4bfdb75b9d6df35cf6f5a15c36755c72203c1b298f83c4570ad888d67c3cc994", "3518dc41a41ea840278982e5eaf00bcec7470c0aa51d24dc373ef2dc0385335c", "e1558f44ecaba9d1f2fa6809045da08e3ef0aef92458c0ccd20fbe3064fc65cc" ], "mean": 0.9, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2022, "output_tokens": 799, "cached_tokens": 1817, "wall_ms": 14872 }, "judge_usage": { "input_tokens": 31531, "output_tokens": 1822, "cached_tokens": 29178, "wall_ms": 20728 } } ], "issue408": { "attempt_id": "178358a690b1878459d930c59acd931b5841521d7d88bb02ae11611c56e9ea03", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "setup-matt-pocock-skills", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/setup-matt-pocock-skills/SKILL.md", "declared_name": "setup-matt-pocock-skills", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/setup-matt-pocock-skills/SKILL.md", "skill_blob_sha1": "293c25b75590119794af03b037ba4288054a32b3", "skill_sha256": "f148858ffb7f63c03c06fca06e6fdb1d91a35c6b293244b8fdfacd5f4cc028c4", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--setup-matt-pocock-skills.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:27:44.803Z", "configuration_unchanged": true }, "budget_cost_usd": 1.456865, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 6, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 7 } }, { "id": "labels", "mode": "with_skill", "outcome": "borderline", "score": 0.69, "mean": 0.69, "stddev": 0.096177, "samples": [ 0.5, 0.5, 0.75, 0.5, 0.75 ], "generation_hash": "a8d1fbef3bc69a822fe24312de31682a4e05a5dd7f469b568ab48258c8a249b6", "judge_sample_hashes": [ "e11ccf28e27102a7116f79f1a15a2328c147f6007983f3a3de7962a839988c0e", "234a690735f765e86919b7d9861ea06722a779164a81814f590189a940206977", "21131d58b2f66463c59f1cc79bd364b6cc41e9a09c0f6bb21983c3873320d1f0", "bb906eb2b47d6cb3c59129fb1fc5b68c666c4202187de46af70b0dcff1982c8b", "6e92ed9ebbfd55cb2031020637cdf2d7fa819eff32674e8c0ebb0d0010ba8493" ], "threshold": 0.75, "reason": "Criteria 1 (maps bug:triage/bug:ready to canonical roles) and 3 (preserves label strings) and 4 (no duplicate defaults) are met, but it does not ask about unmapped roles and instead fills in defaults for them, missing criterion 2.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "7341d9aa25b6e7181025cd52ea58abec14b068a2d363443817e8c1084726699c" }, "judge_usage": { "input_tokens": 32910, "output_tokens": 1418, "cached_tokens": 30278, "wall_ms": 18927 }, "generation": { "n_planned": 3, "n_drawn": 5, "n_measured": 5, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.69, "sd": 0.096177, "judge_sd_mean": 0.131905, "variance_ratio": 0.729138, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "be494df3bd6effd7e988a66eb2b247aee18d3bdd69de2d0233f4fc4822cd7c26", "status": "measured", "samples": [ 0.75, 0.5, 0.75, 0.5, 0.75 ], "judge_sample_hashes": [ "202d72f71ae9afc672acf918fcd0ffe0fd214427696de44a8a964bdd361695da", "2841af0f957243b3cba49a901b906b98184fb556d2e7b48ac6af55309d2cf30a", "3c85fb5b90e86d727233b843cb28b5289bc0f7a13e7dba7075e55df68c6580e2", "c7881fb53e06045ec66346826ea50785c81794ed43804b7d1f78dd1d7ac50ceb", "2cef1fd58788f113c4eb836b094254126bdc30f89fd6b3a6e6efee3feb0e93e1" ], "mean": 0.65, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4634, "output_tokens": 1176, "cached_tokens": 4066, "wall_ms": 13047 }, "judge_usage": { "input_tokens": 33380, "output_tokens": 2160, "cached_tokens": 30654, "wall_ms": 22680 } }, { "draw_index": 1, "generation_hash": "044f7d5b65cfc9e80d98511e94b7bdfd0f0b0842b555b470416d2b85c327575a", "status": "measured", "samples": [ 0.75, 0.75, 0.5, 0.5, 0.75 ], "judge_sample_hashes": [ "ddd584396f8c9e1559ca5c5068163c42fdccbfbea6b79ef0ed4ac8e08632aa89", "b1ad5992be1c585074302b8744caa9c81fbf87148b5ef150176acde7ea733134", "19543ea4cb58d25bd3d9223f027a43b09f803c95b77420905d4f3661a49ace4f", "9ea76d505ac03a724b8bdafaa0acd7c5ab67f1d84591c094434ce259a1fda79c", "255283904879d334a58278a7094013298ae561c25bf127ff61170791f0d07e73" ], "mean": 0.65, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4633, "output_tokens": 1032, "cached_tokens": 4428, "wall_ms": 12817 }, "judge_usage": { "input_tokens": 32608, "output_tokens": 2176, "cached_tokens": 30038, "wall_ms": 23001 } }, { "draw_index": 2, "generation_hash": "44364f219ad24994ac2f023e70d9caba5355a1c95a437dbfffc8097cf3127ecf", "status": "measured", "samples": [ 1, 0.75, 1, 0.75, 0.75 ], "judge_sample_hashes": [ "b7a64816e13f5ca85702b9978929322662d98bdb08dd2a33f43c7f9cb2f146d9", "24cd7641c8275f083bf7beb96b8bf3437dfdfb7c90c9dc05d1653705ee55ffdc", "9c3d7f71f2156b00771c7aab375f3394f828f5413ecb040cc4ed4e1eee8ef1f7", "99fcb3eae63fd3fd812684d82a25cb9f569bf27b1b77b39df7deebff775e4340", "d0ecff1753dc169b588d9f4e48149e8c9c8ef1ca93333655d4127eb2e87800cb" ], "mean": 0.85, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4632, "output_tokens": 1221, "cached_tokens": 4428, "wall_ms": 16170 }, "judge_usage": { "input_tokens": 33318, "output_tokens": 3783, "cached_tokens": 30606, "wall_ms": 28006 } }, { "draw_index": 3, "generation_hash": "63ac46ded7cc8eaf57d3043fa20f4775f00a1aa78b2710a24d6d3df832bffe25", "status": "measured", "samples": [ 0.5, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "9619d295a0c342b7d7c1c6d30bf421a6d02c59aaea97c8d002dd0999f01660cf", "72ea11f84c0998723197d39dc7f10ad9f674fdc02fec808abc53283822cd8c9e", "a384e8d6897f0390c9916a3ed4beadc8219c54119b0c92d48b4bca9bf5997aba", "71f6a12ae062bdeeb7e441d54e4912fb5a4cb507a2986b9df76d4d3ac35ee8b3", "3888cfd76e72983972709ee6d62b2f5f5c8b789f03b595374a73694e2b9a4f62" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4633, "output_tokens": 1117, "cached_tokens": 4428, "wall_ms": 13027 }, "judge_usage": { "input_tokens": 32645, "output_tokens": 2695, "cached_tokens": 30070, "wall_ms": 23332 } }, { "draw_index": 4, "generation_hash": "a8d1fbef3bc69a822fe24312de31682a4e05a5dd7f469b568ab48258c8a249b6", "status": "measured", "samples": [ 0.5, 0.5, 0.75, 0.5, 0.75 ], "judge_sample_hashes": [ "e11ccf28e27102a7116f79f1a15a2328c147f6007983f3a3de7962a839988c0e", "234a690735f765e86919b7d9861ea06722a779164a81814f590189a940206977", "21131d58b2f66463c59f1cc79bd364b6cc41e9a09c0f6bb21983c3873320d1f0", "bb906eb2b47d6cb3c59129fb1fc5b68c666c4202187de46af70b0dcff1982c8b", "6e92ed9ebbfd55cb2031020637cdf2d7fa819eff32674e8c0ebb0d0010ba8493" ], "mean": 0.6, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4632, "output_tokens": 1180, "cached_tokens": 4428, "wall_ms": 12934 }, "judge_usage": { "input_tokens": 32910, "output_tokens": 1418, "cached_tokens": 30278, "wall_ms": 18927 } } ], "issue408": { "attempt_id": "178358a690b1878459d930c59acd931b5841521d7d88bb02ae11611c56e9ea03", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "setup-matt-pocock-skills", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/setup-matt-pocock-skills/SKILL.md", "declared_name": "setup-matt-pocock-skills", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/setup-matt-pocock-skills/SKILL.md", "skill_blob_sha1": "293c25b75590119794af03b037ba4288054a32b3", "skill_sha256": "f148858ffb7f63c03c06fca06e6fdb1d91a35c6b293244b8fdfacd5f4cc028c4", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--setup-matt-pocock-skills.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:27:44.803Z", "configuration_unchanged": true }, "budget_cost_usd": 1.456865, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 6, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 5 } }, { "id": "labels", "mode": "baseline", "outcome": "borderline", "score": 0.7125, "mean": 0.7125, "stddev": 0.047871, "samples": [ 0.75, 0.75, 0.75, 0.75, 0.5 ], "generation_hash": "7ebb5bbaa06028e329db8077f0a0c7d85ed5a7f88d9cb2c0c21cb03ae787e9c1", "judge_sample_hashes": [ "f3796527f0e63f638b82e9ae0b848d04cd480c2470474649139176d0110aa6d8", "e43d646f0bc0ca68544405431ffcf0493bdf551d648d516f466eac7114a3f0b4", "2a69d70e670440cad4f313711fdd4de6623afab8c11176a5673b92c22df6e8e4", "6eb7b16ff8d1bab4261b7abd01c2b06ce0e3eacc239d51d0feb9767419284a42", "7cd95a88591e5bb24c75f61b98daa7e87d37188da2aaa34fd374be6d2d8fda50" ], "threshold": 0.75, "reason": "Meets criteria 1, 3, and 4: it maps triage and ready to the existing bug: labels, preserves exact label strings, and disables label creation to avoid duplicates. Misses criterion 2, since it never asks about unmapped roles and instead offers a mapping for only the two named roles.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "7341d9aa25b6e7181025cd52ea58abec14b068a2d363443817e8c1084726699c" }, "judge_usage": { "input_tokens": 32418, "output_tokens": 1758, "cached_tokens": 29886, "wall_ms": 21086 }, "generation": { "n_planned": 3, "n_drawn": 4, "n_measured": 4, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.7125, "sd": 0.047871, "judge_sd_mean": 0.062184, "variance_ratio": 0.769828, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "f531352a8b494b86c0de9528bace50223a3c754c53951b180c546c55b997900e", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "6e6cf4ef28718f82217bbc4f758eb0dfe7ac2843bfd783ebfcd0af515cb3a0f9", "446e21b5c08adf5683808a3bce6a7944d313370d8326339498793c162c6b261c", "ec8b45b3bd2575a32a46a86bf85b56773b0e64600402e101596d85fdfa6ee5e8", "35d441c8d5cab029cd2258c00a1f80742c8fe266105737587ce4309408e89937", "3c8794bcf881b409db114ddaa3aae62f5364d80d33ce39045e47159d36ef06cd" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2033, "output_tokens": 987, "cached_tokens": 1467, "wall_ms": 12916 }, "judge_usage": { "input_tokens": 32467, "output_tokens": 2227, "cached_tokens": 29926, "wall_ms": 21222 } }, { "draw_index": 1, "generation_hash": "f832570c68937ae272b52d2e359aa14374f30e4abeb94f14c6352db70c647bff", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.5, 0.5 ], "judge_sample_hashes": [ "e17a09a9566c14ecdd8398c1a824cfd7f0ecb8cff47e36b00f32edfc29ae0465", "1cf73efe222109df4f73f3c4ef335c57a60a751195829e48b26453798d0ac188", "9a56edca8e5f89963bb5b657b420aeb09594730d6860a3491fa061e6859d3182", "60e38e1f835692341406c5e52a6e996325d71461adfc8ac03247083bc4db9e5d", "6288bf0c8603de42dced2fbf7fa1ca626bf5cc885d2f791f0cddf500dbfe884a" ], "mean": 0.65, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2033, "output_tokens": 911, "cached_tokens": 1829, "wall_ms": 11925 }, "judge_usage": { "input_tokens": 32138, "output_tokens": 1426, "cached_tokens": 29662, "wall_ms": 17897 } }, { "draw_index": 2, "generation_hash": "63f723da8c38f541d55bd0d17537b06a446f1067fcd8441f253454334e8c050b", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "041c35637d66385c0ede1ef8edde77cf70b60952b3a6f27fc688aff1268a5532", "79d83b098278f5e16e31092d0a215695eed3774fb80dd6ce58cac6a0c5185d1c", "4302bc85d537f6aa816afac7c46ceb7b863f5efad1772b711a8842fc50259d34", "c29b2e0de320e5890e297286921d2ec44b0f75e77b6562aab7c780283ba860fb", "2edeae9a54c5cc7e9374b6ed96a1486619c5d07dc151fc50ce41e2c0231eabf8" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2034, "output_tokens": 995, "cached_tokens": 1829, "wall_ms": 13372 }, "judge_usage": { "input_tokens": 32563, "output_tokens": 2260, "cached_tokens": 30002, "wall_ms": 20941 } }, { "draw_index": 3, "generation_hash": "7ebb5bbaa06028e329db8077f0a0c7d85ed5a7f88d9cb2c0c21cb03ae787e9c1", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.5 ], "judge_sample_hashes": [ "f3796527f0e63f638b82e9ae0b848d04cd480c2470474649139176d0110aa6d8", "e43d646f0bc0ca68544405431ffcf0493bdf551d648d516f466eac7114a3f0b4", "2a69d70e670440cad4f313711fdd4de6623afab8c11176a5673b92c22df6e8e4", "6eb7b16ff8d1bab4261b7abd01c2b06ce0e3eacc239d51d0feb9767419284a42", "7cd95a88591e5bb24c75f61b98daa7e87d37188da2aaa34fd374be6d2d8fda50" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2034, "output_tokens": 958, "cached_tokens": 1829, "wall_ms": 14125 }, "judge_usage": { "input_tokens": 32418, "output_tokens": 1758, "cached_tokens": 29886, "wall_ms": 21086 } } ], "issue408": { "attempt_id": "178358a690b1878459d930c59acd931b5841521d7d88bb02ae11611c56e9ea03", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "setup-matt-pocock-skills", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/setup-matt-pocock-skills/SKILL.md", "declared_name": "setup-matt-pocock-skills", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/setup-matt-pocock-skills/SKILL.md", "skill_blob_sha1": "293c25b75590119794af03b037ba4288054a32b3", "skill_sha256": "f148858ffb7f63c03c06fca06e6fdb1d91a35c6b293244b8fdfacd5f4cc028c4", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--setup-matt-pocock-skills.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:27:44.803Z", "configuration_unchanged": true }, "budget_cost_usd": 1.456865, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 6, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 4 } }, { "id": "monorepo", "mode": "with_skill", "outcome": "pass", "score": 1, "mean": 1, "stddev": 0, "samples": [ 1, 1, 1, 1, 1 ], "generation_hash": "c2d065c18ac036544a8c594ba7ff4e25f6db59e6a242898a01596b213ad4a0fb", "judge_sample_hashes": [ "dde6eaffca3271ed8bd585a3bce0a2c45a893bbb65ea0fb146b07204919fb9af", "4e88040120418cb695e6659772970a5699f52e11575014211945c26db21c72af", "9c8ff749ed9637a85626571b47bcc07c6490ad2bfcf38377a0f27ee35ec972a5", "7c643dadd40023b4ae69dab9566158aa775509e08ff4dc3bce8580dff03bbbde", "d9cf99071dbe742b4000381d3f7fef8782ba06bf50b445d8628953b6d07c78ee" ], "threshold": 0.75, "reason": "Cites each package's own glossary and ADR folder as multi-context evidence, proposes a root GLOSSARY-MAP.md to those contexts, keeps existing locations, and asks for confirmation before writing anything.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "e92c9ec08a1d09ee962d1945f3fc63a438da2b1325fbf83d1c04f71513c012fc" }, "judge_usage": { "input_tokens": 33157, "output_tokens": 2500, "cached_tokens": 30478, "wall_ms": 23382 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 1, "sd": 0, "judge_sd_mean": 0, "variance_ratio": null, "variance_ratio_unavailable": "judge_sd_zero", "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "1ceb5958b52a36ff051b6a503bbe0a551ffeed7a8489ecc4528b7ec7dc6c69a9", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "359b1ae4d944fb7d654f4e7ef7bf7c360e6677a92c20530b2a7b559ea7949441", "865b5a40224fb8624bb6f06207ff6eae24d42e0c8a94a6e25eff3150d7b3d135", "76695ab260d28bf5531832c515316cce3c24904b91d882489235cfe549bbb644", "9e79372a55c5dc2c62f65be687f4dbbcd2c3de52ffc18fab7540bfbccc153bdb", "c6a2d5849bedf56e15465467169f26b558cb6d506d3665f3c9cdd75f135cf14e" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4627, "output_tokens": 923, "cached_tokens": 4066, "wall_ms": 11185 }, "judge_usage": { "input_tokens": 32067, "output_tokens": 2032, "cached_tokens": 29606, "wall_ms": 20458 } }, { "draw_index": 1, "generation_hash": "19390fcec7fb7c18aaa0ed23d3ddc18a3f7107f16e7bb08e712e6ca5c6133032", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "6741f97b3aa7bc62cd3f9ea6999e07e94f6d203b22d6c6ae360d60fd21b2a938", "66001164c4be5521bbbfb0cd2febb6b8de78bfeaf5a7af2adceb6ffe100c726d", "d0d916368ed698ace16b2d55c0d0c347987a87b51343dd501628665c5a6a28de", "bd0d79743bf22d1d609ccbec735f831a2ef8961968e39d4da6f1b68fedb4dae7", "a315f4a61ad412a8a77e99f716ce91fae556a4a93f02e5d5ed8b813f47b84f17" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4626, "output_tokens": 1159, "cached_tokens": 4422, "wall_ms": 14665 }, "judge_usage": { "input_tokens": 32804, "output_tokens": 1202, "cached_tokens": 30194, "wall_ms": 16340 } }, { "draw_index": 2, "generation_hash": "c2d065c18ac036544a8c594ba7ff4e25f6db59e6a242898a01596b213ad4a0fb", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "dde6eaffca3271ed8bd585a3bce0a2c45a893bbb65ea0fb146b07204919fb9af", "4e88040120418cb695e6659772970a5699f52e11575014211945c26db21c72af", "9c8ff749ed9637a85626571b47bcc07c6490ad2bfcf38377a0f27ee35ec972a5", "7c643dadd40023b4ae69dab9566158aa775509e08ff4dc3bce8580dff03bbbde", "d9cf99071dbe742b4000381d3f7fef8782ba06bf50b445d8628953b6d07c78ee" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4627, "output_tokens": 1214, "cached_tokens": 4422, "wall_ms": 13618 }, "judge_usage": { "input_tokens": 33157, "output_tokens": 2500, "cached_tokens": 30478, "wall_ms": 23382 } } ], "issue408": { "attempt_id": "178358a690b1878459d930c59acd931b5841521d7d88bb02ae11611c56e9ea03", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "setup-matt-pocock-skills", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/setup-matt-pocock-skills/SKILL.md", "declared_name": "setup-matt-pocock-skills", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/setup-matt-pocock-skills/SKILL.md", "skill_blob_sha1": "293c25b75590119794af03b037ba4288054a32b3", "skill_sha256": "f148858ffb7f63c03c06fca06e6fdb1d91a35c6b293244b8fdfacd5f4cc028c4", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--setup-matt-pocock-skills.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:27:44.803Z", "configuration_unchanged": true }, "budget_cost_usd": 1.456865, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 6, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } }, { "id": "monorepo", "mode": "baseline", "outcome": "borderline", "score": 0.8, "mean": 0.8, "stddev": 0.061237, "samples": [ 0.75, 0.5, 1, 1, 0.75 ], "generation_hash": "46a164d8d9506cdf46dfc68e23dc76c27762111dc317e3691829c50a6fa5076e", "judge_sample_hashes": [ "3cd85b3d81bce418c41376452addc9d7b68fd9dc2a2a60f4805c9cec8b599e6b", "af66742f8b9ce62d40c591e5c9d096a564258cf32b71167c6d2fa016b2fd6fdd", "0932c7ef9f70e5403966c00bbd8593a6b9c8f00561adadccbe51ff723d2da156", "1d6ad7922aad002bf3fd77191cf4758480ceb6f62ad3d51e045ec7e97d1928ba", "6aef0386392a8d22ebda610d875ffb81219cef4b98c715cebd073bd8857e60e7" ], "threshold": 0.75, "reason": "Met criteria 1, 3, and 4: it treats Orders and Catalog as separate contexts, keeps existing glossary and ADR locations, and asks for confirmation before writing. Criterion 2 is debatable because it rejects a merged root glossary, though CONTEXT-MAP.md maps to the existing contexts.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "e92c9ec08a1d09ee962d1945f3fc63a438da2b1325fbf83d1c04f71513c012fc" }, "judge_usage": { "input_tokens": 33526, "output_tokens": 3033, "cached_tokens": 30774, "wall_ms": 25058 }, "generation": { "n_planned": 3, "n_drawn": 5, "n_measured": 5, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.8, "sd": 0.061237, "judge_sd_mean": 0.09158, "variance_ratio": 0.668672, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "1f787a45670438f7b41aa909d0ace773bd8924ac25dcd5bf57d9b7ca6f567baf", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "cdaae8d0c32ea662d5961ac8ef3af83a20ddcf6ecf1b12707bae15eae1865b0a", "9ceec30b4e40b063e58090dea63053d9e54c00f355e68a872743942dd0105447", "8a54437524a0b68ed1320b50b9a9508520898c630091816dd5d087d8317b67ad", "45f214e8fc3ac5b40d9666149b1a14e7275c5f0899204ab99e37651a446be431", "4d6dfd58bb4f4a7670235a737fb71eed9d23b424f26d77f72cd708f4fddf28d7" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2028, "output_tokens": 1149, "cached_tokens": 1467, "wall_ms": 19674 }, "judge_usage": { "input_tokens": 33011, "output_tokens": 4784, "cached_tokens": 30358, "wall_ms": 31964 } }, { "draw_index": 1, "generation_hash": "dca866715992b01118c89dcff164bdff2dec753ee05d6ad85d22773e04211925", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "2dc099d04bea73f49c09c475b1015bf5bcbc03c73cfae8b04f8312e0810ba5b2", "5ff6d84081a82415c0d28401e35abae0d7cfab21a6265a6de457563acce34f07", "068a7fe7619522605bf42adba4dbe9acee90d6c3c7a0bf208ae80351f39eae4c", "e6b0e1ad4bb40be1a016facf8f4e38b25ed287386cc9b392d4c18f0224c428be", "d426d69ca67894ebbf3bc4e06c91b313954502a3da3f52a4762bbbce4504f7dc" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2027, "output_tokens": 1167, "cached_tokens": 1823, "wall_ms": 14540 }, "judge_usage": { "input_tokens": 33095, "output_tokens": 3696, "cached_tokens": 30426, "wall_ms": 28053 } }, { "draw_index": 2, "generation_hash": "dffe0643d30b3571eb3237caf96b0a8f89b32b79f5045b49d0e8fa1129928417", "status": "measured", "samples": [ 1, 0.75, 1, 1, 0.75 ], "judge_sample_hashes": [ "b830c266b65dc527ef9e17b6d746ad124919f259d3e2c0541e830d9f959ee1a4", "6b32e1127f6ae591d9c29e2870f84d2313c0d88a35d2989c505ce26ae04b2fb2", "95d38897d47d861d630fe4b0f6a04be81628cb2939aa1d72868899dc601e9faf", "6f5bb2f39ce5732cddaab566e6ad5139ce38e357725d4ca179c364eb4328184a", "40ae14bbee34ad4b1f674d98a7d122bab41c3e9fdf65baecbdb1f91094558fa7" ], "mean": 0.9, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2028, "output_tokens": 1208, "cached_tokens": 1823, "wall_ms": 15635 }, "judge_usage": { "input_tokens": 33548, "output_tokens": 4874, "cached_tokens": 30786, "wall_ms": 32649 } }, { "draw_index": 3, "generation_hash": "86edbee2b49eda4f37ac6f37b93f05ebd410fc1026f759038b4def6d740cb5b4", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 1, 0.75 ], "judge_sample_hashes": [ "ca25fb231531216ea9773af5bdcce226eac63a2ec52ca145bd2b7001fc52c43c", "48fe8b61e8881e934d3144e4c56e86d327d37842f3d0df27f80f90246c8a1cb3", "272c36347e7872e5cf042c705aa4bae2607f73b8cf60ee58ea0c0f9d03de8a90", "d058198345ecc7e91415128ec3e76fc59f890ac2d748e2d722beff1f2f4bf8fe", "b254dc338bf1f876e3ac58b4fecde72a123f9e6b4ca51e1b5c73682973e291c9" ], "mean": 0.8, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2028, "output_tokens": 1077, "cached_tokens": 1823, "wall_ms": 13269 }, "judge_usage": { "input_tokens": 32490, "output_tokens": 5048, "cached_tokens": 29946, "wall_ms": 33362 } }, { "draw_index": 4, "generation_hash": "46a164d8d9506cdf46dfc68e23dc76c27762111dc317e3691829c50a6fa5076e", "status": "measured", "samples": [ 0.75, 0.5, 1, 1, 0.75 ], "judge_sample_hashes": [ "3cd85b3d81bce418c41376452addc9d7b68fd9dc2a2a60f4805c9cec8b599e6b", "af66742f8b9ce62d40c591e5c9d096a564258cf32b71167c6d2fa016b2fd6fdd", "0932c7ef9f70e5403966c00bbd8593a6b9c8f00561adadccbe51ff723d2da156", "1d6ad7922aad002bf3fd77191cf4758480ceb6f62ad3d51e045ec7e97d1928ba", "6aef0386392a8d22ebda610d875ffb81219cef4b98c715cebd073bd8857e60e7" ], "mean": 0.8, "stddev": 0.209165, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2028, "output_tokens": 1229, "cached_tokens": 1823, "wall_ms": 14350 }, "judge_usage": { "input_tokens": 33526, "output_tokens": 3033, "cached_tokens": 30774, "wall_ms": 25058 } } ], "issue408": { "attempt_id": "178358a690b1878459d930c59acd931b5841521d7d88bb02ae11611c56e9ea03", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "setup-matt-pocock-skills", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/setup-matt-pocock-skills/SKILL.md", "declared_name": "setup-matt-pocock-skills", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/setup-matt-pocock-skills/SKILL.md", "skill_blob_sha1": "293c25b75590119794af03b037ba4288054a32b3", "skill_sha256": "f148858ffb7f63c03c06fca06e6fdb1d91a35c6b293244b8fdfacd5f4cc028c4", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--setup-matt-pocock-skills.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:27:44.803Z", "configuration_unchanged": true }, "budget_cost_usd": 1.456865, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 6, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 5 } } ], "aggregates": { "with_skill": { "case_count": 3, "pass_count": 2, "borderline_count": 1, "mean_score": 0.896667, "stddev": 0.178979 }, "baseline": { "case_count": 3, "pass_count": 0, "borderline_count": 3, "mean_score": 0.773214, "stddev": 0.052701 }, "band_rule": "per arm: the sample standard deviation (n-1) of the per-case means across the included cases; the comparison band is the quadrature sum of the two arms; null where an arm has fewer than two included cases" } }, "comparison": { "with_skill_score": 0.896667, "baseline_score": 0.773214, "delta": 0.123453, "delta_uncertainty": 0.186577 }, "verification_level": "TESTED", "receipt_hash": "a6387128d860520217327680b2b121a6002ff72a6eda508807526fb9cfbf39a9", "economics": { "basis": "metered-equivalent", "surface": "claude-cli", "with_skill": { "call_count": 11, "mean_input_tokens": 4627.73, "mean_output_tokens": 1139.64, "mean_cost_usd_per_call": 0.041304, "median_wall_ms": 13618, "wall_ms_p25": 12934, "wall_ms_p75": 14665, "wall_ms_iqr": 1731 }, "baseline": { "call_count": 16, "mean_input_tokens": 2026.5, "mean_output_tokens": 1032.56, "mean_cost_usd_per_call": 0.028757, "median_wall_ms": 14010, "wall_ms_p25": 13154.5, "wall_ms_p75": 14706, "wall_ms_iqr": 1551.5 }, "skill_incremental_cost_usd_per_call": 0.012547, "skill_incremental_cost_usd_per_1k_calls": 12.547, "output_tokens_delta": 107.08, "median_wall_ms_delta": -392, "judge_excluded": true, "judge_overhead": { "note": "Measurement overhead imposed by Driftproof, NOT a cost of running the skill. Excluded from every field above. Counted over every draw sent to the judge, each once: the usage its answered judge samples reported, at the frozen judge price. Retried attempts and samples that reported no usage are not in it.", "total_cost_usd": 0.120929, "case_rows_measured": 6, "judged_draws_counted": 27, "judged_draws_missing_usage": 0 }, "notes": { "absolute_cost": "Absolute per-call cost on a CLI surface includes a fixed harness preamble (observed ~25k input tokens on claude-cli, ~11k on codex) that we do not control. It is identical in the with-skill and baseline arms, so it cancels in the Δ figures — read the incremental columns, not the absolute ones.", "cache_pricing": "Cached input tokens are costed at the list input rate (an over-estimate where caching is heavy); cached_tokens is recorded per call so a reader can recompute.", "latency": "observed on subscription CLI surface, indicative" } } }