{ "schema_version": "0.13", "generation_sampled": true, "skill": { "name": "setup-matt-pocock-skills", "version": "0.0.0", "content_hash": "f11021a04851b50f14fb1584bb974c9eb44fe0c81027df4f7d04c25deff52771", "tokens": 1745 }, "suite": { "format": "agentskills.io/evals", "suite_hash": "a11911f9dcca438ccbffe323bcb319823ffe09aab693961e8cd07c48779c3cdd", "case_count": 3, "canary": "2eaaef5c-0ec2-72d5-72fd-bae7e65d80bd" }, "run": { "model_id": "claude-sonnet-5-5", "model_release_date": null, "provider": "anthropic", "surface": "claude-cli", "runner_version": "0.22.2", "date_utc": "2026-10-18T06:31:35.010Z", "generated_at": "2026-10-18T06:15:23.051Z", "registry": "registered", "transcripts": "retained-local", "judge": { "samples": 5, "temperature": null, "sampling": "surface-controlled", "surface": "claude-cli", "model_id": "claude-haiku-5-5", "prompt_template_hash": "82586d1e44f84df2b8ce5f18ac777866465cb5ad9686503c93d9776c93c344ed", "harness": { "name": "claude-code", "version": "2.1.287" }, "isolation": "eval-user", "endpoint": { "custom": false, "sha256": null, "resolved_by": "cli", "routing": { "ANTHROPIC_BEDROCK_BASE_URL": { "set": false, "sha256": null }, "ANTHROPIC_VERTEX_BASE_URL": { "set": false, "sha256": null }, "CLAUDE_CODE_USE_BEDROCK": { "set": false, "sha256": null }, "CLAUDE_CODE_USE_VERTEX": { "set": false, "sha256": null }, "HTTPS_PROXY": { "set": false, "sha256": null } } }, "code_revision": "judge-code-bb552899a1df887e", "reported_models": [ "claude-haiku-5-5" ] }, "answered_by": { "kind": "model", "attested": true, "reported_model": "claude-sonnet-5-5", "reported_models": [ "claude-haiku-5-5", "claude-sonnet-5-5" ], "isolation": "eval-user" }, "capture": { "mode": "text" }, "harness": { "name": "claude-code", "version": "2.1.287" }, "pricing_snapshot": { "frozen_at": "2026-10-18T06:31:35.010Z", "source": "config/models.json", "currency": "USD", "models": { "claude-sonnet-5-5": { "input_per_mtok": 2, "output_per_mtok": 10, "registered": true }, "claude-haiku-5-5": { "input_per_mtok": 0.1, "output_per_mtok": 0.5, "registered": true } }, "note": "Prices frozen at run time. Every derived cost in this receipt is computed from THIS snapshot, never from the live registry, so the receipt keeps its meaning when registry prices later change." }, "counts": { "judge_samples_per_generation": 5 } }, "results": { "cases": [ { "id": "local", "mode": "with_skill", "outcome": "pass", "score": 1, "mean": 1, "stddev": 0, "samples": [ 1, 1, 1, 1, 1 ], "generation_hash": "ef00d007b02310e35f4860328b76aaea765cb0a472e917138ae9370d686108d4", "judge_sample_hashes": [ "42551ec6c63bfddd5ef3e76438a1cddeda7100ad93139e1bf103c7cc67537d17", "a86e7cdd79dab129b982e46a792affbf6a3aa4a74e2ef5ed5ce3a39f8b53f3b0", "d0586a29873f5c4efddf0b672f31b2dd3f5c3d59ba005f9354614cc7ac2eaf8c", "db64a54efca2e3444dc445060f0e9d86662df7d66adb959ed4e8b4337c059511", "4e2339c452a22f4502c3086929745edbcf8cea8e53095d748ca2de47cd974cb5" ], "threshold": 0.75, "reason": "Proposes the existing local-markdown tracker, uses the single-context domain layout, skips triage-label setup, and presents a draft while stating nothing was written.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "6c3320a480a3acfb99cb36901c93a2947141c55ce52d5de9403eebb0745700e9" }, "judge_usage": { "input_tokens": 31481, "output_tokens": 1014, "cached_tokens": 29134, "wall_ms": 16498 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 1, "sd": 0, "judge_sd_mean": 0, "variance_ratio": null, "variance_ratio_unavailable": "judge_sd_zero", "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "bcd275f99e1f1e8ba5ce3d42865e3df0bf618b9fe77f7a9b6fba4aab4009e70b", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "5e6c8247cd7b56923076ebebaefad0de5301773acfada9e9577cad34f7f3fd59", "993c6101b3150357418bb29e2c0d920e4e76df12de1f576456db970300285f8c", "f0dee57705cd047fe034c1b13f6de783d3fac13bec3263f2d370ae3fb3dd72a1", "f4e9a877f45d0261db31fc3e4d6c82e6bb1c61d7238a259fa0c521f62650beb8", "2973bfda188bb938fa18c055eb9f74dba0f4c9680e56e0e5d7c5d682b59c52fe" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4534, "output_tokens": 592, "cached_tokens": 544, "wall_ms": 6700 }, "judge_usage": { "input_tokens": 30820, "output_tokens": 763, "cached_tokens": 28606, "wall_ms": 14596 } }, { "draw_index": 1, "generation_hash": "0acac83dc75aa3a26c1bd0d3bd048b65185ffb366c444fa31e84f9f8902e7500", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "788bd8cb4b1effbb826b3c09077ecf3d1c05beb76c0782cc12f783d36ef1a5a9", "6a690f52314b8619bed00b856821584276ef40e0cfe0f38561b35bb3dfd291c8", "417ad22bb70afe6a470847ffb1184484f825e46d708918d50cf7b8622986b399", "3994ab4fc4199c06056fad0d0f121c84038e71507642c8dd55bc10ef65ce1f0d", "fd4307ef2f292f0eab62b2c307f115870eb6a51f116b3dbe4b3563ad837985a9" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4535, "output_tokens": 754, "cached_tokens": 4326, "wall_ms": 9067 }, "judge_usage": { "input_tokens": 31630, "output_tokens": 733, "cached_tokens": 29254, "wall_ms": 15732 } }, { "draw_index": 2, "generation_hash": "ef00d007b02310e35f4860328b76aaea765cb0a472e917138ae9370d686108d4", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "42551ec6c63bfddd5ef3e76438a1cddeda7100ad93139e1bf103c7cc67537d17", "a86e7cdd79dab129b982e46a792affbf6a3aa4a74e2ef5ed5ce3a39f8b53f3b0", "d0586a29873f5c4efddf0b672f31b2dd3f5c3d59ba005f9354614cc7ac2eaf8c", "db64a54efca2e3444dc445060f0e9d86662df7d66adb959ed4e8b4337c059511", "4e2339c452a22f4502c3086929745edbcf8cea8e53095d748ca2de47cd974cb5" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4535, "output_tokens": 724, "cached_tokens": 4326, "wall_ms": 8697 }, "judge_usage": { "input_tokens": 31481, "output_tokens": 1014, "cached_tokens": 29134, "wall_ms": 16498 } } ], "issue408": { "attempt_id": "4a816239a96f2989b32764676989b082b6d5a23c905f8edffb0671783e2fdd70", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "setup-matt-pocock-skills", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/setup-matt-pocock-skills/SKILL.md", "declared_name": "setup-matt-pocock-skills", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/setup-matt-pocock-skills/SKILL.md", "skill_blob_sha1": "293c25b75590119794af03b037ba4288054a32b3", "skill_sha256": "f148858ffb7f63c03c06fca06e6fdb1d91a35c6b293244b8fdfacd5f4cc028c4", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--setup-matt-pocock-skills.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:15:22.958Z", "configuration_unchanged": true }, "budget_cost_usd": 0.843738, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } }, { "id": "local", "mode": "baseline", "outcome": "fail", "score": 0.5875, "mean": 0.5875, "stddev": 0.075, "samples": [ 0.75, 0.75, 0.5, 0.5, 0.75 ], "generation_hash": "21e3531d4f088d27586562f1140f304a4adf1b8596119b575ee95a2a505d3571", "judge_sample_hashes": [ "2d78046aa6f099affd0b718bd52caee6fc0a4c4b2777b1ed77dbbcc28b485eac", "3daa3cebf53f2bfd6ac4703b6b7edd2f353690d143a583140381aa2553ff72e2", "452a7c65317de1739037ce43cd7c03aec2e69c3860d9f8d454b87ef0e8b2640a", "b332e16508ca92b51dbe27488680c04f0f10722f387831530c3ac607eed17387", "a7521e7516ae474d43cc6c20d8e8bdd787beb56461d61d4ae2aa1fad842d4e35" ], "threshold": 0.75, "reason": "Meets preserve-local-tracker, single-context layout, and draft-before-changes; misses skip-triage-label setup because it still recommends installing triage with defined states.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "6c3320a480a3acfb99cb36901c93a2947141c55ce52d5de9403eebb0745700e9" }, "judge_usage": { "input_tokens": 31639, "output_tokens": 3268, "cached_tokens": 29262, "wall_ms": 25764 }, "generation": { "n_planned": 3, "n_drawn": 4, "n_measured": 4, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.5875, "sd": 0.075, "judge_sd_mean": 0.096416, "variance_ratio": 0.777879, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "fbc1c237ab10ed5c318f293be067e94414ccbb642d4505accd456cc4b487ec19", "status": "measured", "samples": [ 0.5, 0.75, 0.5, 0.5, 0.5 ], "judge_sample_hashes": [ "e69caf92dd148e802f89a87e00d5cc0592bb9190782468d692f41f7279475912", "43dd53556422939ff4b2e98e65144d2d3187458ff36521ec21a02d490c96aa82", "fa7cd1bcc28a59c564d1465e6beea45758ef2e5cff2a62c5f0c90aada421fd00", "4f38cddb8eb4001c757fe3beef8d98616a4fd4e3d8532dd6921034cc3fef97b7", "8156089081b23b091e7789a8d9d56157bd1f84cb4bbdf961472d5f2fa1e5e9ab" ], "mean": 0.55, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 1935, "output_tokens": 809, "cached_tokens": 1467, "wall_ms": 11698 }, "judge_usage": { "input_tokens": 31293, "output_tokens": 2854, "cached_tokens": 28986, "wall_ms": 24313 } }, { "draw_index": 1, "generation_hash": "4ac4887235dfdc971c10380948429ea880e1c8228453718cbbbdd7fa4ed34134", "status": "measured", "samples": [ 0.5, 0.5, 0.5, 0.5, 0.5 ], "judge_sample_hashes": [ "8ee89409cb1dabe7b36911b509e6a3999010c2833be489ce3cb7574af39da0ba", "1de17b6fa94bb25ca488bfacfb824590d79f8847ba497f2313391e1bbec893f1", "f91396825dfbee7af9a02d895bac2a78965df9d3fc5432444bc0a09a2636a812", "95f1d44404051587198a13dd1d6aab11cfe15e79fe756d8f972d990033c99b37", "79e2c3cdc4c0954f0c3dce6a30f0240f0ded74b631f4f7855e9228c22bfd2ee2" ], "mean": 0.5, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 1937, "output_tokens": 787, "cached_tokens": 1727, "wall_ms": 9225 }, "judge_usage": { "input_tokens": 31603, "output_tokens": 3074, "cached_tokens": 29234, "wall_ms": 25628 } }, { "draw_index": 2, "generation_hash": "c2a4f3f322b4a584112a8b58f61a5ffa5ac39c4b51b29601a32497225e164202", "status": "measured", "samples": [ 0.75, 0.75, 0.5, 0.75, 0.5 ], "judge_sample_hashes": [ "f47bceb2bc1e953ce7a6488215a1c0de7146ad919d8447aa6dfa77018d3552c3", "fc94b5204bf1252558287000c5f016e75fc9f3f4df3e32cc0df38ffe6be3e188", "ad2f712e9fb3041882f3af5e1ef5ec6c9de265f6c391753cef192597eab2b3f7", "1bb5a8fad28928a276dcf1a0ef831d5d8e832f8e322083a6e73d1447dd9ab1bd", "00b584344aeea54dc2d266c21abca3db53cb83758e9d26f456cf4c1377a108e7" ], "mean": 0.65, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2026, "output_tokens": 780, "cached_tokens": 1467, "wall_ms": 11139 }, "judge_usage": { "input_tokens": 31742, "output_tokens": 2784, "cached_tokens": 29342, "wall_ms": 24329 } }, { "draw_index": 3, "generation_hash": "21e3531d4f088d27586562f1140f304a4adf1b8596119b575ee95a2a505d3571", "status": "measured", "samples": [ 0.75, 0.75, 0.5, 0.5, 0.75 ], "judge_sample_hashes": [ "2d78046aa6f099affd0b718bd52caee6fc0a4c4b2777b1ed77dbbcc28b485eac", "3daa3cebf53f2bfd6ac4703b6b7edd2f353690d143a583140381aa2553ff72e2", "452a7c65317de1739037ce43cd7c03aec2e69c3860d9f8d454b87ef0e8b2640a", "b332e16508ca92b51dbe27488680c04f0f10722f387831530c3ac607eed17387", "a7521e7516ae474d43cc6c20d8e8bdd787beb56461d61d4ae2aa1fad842d4e35" ], "mean": 0.65, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2025, "output_tokens": 822, "cached_tokens": 1817, "wall_ms": 9494 }, "judge_usage": { "input_tokens": 31639, "output_tokens": 3268, "cached_tokens": 29262, "wall_ms": 25764 } } ], "issue408": { "attempt_id": "4a816239a96f2989b32764676989b082b6d5a23c905f8edffb0671783e2fdd70", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "setup-matt-pocock-skills", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/setup-matt-pocock-skills/SKILL.md", "declared_name": "setup-matt-pocock-skills", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/setup-matt-pocock-skills/SKILL.md", "skill_blob_sha1": "293c25b75590119794af03b037ba4288054a32b3", "skill_sha256": "f148858ffb7f63c03c06fca06e6fdb1d91a35c6b293244b8fdfacd5f4cc028c4", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--setup-matt-pocock-skills.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:15:22.958Z", "configuration_unchanged": true }, "budget_cost_usd": 0.843738, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 4 } }, { "id": "labels", "mode": "with_skill", "outcome": "pass", "score": 0.84, "mean": 0.84, "stddev": 0.082158, "samples": [ 1, 0.75, 0.75, 1, 1 ], "generation_hash": "a826da695a363e5a21dbe73c5d63182f2918abd97211999d920216083cbbf760", "judge_sample_hashes": [ "5c28cd68f3c0992f47857f3796afb17fb1630928bb18af655e1e82c3d192b84e", "f32a2e0c8af1bf1712b2666938cd88baa273b95cb164823c153f8bcea9687643", "f380d850093596ac1f66a94ecc9fee870de29fe6681d5d1d8d03936368db747d", "17520eeebd5efe1682eea7ab236a2a6930ae8039896ebe739a419fceb04d1257", "b99e6fae7c6fb4d166cba9b5133c1314843825788f9b38166d28d617605c33b9" ], "threshold": 0.75, "reason": "Maps bug:triage and bug:ready to canonical roles, asks the maintainer to decide the ambiguous ready role and unmapped roles, preserves exact label strings, and avoids recreating existing labels.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "7341d9aa25b6e7181025cd52ea58abec14b068a2d363443817e8c1084726699c" }, "judge_usage": { "input_tokens": 33152, "output_tokens": 3719, "cached_tokens": 30470, "wall_ms": 27924 }, "generation": { "n_planned": 3, "n_drawn": 5, "n_measured": 5, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.84, "sd": 0.082158, "judge_sd_mean": 0.094468, "variance_ratio": 0.869691, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "404736d3edbe4b696d25cb72ba0231a4b32a37f07fa960afd15f217603fd5bdb", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "d96287e0f7f48c9f5a1f89354cc7f16945d006d8c86dd5092f9254896aa2cfb0", "838acff5689d70e6dfb1f4f855b935b37e9db480b5631b2177f9adccb10204d3", "cf25836ead3e1518979181f30d6aa65eb96d3befbcc409d3074e2f633a21a4e1", "39115324e864ff4d1def70ec4bd6cd95961b292e1334f62aa3aa100ecd4f393d", "a5a22e8621475d9c811d4b87d4b501b0439277c09dd959bea0a393f4d7645dfa" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4636, "output_tokens": 1025, "cached_tokens": 4066, "wall_ms": 10182 }, "judge_usage": { "input_tokens": 32130, "output_tokens": 3187, "cached_tokens": 29654, "wall_ms": 27451 } }, { "draw_index": 1, "generation_hash": "383d0ba664272ba36f878a4abc61fecced5afafa9136dc44411e60cab46bf7f9", "status": "measured", "samples": [ 1, 1, 1, 1, 0.75 ], "judge_sample_hashes": [ "eb500815bfa99056129056f61e439780028ef5f9d0b3ceaefdfb86e71ae80676", "09eecd0eeabcecec6a51ae9a7e2b51803a8826649251a2232dc9fb9d141c4ac9", "76e0839add104de9c34c56069ee53b6d2eeccb61afc578fce26604383f9870c3", "fc4c3f8d16ee53d00ac3a928c12ab286748f819f557a34e0089e45b4a665c792", "2555e0c96da4233d33f442c5be148cb9a2aa16471aab15a80b88fcbff3816b67" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4636, "output_tokens": 1140, "cached_tokens": 4428, "wall_ms": 11152 }, "judge_usage": { "input_tokens": 32461, "output_tokens": 2511, "cached_tokens": 29922, "wall_ms": 23780 } }, { "draw_index": 2, "generation_hash": "9b2f4d9abfad8c3fbe3287fe4db6fb47d62e4965b881614c8f7b742a46a422fc", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 1 ], "judge_sample_hashes": [ "117bc0c2a4a0065e855b1b96d8b967c018821c9d56d3b17b0090489e90457bf1", "399af752dca716ccad100b9284fd58e71199c64468817090018b109306b2763a", "5b45ac6c9aeca426aa81ff7c794ac311184d45c8433c5808780dbea1d9da6dee", "a5af715eb2c11f5b6a45610a509b0b15377f6dbb6c792a09b88080bc28ffeb92", "2ae71eac419b7e6bed553de2c442286c2aa2f0b751bd1aba78175f24a09dd9fb" ], "mean": 0.8, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4637, "output_tokens": 1011, "cached_tokens": 4428, "wall_ms": 9632 }, "judge_usage": { "input_tokens": 31476, "output_tokens": 1011, "cached_tokens": 29130, "wall_ms": 16420 } }, { "draw_index": 3, "generation_hash": "cb4ddbc77675ebb16fdffd1625a8a1ea51ff046dc610618ae1591ff298604fc1", "status": "measured", "samples": [ 0.75, 1, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "ffb4e1c3a335260fab85a63f1f5fe9cf833f986ded07e86bb272fb60df2b4189", "7b3e89630494be4cecefc7613d7ab2e78a21019ba9d4804e0a0d742462b5ddb2", "ea6d9453a07df935c2808dc98f101b1c485bca6d845804f2433b7114767e932f", "24b2f4488448125500d811b4c00452b72a12e9cfd73c73a1c3e3687729b81902", "cdc7c45c5fba59c6d933169902d13972e7007981b4bf9c6b0541fd4d06ab3380" ], "mean": 0.8, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4637, "output_tokens": 1279, "cached_tokens": 4428, "wall_ms": 11247 }, "judge_usage": { "input_tokens": 33483, "output_tokens": 4106, "cached_tokens": 30738, "wall_ms": 30065 } }, { "draw_index": 4, "generation_hash": "a826da695a363e5a21dbe73c5d63182f2918abd97211999d920216083cbbf760", "status": "measured", "samples": [ 1, 0.75, 0.75, 1, 1 ], "judge_sample_hashes": [ "5c28cd68f3c0992f47857f3796afb17fb1630928bb18af655e1e82c3d192b84e", "f32a2e0c8af1bf1712b2666938cd88baa273b95cb164823c153f8bcea9687643", "f380d850093596ac1f66a94ecc9fee870de29fe6681d5d1d8d03936368db747d", "17520eeebd5efe1682eea7ab236a2a6930ae8039896ebe739a419fceb04d1257", "b99e6fae7c6fb4d166cba9b5133c1314843825788f9b38166d28d617605c33b9" ], "mean": 0.9, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4636, "output_tokens": 1263, "cached_tokens": 4428, "wall_ms": 10877 }, "judge_usage": { "input_tokens": 33152, "output_tokens": 3719, "cached_tokens": 30470, "wall_ms": 27924 } } ], "issue408": { "attempt_id": "4a816239a96f2989b32764676989b082b6d5a23c905f8edffb0671783e2fdd70", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "setup-matt-pocock-skills", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/setup-matt-pocock-skills/SKILL.md", "declared_name": "setup-matt-pocock-skills", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/setup-matt-pocock-skills/SKILL.md", "skill_blob_sha1": "293c25b75590119794af03b037ba4288054a32b3", "skill_sha256": "f148858ffb7f63c03c06fca06e6fdb1d91a35c6b293244b8fdfacd5f4cc028c4", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--setup-matt-pocock-skills.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:15:22.958Z", "configuration_unchanged": true }, "budget_cost_usd": 0.843738, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 5 } }, { "id": "labels", "mode": "baseline", "outcome": "fail", "score": 0.6875, "mean": 0.6875, "stddev": 0.047871, "samples": [ 0.75, 0.75, 0.75, 0.75, 0.5 ], "generation_hash": "bc08058cb940a0c23456f7a2ab8d964536c7cb5999886079e029a9a27d86aa08", "judge_sample_hashes": [ "22161b38546940c36b9cdfd974cc15d431fe9bede4c9c8f42a35ca10687c7ee1", "6acd3f6e2e042ccb513951e2b001f39e58e631644cdf2f6b34da8d2aba9f49f6", "103f7486d3460434d415324e819f9c91bce93b936260635b66d23890a1804ebc", "90217ed31fb1e6e692f5b18709d9d3fe524306e073005695992db0cdfd124f7f", "e64f043e2125d58a30d53e3dc59bc9c8f150702cbb9b226a60af68e192c098bb" ], "threshold": 0.75, "reason": "Maps the triage and ready roles to bug:triage and bug:ready, preserves exact label strings, and disables the bootstrap step to avoid duplicates; it misses rubric point 2 by assuming two roles rather than asking about unmapped ones.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "7341d9aa25b6e7181025cd52ea58abec14b068a2d363443817e8c1084726699c" }, "judge_usage": { "input_tokens": 31384, "output_tokens": 1976, "cached_tokens": 29058, "wall_ms": 19554 }, "generation": { "n_planned": 3, "n_drawn": 4, "n_measured": 4, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.6875, "sd": 0.047871, "judge_sd_mean": 0.096416, "variance_ratio": 0.496505, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "8beb0e431b5b1f0477568c781f5210728ddcb3900f45e136c8437354f958e1b4", "status": "measured", "samples": [ 0.5, 0.75, 0.5, 0.75, 0.75 ], "judge_sample_hashes": [ "0febe2addda391a2573a9150d4ee85dd5b413bfab835bc72bed3c46f6e1d676e", "c6490f9b300a6af8707e47b8897f2e42a0a531aed2de520c03f2c575f5ea098e", "418150e05b8e421336a5d2a64423f16975ee2c294670e2de4fdbdbad7c9b28ac", "0218e97bc08a59613b43233e2d7e243be8cc57901e1a17641d3de129360cb437", "ce25013648572c12a0638932c525fae5cc4ee516bb903f54d2a201aa54d294c6" ], "mean": 0.65, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2038, "output_tokens": 899, "cached_tokens": 1467, "wall_ms": 10035 }, "judge_usage": { "input_tokens": 31693, "output_tokens": 2012, "cached_tokens": 29306, "wall_ms": 22001 } }, { "draw_index": 1, "generation_hash": "04dcf9fd04ef9ec0ed946634db3699663b40f416658a59a83b5556086765c4e4", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "ac6453aa685d5133a69dd82f3fe711d8dd7560f5375d7305a8f01d71174b61e8", "4bcfd85d921c4e507d2cddddb97431cf3e1aa8a4c87667684d4a6628cdc668ea", "eb66c9f67825d26637d9d0224e06ee3ad75a0c56e7b861f5b151e792c06afbb5", "f8240a4edd464b00b2bd4a4776475901d5058fd2c5f3845cad71b3be528810aa", "cdecc8099dc064d160477673a0eaa678c651b54f5229f66774e89a9f67708d1c" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2039, "output_tokens": 748, "cached_tokens": 1829, "wall_ms": 8733 }, "judge_usage": { "input_tokens": 31208, "output_tokens": 3328, "cached_tokens": 28918, "wall_ms": 25803 } }, { "draw_index": 2, "generation_hash": "aa7aac09b1f0d32b0e3b515e249b3639c109c34dfb880efbdd81ad27ce23b00d", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.5, 0.5 ], "judge_sample_hashes": [ "b6fd495056dbaf1c27c44e6059ce855156833f876932589de08930941aff621e", "891b77d97febc433e59f6ee61e7868eb304c5310647ce89544877463215a275c", "6da90ca33141a3e8b511c67949c3ea098dc62164a2dd0d6effc456c40d655bdc", "e947df3c257b9c3a3aa4dc999668cfcb7f7b691b4c9aca23aaa48b6d9f586667", "29ba9a8da7971e209fef4cebdb062a61e8ef09af4725c08530b11904d07e4aa5" ], "mean": 0.65, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2037, "output_tokens": 928, "cached_tokens": 1829, "wall_ms": 9432 }, "judge_usage": { "input_tokens": 31920, "output_tokens": 1735, "cached_tokens": 29486, "wall_ms": 20257 } }, { "draw_index": 3, "generation_hash": "bc08058cb940a0c23456f7a2ab8d964536c7cb5999886079e029a9a27d86aa08", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.5 ], "judge_sample_hashes": [ "22161b38546940c36b9cdfd974cc15d431fe9bede4c9c8f42a35ca10687c7ee1", "6acd3f6e2e042ccb513951e2b001f39e58e631644cdf2f6b34da8d2aba9f49f6", "103f7486d3460434d415324e819f9c91bce93b936260635b66d23890a1804ebc", "90217ed31fb1e6e692f5b18709d9d3fe524306e073005695992db0cdfd124f7f", "e64f043e2125d58a30d53e3dc59bc9c8f150702cbb9b226a60af68e192c098bb" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2037, "output_tokens": 804, "cached_tokens": 1829, "wall_ms": 9510 }, "judge_usage": { "input_tokens": 31384, "output_tokens": 1976, "cached_tokens": 29058, "wall_ms": 19554 } } ], "issue408": { "attempt_id": "4a816239a96f2989b32764676989b082b6d5a23c905f8edffb0671783e2fdd70", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "setup-matt-pocock-skills", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/setup-matt-pocock-skills/SKILL.md", "declared_name": "setup-matt-pocock-skills", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/setup-matt-pocock-skills/SKILL.md", "skill_blob_sha1": "293c25b75590119794af03b037ba4288054a32b3", "skill_sha256": "f148858ffb7f63c03c06fca06e6fdb1d91a35c6b293244b8fdfacd5f4cc028c4", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--setup-matt-pocock-skills.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:15:22.958Z", "configuration_unchanged": true }, "budget_cost_usd": 0.843738, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 4 } }, { "id": "monorepo", "mode": "with_skill", "outcome": "pass", "score": 0.95, "mean": 0.95, "stddev": 0.086603, "samples": [ 1, 1, 1, 1, 1 ], "generation_hash": "171caa68e64ef64f12f2c57715eb51a4a437742de08c0b2f1f55f84a5b0f2b20", "judge_sample_hashes": [ "fa68632eeb499e3e76ee4fd5c533c052a04b0d5a896f125e189979ee415cde3c", "cef2fab40cf0b582efe443034692ea41bddd7d164e3e5e0a6efd5547ca7b46ea", "9ab174f9c84b5db15df3faf9eb1ed589040f147d02994f1394a0b172f1876796", "5b60ace5d60663ce1cd9e328fdee4c19016d30d40cbcd23aad6eab41194b8a80", "5fe832dbd456c4f9a94d8da3f42dd5c1e9f559f9b35937d9c7d361312be5f4b7" ], "threshold": 0.75, "reason": "Identifies the two-package monorepo as multi-context evidence, proposes a root GLOSSARY-MAP.md pointing to existing per-package glossaries and ADRs without moving them, and asks for confirmation before any writes.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "e92c9ec08a1d09ee962d1945f3fc63a438da2b1325fbf83d1c04f71513c012fc" }, "judge_usage": { "input_tokens": 32328, "output_tokens": 1605, "cached_tokens": 29814, "wall_ms": 18271 }, "generation": { "n_planned": 3, "n_drawn": 5, "n_measured": 5, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.95, "sd": 0.086603, "judge_sd_mean": 0.044721, "variance_ratio": 1.936518, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "23b1f336983a55cebcda3c56fb9a135de333740112a3633e7e0af575e024badb", "status": "measured", "samples": [ 0.75, 1, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "8e2a0a484d79e9d3b49f2d8abd1aaec500bf579d13ea929cc19177598e388270", "8b5b1200a2805ee54e58ed9cdb6ce8788db5d924f0777868173307686e7aabb7", "181b11d23b911bd5b8bf5e4feda9a796eb3baf2b73f959fd6972a3fbf0381b77", "dff7f1ba285a3bba8e663a81cabda5c7c3f2110081e8d35ca4861498da290233", "485405d0381e55f6107229288ac100b136983a6bee1e46bfd5acec8ba21331aa" ], "mean": 0.8, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4631, "output_tokens": 723, "cached_tokens": 4066, "wall_ms": 7967 }, "judge_usage": { "input_tokens": 31985, "output_tokens": 3650, "cached_tokens": 29538, "wall_ms": 27163 } }, { "draw_index": 1, "generation_hash": "65542a8ee3f826487944412fc76790b54df1fc442f76e2c6f61d1616979e253c", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "f08d8e0d72841d2a3dfde11fed15838292545d9abf6eb3ea968a0f30bf37b316", "fc0f376b8cf00823bcc0c6053863becf20a92faba4d9f4092c2d3419a4cae4b5", "dc6e7df08cf6dfcfc2f92dd715743e2700ebacfe2272b671cb5cf5830f0f2188", "bcbeb84d669a67ce3819b945835eac4d4a9834b8a2bb404a004190fbaab3df31", "d5752e5a4f83322d064f6d6782655a3e5017423dfb0d6534f3f90461c339ed01" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4631, "output_tokens": 635, "cached_tokens": 4422, "wall_ms": 6810 }, "judge_usage": { "input_tokens": 31544, "output_tokens": 3135, "cached_tokens": 29186, "wall_ms": 25886 } }, { "draw_index": 2, "generation_hash": "dcadb0d4d181b7a81d633c65dbe28ed82f92001b1021245af9529c30e3571885", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "7fe4f38690bac2c075bb5a1d12f507f8b5dae228d6787a33d777f875da0ad40c", "6a84c0015a9365ce0ad7427ebc5ef07ce51f6410e938566071e8c179a29177f9", "b32b9e55948b3ea306d97f4a48e2788c1c1a5f4a199f09fbaf5daad2d6caef0f", "e9d73835ac006d2bc390bae825e91afd113bd2fab422c2c2a625a24e3563b025", "42ae4ca9d983fd5fcadb4097ae53a3a41f08a2d9bdfb7fb4d07e70af370f973f" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4629, "output_tokens": 736, "cached_tokens": 4422, "wall_ms": 7372 }, "judge_usage": { "input_tokens": 32048, "output_tokens": 2177, "cached_tokens": 29590, "wall_ms": 21514 } }, { "draw_index": 3, "generation_hash": "e4579668e15a91215dc9c757c8ed89d5c1f65c2ed1599714607518559961622d", "status": "measured", "samples": [ 1, 1, 1, 0.75, 1 ], "judge_sample_hashes": [ "89a07abe38cfbebad4b0b37ff9ff31eb0139bcf598f57a44616be7b3535dd89d", "77425c5e5d52ff606af392ade3b15766dc363ad62e42002e930bb09f37db6c14", "fddcdac12db76f1d2fa7854e765f31484afe20cc876cb22c1d1fa99689822b88", "f09542efe9f42e851c5598c710f6faa4798c942103c47524abddb648202a5a3a", "e8b54b4f146e321aa94df7b7bd097d5d6fcee8ac1b76fff2a8e0f277ad18d468" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4632, "output_tokens": 631, "cached_tokens": 4422, "wall_ms": 6902 }, "judge_usage": { "input_tokens": 31525, "output_tokens": 3618, "cached_tokens": 29170, "wall_ms": 27451 } }, { "draw_index": 4, "generation_hash": "171caa68e64ef64f12f2c57715eb51a4a437742de08c0b2f1f55f84a5b0f2b20", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "fa68632eeb499e3e76ee4fd5c533c052a04b0d5a896f125e189979ee415cde3c", "cef2fab40cf0b582efe443034692ea41bddd7d164e3e5e0a6efd5547ca7b46ea", "9ab174f9c84b5db15df3faf9eb1ed589040f147d02994f1394a0b172f1876796", "5b60ace5d60663ce1cd9e328fdee4c19016d30d40cbcd23aad6eab41194b8a80", "5fe832dbd456c4f9a94d8da3f42dd5c1e9f559f9b35937d9c7d361312be5f4b7" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 4632, "output_tokens": 792, "cached_tokens": 4422, "wall_ms": 8183 }, "judge_usage": { "input_tokens": 32328, "output_tokens": 1605, "cached_tokens": 29814, "wall_ms": 18271 } } ], "issue408": { "attempt_id": "4a816239a96f2989b32764676989b082b6d5a23c905f8edffb0671783e2fdd70", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "setup-matt-pocock-skills", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/setup-matt-pocock-skills/SKILL.md", "declared_name": "setup-matt-pocock-skills", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/setup-matt-pocock-skills/SKILL.md", "skill_blob_sha1": "293c25b75590119794af03b037ba4288054a32b3", "skill_sha256": "f148858ffb7f63c03c06fca06e6fdb1d91a35c6b293244b8fdfacd5f4cc028c4", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--setup-matt-pocock-skills.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:15:22.958Z", "configuration_unchanged": true }, "budget_cost_usd": 0.843738, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 5 } }, { "id": "monorepo", "mode": "baseline", "outcome": "borderline", "score": 0.79375, "mean": 0.79375, "stddev": 0.123744, "samples": [ 1, 1, 1, 1, 0.75 ], "generation_hash": "67d66d552bee567e83ffaea5b119aad952ead6802d8b1ace38df45fd37d4e148", "judge_sample_hashes": [ "bb447ab7dd019030e2df1856ebfdf31c75d84457d73b5e88464f7a02cb6aeb53", "5451235b6537efe16108b4191fbe99a8ebc12bda93d0e6672cc420a8b982ad4a", "1a3fcb403a8e2f7a9885df9a63a4252b147126ef6ef02e3b0b44d328a68d738d", "70a3ecd539e0551980fe891da17b221645814c97533f0e673e6901b161e81ac9", "5fbaa112329df56e580774457323424dcf63a950e6faa696d01ff1cd04a60bdf" ], "threshold": 0.75, "reason": "Recognizes the two packages as bounded contexts with separate glossaries, proposes a root index linking each context's glossary and ADRs, preserves existing locations, and asks for confirmation before any writes.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "e92c9ec08a1d09ee962d1945f3fc63a438da2b1325fbf83d1c04f71513c012fc" }, "judge_usage": { "input_tokens": 32712, "output_tokens": 3940, "cached_tokens": 30122, "wall_ms": 27981 }, "generation": { "n_planned": 3, "n_drawn": 8, "n_measured": 8, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.79375, "sd": 0.123744, "judge_sd_mean": 0.099164, "variance_ratio": 1.247872, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "2778b3aaacfed044727108d80d206c2c533ee9e5c06d4f3d1bac19923cef2ed8", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "c053f974b6183c576bb7df3c4de1c5925cd931676d689a7c470347c951c97e54", "80683b72fd7e86b5d05ceee16b31c79247cddf03f7d5f8e1794cb34ab6bc29d0", "c5bf7e34936ae0950c901cd77848df94d67c3b33dc3c1bb7725ee1cc0d47cea2", "97266b458e2957ad0a29bdb5a1c267171ec6d8f611bcebc64ab454cdd7d127dc", "d960c9439141fe6ce7109e281e3644d0e71a4521d0d3409a38dc45b7b9077681" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2033, "output_tokens": 772, "cached_tokens": 1467, "wall_ms": 8578 }, "judge_usage": { "input_tokens": 32230, "output_tokens": 2683, "cached_tokens": 29734, "wall_ms": 22186 } }, { "draw_index": 1, "generation_hash": "007f308e298887f005b3ce34873243deb11c87f95ecd3b34dd2bb0ffaca362e1", "status": "measured", "samples": [ 0.75, 0.75, 0.5, 0.75, 0.5 ], "judge_sample_hashes": [ "936bd2d81a0bc813aa0bb7718f53f17c95cad57f0ccf7e5062dd209974d7ece7", "6993b78aecc895bbca15735d51eb55b033c49a0ed6fa609d4fdd5808c8eb58e5", "91d308e8dc4468fdf5a856e9d10ec6ad1a9c748d0333cd242db656223dd00a5f", "88a72d92e185d6ea7be00f27c0df5bc992044f83bf3ce32d599328154691504b", "cc31f5e7a322a33075cfa264d20eac2c00f3c29bc937c3ef7f12bf867673b6a3" ], "mean": 0.65, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2032, "output_tokens": 889, "cached_tokens": 1823, "wall_ms": 9790 }, "judge_usage": { "input_tokens": 32810, "output_tokens": 4125, "cached_tokens": 30202, "wall_ms": 29089 } }, { "draw_index": 2, "generation_hash": "1051fdd8ca31b689099b80f43260a81f9017f4c30a231c78b9db7253fcd407e1", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.5, 0.75 ], "judge_sample_hashes": [ "eddabc8cfb6270bb6754f7c91313ff805da15cb6e441bd1422da7ad364a6e6de", "579f9b829577a2a2bae8dc75c8685599467b6eabdf67ce0b99ad13b7783ce959", "71112904d2e8a5e261df73063d3b1d478ab9d5db3a114833a668f042005595d8", "a2c26825f893ad0f0e2e8ae1f9cb31b99f8a0e56803ef0886fb3b0f6de73a64c", "a57bfd8e6fbf53cc7ba7e433db5222521e49b8909be0517bd5d1a3ad29288687" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2032, "output_tokens": 844, "cached_tokens": 1823, "wall_ms": 9601 }, "judge_usage": { "input_tokens": 32588, "output_tokens": 4939, "cached_tokens": 30022, "wall_ms": 32319 } }, { "draw_index": 3, "generation_hash": "ef930e680747a994d6cdc29564b3a5cb5a6be81bb0414947697ca5673984ec4c", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "e58dcecdfe1f2fcd0691d459d06ce1adb009ef2de4287b5877c14c11307f6fc4", "83a1cced1fc80477c6e9c505627d612911bb3c3cdd66f759031b52262a35a12f", "b5ffe26776652fdc17eda64870a389c865536398aedd4866dbebf92cf8b6b973", "b36a23ff52f89ef657888361155f1573b71b9b17b09a2f141900494f2fb14c21", "5a76727dc6d0159bf19d51968b10de4b3061ee705ceceb2fce86b1f817055aaf" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2032, "output_tokens": 1049, "cached_tokens": 1823, "wall_ms": 10903 }, "judge_usage": { "input_tokens": 31605, "output_tokens": 3332, "cached_tokens": 29234, "wall_ms": 26716 } }, { "draw_index": 4, "generation_hash": "949af614273c1b767e72d6da553cc8a7851adddc70ff25973206adde9d0b4ae4", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.5, 0.75 ], "judge_sample_hashes": [ "31a29059bfccaf317feac1204b916c48ea1041ebef3536d0d76c8b88c5c9e884", "5fdd4f55246b197f2e70ff8eb35a2fb2f02875a32113940752b3191ff6ae13ce", "a2f54d996f9a30607c53cd165ce527640682cc5aa0526fd9900c48a66a02c56b", "287e602d316c55d310bc218c114ef67d4006c2df304b5e32cc7d45d207b0a07b", "af566e10615dda7a84c9393783466bbd838370f5dba6469e9f2ca15dbce88b69" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2032, "output_tokens": 841, "cached_tokens": 1823, "wall_ms": 9133 }, "judge_usage": { "input_tokens": 31755, "output_tokens": 2936, "cached_tokens": 29358, "wall_ms": 24958 } }, { "draw_index": 5, "generation_hash": "42e6826d302687178128b26d7f27c60035e9466e12d249033e17b9cf66f74b79", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 1, 0.75 ], "judge_sample_hashes": [ "89162b4b63bf7b58476a5b9774eadd1c6998991be275150df590e291be2ba8ef", "04bdff1de90d0b5faa203523331142deb7356be69ef0882488aeccf3cad47103", "461c15f0222cba5da9b52706c0320be791b3ad1633ff42b787965a72f6df1be1", "c089925540ca9b3a20c54df01b1574912a1d46416938f30bb04771a2e00186fa", "2fae2c5a6d793353adc49f29de2d8176dc98fdfa71a8c9f91d6392e90332f51f" ], "mean": 0.8, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2031, "output_tokens": 808, "cached_tokens": 1823, "wall_ms": 8820 }, "judge_usage": { "input_tokens": 32404, "output_tokens": 4289, "cached_tokens": 29878, "wall_ms": 29741 } }, { "draw_index": 6, "generation_hash": "f2f592330866a402c5f5cf784637464aa79a001a4b486f68411771441f0e0b83", "status": "measured", "samples": [ 1, 0.5, 0.75, 0.75, 1 ], "judge_sample_hashes": [ "e22a86bccd4074d5a04b7130418e8f295bdfb504610ed323c617523903bbb899", "5a2056905861f147a5dfaf91b3b81c04723debd69e696e59188d955ee410c3bf", "239cbaef9c0f0c3764945fafd61b538f50dd8dbef45f97e9b2b15cce1232360e", "fd03ad5c08682d0a12b8188a9db1004d24160c6ee97b3be777acbbbb7926c957", "bfb082ac10368cb347188aacfb160ffde905397d402f0eef95e0bfbd00750e16" ], "mean": 0.8, "stddev": 0.209165, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2030, "output_tokens": 855, "cached_tokens": 1823, "wall_ms": 9556 }, "judge_usage": { "input_tokens": 32645, "output_tokens": 3729, "cached_tokens": 30066, "wall_ms": 27725 } }, { "draw_index": 7, "generation_hash": "67d66d552bee567e83ffaea5b119aad952ead6802d8b1ace38df45fd37d4e148", "status": "measured", "samples": [ 1, 1, 1, 1, 0.75 ], "judge_sample_hashes": [ "bb447ab7dd019030e2df1856ebfdf31c75d84457d73b5e88464f7a02cb6aeb53", "5451235b6537efe16108b4191fbe99a8ebc12bda93d0e6672cc420a8b982ad4a", "1a3fcb403a8e2f7a9885df9a63a4252b147126ef6ef02e3b0b44d328a68d738d", "70a3ecd539e0551980fe891da17b221645814c97533f0e673e6901b161e81ac9", "5fbaa112329df56e580774457323424dcf63a950e6faa696d01ff1cd04a60bdf" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-sonnet-5-5", "usage": { "input_tokens": 2031, "output_tokens": 869, "cached_tokens": 1823, "wall_ms": 9928 }, "judge_usage": { "input_tokens": 32712, "output_tokens": 3940, "cached_tokens": 30122, "wall_ms": 27981 } } ], "issue408": { "attempt_id": "4a816239a96f2989b32764676989b082b6d5a23c905f8edffb0671783e2fdd70", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "setup-matt-pocock-skills", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/setup-matt-pocock-skills/SKILL.md", "declared_name": "setup-matt-pocock-skills", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/setup-matt-pocock-skills/SKILL.md", "skill_blob_sha1": "293c25b75590119794af03b037ba4288054a32b3", "skill_sha256": "f148858ffb7f63c03c06fca06e6fdb1d91a35c6b293244b8fdfacd5f4cc028c4", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--setup-matt-pocock-skills.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T06:15:22.958Z", "configuration_unchanged": true }, "budget_cost_usd": 0.843738, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "local", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "local", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "labels", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "labels", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 6, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": null, "model": "claude-sonnet-5-5", "reported_models": [ "claude-sonnet-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "monorepo", "mode": "baseline", "draw": 7, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 8 } } ], "aggregates": { "with_skill": { "case_count": 3, "pass_count": 3, "borderline_count": 0, "mean_score": 0.93, "stddev": 0.081854 }, "baseline": { "case_count": 3, "pass_count": 0, "borderline_count": 1, "mean_score": 0.689583, "stddev": 0.103141 }, "band_rule": "per arm: the sample standard deviation (n-1) of the per-case means across the included cases; the comparison band is the quadrature sum of the two arms; null where an arm has fewer than two included cases" } }, "comparison": { "with_skill_score": 0.93, "baseline_score": 0.689583, "delta": 0.240417, "delta_uncertainty": 0.131674 }, "verification_level": "TESTED", "receipt_hash": "6983aa785fcd5d04730835a4a3b24ad45d0692ce4b95521bddaf31150a8e1f86", "economics": { "basis": "metered-equivalent", "surface": "claude-cli", "with_skill": { "call_count": 13, "mean_input_tokens": 4610.85, "mean_output_tokens": 869.62, "mean_cost_usd_per_call": 0.017918, "median_wall_ms": 8697, "wall_ms_p25": 7137, "wall_ms_p75": 10529.5, "wall_ms_iqr": 3392.5 }, "baseline": { "call_count": 16, "mean_input_tokens": 2020.44, "mean_output_tokens": 844, "mean_cost_usd_per_call": 0.012481, "median_wall_ms": 9533, "wall_ms_p25": 9179, "wall_ms_p75": 9981.5, "wall_ms_iqr": 802.5 }, "skill_incremental_cost_usd_per_call": 0.005437, "skill_incremental_cost_usd_per_1k_calls": 5.437, "output_tokens_delta": 25.62, "median_wall_ms_delta": -836, "judge_excluded": true, "judge_overhead": { "note": "Measurement overhead imposed by Driftproof, NOT a cost of running the skill. Excluded from every field above. Counted over every draw sent to the judge, each once: the usage its answered judge samples reported, at the frozen judge price. Retried attempts and samples that reported no usage are not in it.", "total_cost_usd": 0.133848, "case_rows_measured": 6, "judged_draws_counted": 29, "judged_draws_missing_usage": 0 }, "notes": { "absolute_cost": "Absolute per-call cost on a CLI surface includes a fixed harness preamble (observed ~25k input tokens on claude-cli, ~11k on codex) that we do not control. It is identical in the with-skill and baseline arms, so it cancels in the Δ figures — read the incremental columns, not the absolute ones.", "cache_pricing": "Cached input tokens are costed at the list input rate (an over-estimate where caching is heavy); cached_tokens is recorded per call so a reader can recompute.", "latency": "observed on subscription CLI surface, indicative" } } }