{ "schema_version": "0.13", "generation_sampled": true, "skill": { "name": "improve-codebase-architecture", "version": "0.0.0", "content_hash": "392c6e6a10d199b2c6d895cd6ee91f0934292e3cf0c71279e79bce94cae04418", "tokens": 1500 }, "suite": { "format": "agentskills.io/evals", "suite_hash": "808374fd9d1978c553ce7aef6bf5722469a74851217bd22dc31eea30d6f4d5e1", "case_count": 3, "canary": "4b16b436-88eb-1ca4-6df5-c26fe31a4d2b" }, "run": { "model_id": "claude-opus-5-5", "model_release_date": null, "provider": "anthropic", "surface": "claude-cli", "runner_version": "0.22.2", "date_utc": "2026-10-18T07:27:44.546Z", "generated_at": "2026-10-18T07:13:46.215Z", "registry": "registered", "transcripts": "retained-local", "judge": { "samples": 5, "temperature": null, "sampling": "surface-controlled", "surface": "claude-cli", "model_id": "claude-haiku-5-5", "prompt_template_hash": "82586d1e44f84df2b8ce5f18ac777866465cb5ad9686503c93d9776c93c344ed", "harness": { "name": "claude-code", "version": "2.1.287" }, "isolation": "eval-user", "endpoint": { "custom": false, "sha256": null, "resolved_by": "cli", "routing": { "ANTHROPIC_BEDROCK_BASE_URL": { "set": false, "sha256": null }, "ANTHROPIC_VERTEX_BASE_URL": { "set": false, "sha256": null }, "CLAUDE_CODE_USE_BEDROCK": { "set": false, "sha256": null }, "CLAUDE_CODE_USE_VERTEX": { "set": false, "sha256": null }, "HTTPS_PROXY": { "set": false, "sha256": null } } }, "code_revision": "judge-code-bb552899a1df887e", "reported_models": [ "claude-haiku-5-5" ] }, "answered_by": { "kind": "model", "attested": true, "reported_model": "claude-opus-5-5", "reported_models": [ "claude-haiku-5-5", "claude-opus-5-5" ], "isolation": "eval-user" }, "capture": { "mode": "text" }, "harness": { "name": "claude-code", "version": "2.1.287" }, "pricing_snapshot": { "frozen_at": "2026-10-18T07:27:44.546Z", "source": "config/models.json", "currency": "USD", "models": { "claude-opus-5-5": { "input_per_mtok": 4, "output_per_mtok": 20, "registered": true }, "claude-haiku-5-5": { "input_per_mtok": 0.1, "output_per_mtok": 0.5, "registered": true } }, "note": "Prices frozen at run time. Every derived cost in this receipt is computed from THIS snapshot, never from the live registry, so the receipt keeps its meaning when registry prices later change." }, "counts": { "judge_samples_per_generation": 5 } }, "results": { "cases": [ { "id": "candidate", "mode": "with_skill", "outcome": "borderline", "score": 0.76, "mean": 0.76, "stddev": 0.054772, "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "generation_hash": "f034dc8ae79747fbaf4894c23f03a5e5f174d9c79d71d59004b3512f6b3079af", "judge_sample_hashes": [ "2a0bc3c71bac1c9b70ffb076dcb4635877c3214fc120bbe97ff0b164e8344b47", "08a8a340af457f0237ec7c2007832d64e1571be8642d7efc65ee3fcc605e6bba", "577f75a79fc03908c4890b1567fcf26dde103c23cf411c654303f7c839d71f1d", "b170232436f081fc96bfffd3aacf71466416cd32567d3925af9604532b2ec947", "317fbd33b969839de66119a8ce8e7d327cbe5e3e99f2f4079f3fa5f931967139" ], "threshold": 0.75, "reason": "Prioritizes the validation area, identifies the ValidateOrder forwarder, and explains that deletion removes a layer rather than moving complexity; it names OrderRules as the test surface but never explains why that seam survives the change.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "1c4b139b8806acaa1fea36149f4b9ce95a037c206e8629cc011e0e5ea03fd335" }, "judge_usage": { "input_tokens": 32758, "output_tokens": 2906, "cached_tokens": 30158, "wall_ms": 25909 }, "generation": { "n_planned": 3, "n_drawn": 5, "n_measured": 5, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.76, "sd": 0.054772, "judge_sd_mean": 0.049747, "variance_ratio": 1.101011, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "44e4e3ab28ebb4dce2603160d037a246bcf0b368147e0a2af9f67d79549456ff", "status": "measured", "samples": [ 0.75, 0.75, 0.5, 0.75, 0.75 ], "judge_sample_hashes": [ "ef02d91535562d6c4555bfac7d3c2df2a0b6782fc203a3fcf6454c50e45ac308", "5e8a6effd5f6d35b2e72e4341975c60e478ba734af71fa2be99ed8bbb3e9e133", "2293d4651c804ea7e17a3230f07bc194158cf1a3b99c428f603a444123fdf73f", "bb3cfec16fadc894c5b484e8d2bf6077d2849064562ca586671ad2d9b54fbf92", "6e040cb807621cc0ac7fdd6bee1f9ab2f13943b9cb060dc0fa7a3c2ced4d8b77" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4195, "output_tokens": 1310, "cached_tokens": 544, "wall_ms": 15543 }, "judge_usage": { "input_tokens": 32869, "output_tokens": 3758, "cached_tokens": 30246, "wall_ms": 28100 } }, { "draw_index": 1, "generation_hash": "fd6fbf6e3937e189060d448d05622e3d7cabd389eedfe205d55713b37ca6b1bf", "status": "measured", "samples": [ 1, 0.75, 0.75, 0.75, 1 ], "judge_sample_hashes": [ "1db3b4d35ddbef747f68bb4f6a91f0439ed3468b30cc8c6d6134410458d77a61", "d9eeee53dfaf49312488f5a8cee14c8820db929ab63680d6f776d673cbe23f28", "d38b77b0b6b1a1cd6ebaf24db6cf8e41803bfc5a1f6eefa45d60df0e6dc76060", "d51212341fb8a043aa71c389eddd70e36ad3905bdaca7d117b2289e719644b71", "98751fa6eaf91389f1d2ddf8d0d5c5a4c7e3ed06d88c47f67cfd31f33833c1e6" ], "mean": 0.85, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4195, "output_tokens": 1083, "cached_tokens": 3990, "wall_ms": 29196 }, "judge_usage": { "input_tokens": 33224, "output_tokens": 4675, "cached_tokens": 30530, "wall_ms": 34622 } }, { "draw_index": 2, "generation_hash": "ddd18617cfe9f330492536db8a18684af2cb05f93535a38f6defd0d6074e7c0d", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "eaeaeb2501d3fdc3d3cfae9cbe6813843c20668cec29d6a6163946564ced4593", "e609fd913771f165119a46a2d1d332a6b50419410105d75e33d813ffb5fc214b", "bae39848d4b3c9fac483d7878c0f24e1d19f27f1beacf0354fb8b0a9e2aeb567", "2aab5c55b3a05efebef8e8ffd5491eea00805c9bb523f39c94e7e6a46e302c6d", "389c1d20d46a9313b9918c9d8c9f73005a743bae1631f94848e0d1ff4bff3483" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4196, "output_tokens": 1165, "cached_tokens": 3990, "wall_ms": 13679 }, "judge_usage": { "input_tokens": 32869, "output_tokens": 3217, "cached_tokens": 30246, "wall_ms": 26457 } }, { "draw_index": 3, "generation_hash": "47d684048fd319d647f094c4b0e16ab992fb284ea960e94cd62eb9b006e9f25b", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "c967b0620eae014d020d20ff50c55b882da435cb46ef1a346d9135def1baae51", "23f2d542f34a4289c007238da83ef4bca6027b3869a42835899e6d784b8d018d", "adead34e6da2af13418cb9d2978fef0b452d5da5261e87918703c5c523b89a4e", "6b97bd55dcfa0c87e40ed41d831638e329ce789c86b7e453af83d6eb8f46b060", "245177ca35186ad00e249231c61588ea118bb2f1597efca78c6093fcf524cda1" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4195, "output_tokens": 1162, "cached_tokens": 3990, "wall_ms": 14165 }, "judge_usage": { "input_tokens": 33507, "output_tokens": 3207, "cached_tokens": 30758, "wall_ms": 27236 } }, { "draw_index": 4, "generation_hash": "f034dc8ae79747fbaf4894c23f03a5e5f174d9c79d71d59004b3512f6b3079af", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "2a0bc3c71bac1c9b70ffb076dcb4635877c3214fc120bbe97ff0b164e8344b47", "08a8a340af457f0237ec7c2007832d64e1571be8642d7efc65ee3fcc605e6bba", "577f75a79fc03908c4890b1567fcf26dde103c23cf411c654303f7c839d71f1d", "b170232436f081fc96bfffd3aacf71466416cd32567d3925af9604532b2ec947", "317fbd33b969839de66119a8ce8e7d327cbe5e3e99f2f4079f3fa5f931967139" ], "mean": 0.75, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4194, "output_tokens": 1160, "cached_tokens": 3990, "wall_ms": 18269 }, "judge_usage": { "input_tokens": 32758, "output_tokens": 2906, "cached_tokens": 30158, "wall_ms": 25909 } } ], "issue408": { "attempt_id": "dd48f6ad46831a943dbf343b4f00a54df200f1ec07fe5f7c2db201abd622bd35", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "improve-codebase-architecture", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/improve-codebase-architecture/SKILL.md", "declared_name": "improve-codebase-architecture", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/improve-codebase-architecture/SKILL.md", "skill_blob_sha1": "44cc7d3136c44534d68ac523fa6cd37e759d2846", "skill_sha256": "552240a5ab5cec6b67c15dd1ad6e9f6962b1bca6ce870059a8c2713760c20392", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--improve-codebase-architecture.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:13:46.106Z", "configuration_unchanged": true }, "budget_cost_usd": 1.231904, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 5 } }, { "id": "candidate", "mode": "baseline", "outcome": "fail", "score": 0.6375, "mean": 0.6375, "stddev": 0.075, "samples": [ 0.75, 0.75, 0.75, 0.75, 0.5 ], "generation_hash": "9ca974ee631fa490ae85b9994003786db73240292140904ef30b20a0e16db1cf", "judge_sample_hashes": [ "4dec917873d5540506c079f572b1365f8943c07d9d070059aac3f5934d51a2ac", "f2748f290c16b9c3afb329cd3205bc25055e4fcb12daa63bbb0789ed545524e8", "32a6b6653583ec27d92397bade73c575985538704180db0025201f27ff2b91c1", "cac367af0346d62317df1eaeaecc290a35f07e22e24b543d8431d9c54a30376c", "a498a8e9eeaa84f1056758f7507591530002a0d82e6f7c288ad5d96a2a6e50b3" ], "threshold": 0.75, "reason": "Meets criteria 1-3: prioritizes validation, flags the ValidateOrder forwarder as shallow, and explains consolidation. Misses 4: no explicit public test seam or reason it survives the change.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "1c4b139b8806acaa1fea36149f4b9ce95a037c206e8629cc011e0e5ea03fd335" }, "judge_usage": { "input_tokens": 31803, "output_tokens": 2764, "cached_tokens": 29394, "wall_ms": 24280 }, "generation": { "n_planned": 3, "n_drawn": 4, "n_measured": 4, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.6375, "sd": 0.075, "judge_sd_mean": 0.118085, "variance_ratio": 0.635136, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "3aae854b2e2d1a3bd4b6e55e74b383fd9dcb117123876dbf83e67fa6d8672075", "status": "measured", "samples": [ 0.5, 0.75, 0.5, 0.5, 0.5 ], "judge_sample_hashes": [ "786e53ec806e332df0987077600e07cf0dbeffc98029197fbb71cf33b021f6e3", "02384acb493ae624643ebbfd5175e1cba6adbbd8dce3aacc0330f8fa36ac99e4", "e07e5bbb85bdf04c3bca003353d3bbf268f49e9628f82290fcadf35765b82d89", "78610882245dd9dcd2b0e09db4a7a0d1c41e7a9c43f31f514cc384e84d900dec", "fe0f8017a8c09ca94356e8b506242c50adf9399df6385e5411925d1a9ca9c400" ], "mean": 0.55, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2058, "output_tokens": 855, "cached_tokens": 1467, "wall_ms": 11665 }, "judge_usage": { "input_tokens": 32085, "output_tokens": 1735, "cached_tokens": 29618, "wall_ms": 18969 } }, { "draw_index": 1, "generation_hash": "8669bfa5a75ec92dce571df1bd045141ae1b5fded9dd7491501960c6fdf8978f", "status": "measured", "samples": [ 0.5, 0.5, 0.75, 0.5, 0.75 ], "judge_sample_hashes": [ "14303328fb9d7da76bc9fe3b6e1a5245964995b09c81476ab7b727531925e025", "8ba8c757bf4e131bf8fac6c6875467a6e8cead83e0b092f94e7822dbe01f6893", "1bd20828f5a8a6ff8ff3158dab0e7468cce8ad7d9f9fd409d9af6903e97e7115", "8232b1cb270cfbd643cc3c56dcdeb980ad7e3988775f2e99fedb93dd1f8ada0a", "8fb1d1068e8559e835f9eff949bdb026db7f209571c93103f46d53e0acbc909b" ], "mean": 0.6, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2058, "output_tokens": 813, "cached_tokens": 1852, "wall_ms": 12967 }, "judge_usage": { "input_tokens": 31743, "output_tokens": 2806, "cached_tokens": 29346, "wall_ms": 24135 } }, { "draw_index": 2, "generation_hash": "6f7aeeabaef0b3f9a2ae9594fc7d9f4e400ef3f696bd1dce05776c0a606e91f9", "status": "measured", "samples": [ 0.75, 0.5, 0.75, 0.75, 0.75 ], "judge_sample_hashes": [ "903f92e7dfb52e85c014169ae4639b337e16d899b5fc2935eb5ac7065eaf2def", "4e4c56a966ceb645994a3ff82bf48b07abc9443f1957ef9d1bc9994aef16d035", "188f67594337a23cdcb706a21cee4acc0c64489974ba69deffcc05f10c1343de", "e085dfb1d594da53f2bc2851d7477534b7bd2785b8eb90667e1b1480f44172a3", "00a900127a9c704cc622bd385a990671d9e77441f04a756886768651bec3bf6a" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2057, "output_tokens": 867, "cached_tokens": 1852, "wall_ms": 12171 }, "judge_usage": { "input_tokens": 31607, "output_tokens": 2828, "cached_tokens": 29238, "wall_ms": 24171 } }, { "draw_index": 3, "generation_hash": "9ca974ee631fa490ae85b9994003786db73240292140904ef30b20a0e16db1cf", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 0.5 ], "judge_sample_hashes": [ "4dec917873d5540506c079f572b1365f8943c07d9d070059aac3f5934d51a2ac", "f2748f290c16b9c3afb329cd3205bc25055e4fcb12daa63bbb0789ed545524e8", "32a6b6653583ec27d92397bade73c575985538704180db0025201f27ff2b91c1", "cac367af0346d62317df1eaeaecc290a35f07e22e24b543d8431d9c54a30376c", "a498a8e9eeaa84f1056758f7507591530002a0d82e6f7c288ad5d96a2a6e50b3" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2057, "output_tokens": 798, "cached_tokens": 1852, "wall_ms": 11375 }, "judge_usage": { "input_tokens": 31803, "output_tokens": 2764, "cached_tokens": 29394, "wall_ms": 24280 } } ], "issue408": { "attempt_id": "dd48f6ad46831a943dbf343b4f00a54df200f1ec07fe5f7c2db201abd622bd35", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "improve-codebase-architecture", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/improve-codebase-architecture/SKILL.md", "declared_name": "improve-codebase-architecture", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/improve-codebase-architecture/SKILL.md", "skill_blob_sha1": "44cc7d3136c44534d68ac523fa6cd37e759d2846", "skill_sha256": "552240a5ab5cec6b67c15dd1ad6e9f6962b1bca6ce870059a8c2713760c20392", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--improve-codebase-architecture.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:13:46.106Z", "configuration_unchanged": true }, "budget_cost_usd": 1.231904, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 4 } }, { "id": "adr", "mode": "with_skill", "outcome": "pass", "score": 0.85, "mean": 0.85, "stddev": 0.05, "samples": [ 0.5, 1, 1, 0.5, 1 ], "generation_hash": "38fec5152c9222c859e1a112953727f42bd05242a8fda1b55c587713fb772f61", "judge_sample_hashes": [ "07d34a885a1626b6e8327772a2397bc8c72dec25614d5c4731493d2f542a31bb", "0e994cfb07a877f4e9e22c46fb1c5979d3f5573716155742340d222b7e1fd6f5", "91220fa83ae50a1a7c19350bbc9b2419baae838125b5f04428e9d3035abecdd1", "e3c0d3ba578ebdb497b05143c73fee076fa11f19da658b5a1373054df8d70180", "221eb51e5497b77a40debd19beed2048379c9ada1ce33570a4e0c1e31781d184" ], "threshold": 0.75, "reason": "Met criterion 1 (uses Reservation term) and criterion 2 (identifies conflict with accepted ADR); missed criterion 3 since it allows reopening via superseding ADR without requiring evidence, and missed criterion 4 since it does not explicitly reject a broad merge solely for fewer modules, though it p", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "77825086b3dcf4079e23b83e6298c25a3025df84719947447f832f34a27d9a13" }, "judge_usage": { "input_tokens": 33448, "output_tokens": 1744, "cached_tokens": 30710, "wall_ms": 21265 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.85, "sd": 0.05, "judge_sd_mean": 0.182574, "variance_ratio": 0.273862, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "5434947641b7ad9a38f71338c63a128d6e8cb5f90f4393bfe5e9da404fe3e9c6", "status": "measured", "samples": [ 0.75, 1, 1, 0.75, 0.75 ], "judge_sample_hashes": [ "83544e0f063cbd5dfc8a384e988889dd344b97bb298ef50953ae533bef1cccea", "2a13ac1694a1f81198d2768702783ad637f4ab3b51d1c249c99300407fd3e07a", "6df771be666798e68e875b1ee1313cce1152fa61589a92806a346182bda3d3b9", "408a25be2c5ff5b817c5853acad2f57a5e976bbb78ed55846a6c1d76e3997f8e", "a6ab4ae9a6627d08e6b8ab4ebc605b232330ef8df87b3db3467d7af72838e62b" ], "mean": 0.85, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4168, "output_tokens": 1176, "cached_tokens": 3605, "wall_ms": 15860 }, "judge_usage": { "input_tokens": 33346, "output_tokens": 1264, "cached_tokens": 30626, "wall_ms": 17129 } }, { "draw_index": 1, "generation_hash": "bf47c9d7ac77a3e5103e6924d5f7dc6e64ce8eed922f8e5e494774146775d891", "status": "measured", "samples": [ 1, 1, 1, 0.75, 0.75 ], "judge_sample_hashes": [ "c3513d2e465f29cd635f19fbba44afe3dae6ee529fe6f722d4f6a81b71a27a0a", "28f560c17ddd8deda25a523f54b5d2dae8ed7b5b4199178982b799e8bcfa9f2a", "a01cdeba494add55fdf47350d508327ffc83133715e51f4649a9ead28dfb154c", "5971828cdf945e66cb0c016674dc87882b87ac239fbb1de3d1562410d55d8aac", "054dea868855727db8a78d5beefbabd1948815d99fb8afab02caae1a754b56bd" ], "mean": 0.9, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4168, "output_tokens": 1098, "cached_tokens": 3963, "wall_ms": 17467 }, "judge_usage": { "input_tokens": 32783, "output_tokens": 1401, "cached_tokens": 30178, "wall_ms": 18546 } }, { "draw_index": 2, "generation_hash": "38fec5152c9222c859e1a112953727f42bd05242a8fda1b55c587713fb772f61", "status": "measured", "samples": [ 0.5, 1, 1, 0.5, 1 ], "judge_sample_hashes": [ "07d34a885a1626b6e8327772a2397bc8c72dec25614d5c4731493d2f542a31bb", "0e994cfb07a877f4e9e22c46fb1c5979d3f5573716155742340d222b7e1fd6f5", "91220fa83ae50a1a7c19350bbc9b2419baae838125b5f04428e9d3035abecdd1", "e3c0d3ba578ebdb497b05143c73fee076fa11f19da658b5a1373054df8d70180", "221eb51e5497b77a40debd19beed2048379c9ada1ce33570a4e0c1e31781d184" ], "mean": 0.8, "stddev": 0.273861, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4168, "output_tokens": 1241, "cached_tokens": 3963, "wall_ms": 14557 }, "judge_usage": { "input_tokens": 33448, "output_tokens": 1744, "cached_tokens": 30710, "wall_ms": 21265 } } ], "issue408": { "attempt_id": "dd48f6ad46831a943dbf343b4f00a54df200f1ec07fe5f7c2db201abd622bd35", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "improve-codebase-architecture", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/improve-codebase-architecture/SKILL.md", "declared_name": "improve-codebase-architecture", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/improve-codebase-architecture/SKILL.md", "skill_blob_sha1": "44cc7d3136c44534d68ac523fa6cd37e759d2846", "skill_sha256": "552240a5ab5cec6b67c15dd1ad6e9f6962b1bca6ce870059a8c2713760c20392", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--improve-codebase-architecture.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:13:46.106Z", "configuration_unchanged": true }, "budget_cost_usd": 1.231904, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } }, { "id": "adr", "mode": "baseline", "outcome": "pass", "score": 0.866667, "mean": 0.866667, "stddev": 0.028868, "samples": [ 0.75, 1, 1, 0.5, 1 ], "generation_hash": "b09841770f93e386e4c1ef3cac4b1abeca9dfd6de493e8b776bda74ad3a47c0d", "judge_sample_hashes": [ "804bebdda9cf5d0253f175d70844ebd12b1ab45bf6e9bc9f0b8d38fd5e6e88a7", "0a4512d7ac5295872f38e02cdc76f8184367f526f87b95e6325700b6e146c9fa", "328f17ec80c7138212e3e5dbe1a98b76853da8477c520a83db5324863b068b2e", "bdf9cee0628b08aa32d097e76b08f12bde96004da5db6f61fa77330a0f9342bd", "6071a53dff3837848216edc8b1e166c59aaacdbee48a306be45d490f1fe062b7" ], "threshold": 0.75, "reason": "Meets criteria 1 (uses Reservation term), 2 (identifies ADR conflict), and 4 (does not propose broad merge); misses criterion 3 since it asks for an ADR rather than requiring evidence before reopening the decision.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "77825086b3dcf4079e23b83e6298c25a3025df84719947447f832f34a27d9a13" }, "judge_usage": { "input_tokens": 31952, "output_tokens": 1531, "cached_tokens": 29514, "wall_ms": 18115 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.866667, "sd": 0.028868, "judge_sd_mean": 0.223607, "variance_ratio": 0.129102, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "cd0fefadb67a274a105a48c8afabe80ce489a57375e611f6aa77650f50e25976", "status": "measured", "samples": [ 0.5, 1, 1, 1, 1 ], "judge_sample_hashes": [ "d831085be619ca05d656e5d7491afe69a76e8c7694b87ab2cec4594d415cb32c", "f8a18a4e8ea0f50a88880caf7d3a96606d6825778e721b89eef8485c1e6f4d3b", "c246b604d95780aab4c959242fbde76ab72686bbc0936129caf5eea5d275475f", "abe3145bab691e7247537fe22003bdf9241f8a512a3f479e1b1a7772c9f243c1", "664306c4ec40cc4812c3d212e65ef4d33ae34d33601f38eefc1e479ee3765f32" ], "mean": 0.9, "stddev": 0.223607, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2030, "output_tokens": 745, "cached_tokens": 1467, "wall_ms": 10191 }, "judge_usage": { "input_tokens": 31655, "output_tokens": 1970, "cached_tokens": 29278, "wall_ms": 20047 } }, { "draw_index": 1, "generation_hash": "95d85695dc1cb5039605d6a1cfa9aee7a3986cc017a96ab620f2d4e969fbd8d8", "status": "measured", "samples": [ 1, 0.75, 1, 1, 0.5 ], "judge_sample_hashes": [ "42e10e496da7a95b003fa71376c2d0b895ab5075316a4c97388a2b36b931fac1", "896935250801b6720c2e76aae3f96b6c6c128e5c97cd41126306c8772e8c0922", "e40eb6ad71c6fb0291c4dc78e176374c00a8ab9aae468e348cf33d943ed6718d", "1554343c560e5b9e226bdb67103160f703b4e12f94d80b16cfca27c0ec6be397", "2f668547055abd1f721e7f51b37d8336ab5c6e1373e0511c8cade4c434b5a8b0" ], "mean": 0.85, "stddev": 0.223607, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2029, "output_tokens": 810, "cached_tokens": 1825, "wall_ms": 11185 }, "judge_usage": { "input_tokens": 31915, "output_tokens": 1758, "cached_tokens": 29482, "wall_ms": 18787 } }, { "draw_index": 2, "generation_hash": "b09841770f93e386e4c1ef3cac4b1abeca9dfd6de493e8b776bda74ad3a47c0d", "status": "measured", "samples": [ 0.75, 1, 1, 0.5, 1 ], "judge_sample_hashes": [ "804bebdda9cf5d0253f175d70844ebd12b1ab45bf6e9bc9f0b8d38fd5e6e88a7", "0a4512d7ac5295872f38e02cdc76f8184367f526f87b95e6325700b6e146c9fa", "328f17ec80c7138212e3e5dbe1a98b76853da8477c520a83db5324863b068b2e", "bdf9cee0628b08aa32d097e76b08f12bde96004da5db6f61fa77330a0f9342bd", "6071a53dff3837848216edc8b1e166c59aaacdbee48a306be45d490f1fe062b7" ], "mean": 0.85, "stddev": 0.223607, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2029, "output_tokens": 835, "cached_tokens": 1825, "wall_ms": 10965 }, "judge_usage": { "input_tokens": 31952, "output_tokens": 1531, "cached_tokens": 29514, "wall_ms": 18115 } } ], "issue408": { "attempt_id": "dd48f6ad46831a943dbf343b4f00a54df200f1ec07fe5f7c2db201abd622bd35", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "improve-codebase-architecture", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/improve-codebase-architecture/SKILL.md", "declared_name": "improve-codebase-architecture", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/improve-codebase-architecture/SKILL.md", "skill_blob_sha1": "44cc7d3136c44534d68ac523fa6cd37e759d2846", "skill_sha256": "552240a5ab5cec6b67c15dd1ad6e9f6962b1bca6ce870059a8c2713760c20392", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--improve-codebase-architecture.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:13:46.106Z", "configuration_unchanged": true }, "budget_cost_usd": 1.231904, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } }, { "id": "report", "mode": "with_skill", "outcome": "pass", "score": 0.9, "mean": 0.9, "stddev": 0.070711, "samples": [ 1, 1, 1, 1, 0.75 ], "generation_hash": "4b0f09f14c95c924058c5e99dfa50ce25b8acf0523183c21a7acf3f328f872f6", "judge_sample_hashes": [ "dde3effcaae9b6d1dcb084159e973635eb7af480aaace908957bcc1667fe2b4a", "3726ccc1473927b4c903097153e44dc60934c0aa0e91956994668fbc8ff7c277", "68dbecf5f12e55d4278e7da3334553662fd4a37b96a0cf50214a9350ab5e444d", "177b00cf3b46847001feabfda2c3b0ef77dcd18c8c2e84990cfd3a2e37e2b78f", "0db88d1c591f08964b797631823b7f36f88025e010854ebec00eb9ff70857709" ], "threshold": 0.75, "reason": "Names involved modules and concrete friction (shallow pass-through, deletion test, leaked knowledge); proposes a deeper domain-level interface; describes benefits and before/after relationship; ends by asking which candidate to explore rather than claiming completion.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "5e132f027de3ee4e4626966671c299b45f6e84fcd04427e2896991538014c3e7" }, "judge_usage": { "input_tokens": 34734, "output_tokens": 1477, "cached_tokens": 31738, "wall_ms": 18094 }, "generation": { "n_planned": 3, "n_drawn": 4, "n_measured": 4, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.9, "sd": 0.070711, "judge_sd_mean": 0.118085, "variance_ratio": 0.598814, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "7367def07675091023a8b0a58ca180cf8052770bef4b0399db91456c8151e9ea", "status": "measured", "samples": [ 0.75, 1, 0.75, 1, 1 ], "judge_sample_hashes": [ "a426c7a10fd5cac49443cebe72512f514828d62cd73d005bc24a5aa3567a5c49", "a21c77829af3185d43deea23247ab1c17392c4dbec438130ef52f5e32d240b64", "f6825381076aec973b8ac163c4911b2ef174e0bee252f3a1fb0314fd8c575585", "87218a3bc0d98a4e9db57d516b7f167047db3dffc259ec0a27da292e2aae6e9b", "73e9df6c7470fa93789f48b3a87df64d29f0d442509a941f14b1798af2c808eb" ], "mean": 0.9, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4167, "output_tokens": 1690, "cached_tokens": 3605, "wall_ms": 19685 }, "judge_usage": { "input_tokens": 35167, "output_tokens": 2298, "cached_tokens": 32086, "wall_ms": 21947 } }, { "draw_index": 1, "generation_hash": "6ca827cac5aabb081804e18e730a46b25a01ba9e04fea1f0a05cd27685174f3c", "status": "measured", "samples": [ 0.75, 1, 1, 1, 1 ], "judge_sample_hashes": [ "629f03c1c67a0783a57c4f54715d972395b3f6dd7ccb3d00cc69519a0d72918d", "b6fb1585e6b443d32712b6032c50887f8eef00e9339f710df3d2d34ca35726b4", "ba11ce776f925cbc575cabe97d19f9ec4dabea17a1a2607f95b355d1c0741c50", "fba1bcbfe9ddd7cdcb53018f06b05972a2991a4591b2f8e717e2e233205718a1", "d8e5182af203b2e2c29af8da5e25861759ce1ca6dd627de3a8b0daa7b8d63161" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4166, "output_tokens": 1613, "cached_tokens": 3962, "wall_ms": 19186 }, "judge_usage": { "input_tokens": 35156, "output_tokens": 857, "cached_tokens": 32074, "wall_ms": 15156 } }, { "draw_index": 2, "generation_hash": "3314f6aa7cb4b13f30c2de3635f55fb1770ccf232f17f9421708b190d1f42f38", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 1, 0.75 ], "judge_sample_hashes": [ "248aaf4bf06c9d5b985b997052986bab8dc44d115a83d9852d3e41aa5be580c2", "26e3c6d309f83e924aca150bd334647c9c56422ca000d526ddd08b6ac1d8f643", "4f79292c30ab1a0a8f51df454d044bfc442b3fbb22012d08dbae4f34cc3a15b6", "d9ecdf9a3ba1b8843161b31d8449d749b5b00c08717ecb0617eaaa49ff8d1eff", "e7e3e28e2e5f17ee08b18d4af5a2958e42e9e547bc9550faeddc55ce3a03b79a" ], "mean": 0.8, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4166, "output_tokens": 1794, "cached_tokens": 3962, "wall_ms": 23270 }, "judge_usage": { "input_tokens": 34577, "output_tokens": 2114, "cached_tokens": 31614, "wall_ms": 22632 } }, { "draw_index": 3, "generation_hash": "4b0f09f14c95c924058c5e99dfa50ce25b8acf0523183c21a7acf3f328f872f6", "status": "measured", "samples": [ 1, 1, 1, 1, 0.75 ], "judge_sample_hashes": [ "dde3effcaae9b6d1dcb084159e973635eb7af480aaace908957bcc1667fe2b4a", "3726ccc1473927b4c903097153e44dc60934c0aa0e91956994668fbc8ff7c277", "68dbecf5f12e55d4278e7da3334553662fd4a37b96a0cf50214a9350ab5e444d", "177b00cf3b46847001feabfda2c3b0ef77dcd18c8c2e84990cfd3a2e37e2b78f", "0db88d1c591f08964b797631823b7f36f88025e010854ebec00eb9ff70857709" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 4167, "output_tokens": 1615, "cached_tokens": 3962, "wall_ms": 18789 }, "judge_usage": { "input_tokens": 34734, "output_tokens": 1477, "cached_tokens": 31738, "wall_ms": 18094 } } ], "issue408": { "attempt_id": "dd48f6ad46831a943dbf343b4f00a54df200f1ec07fe5f7c2db201abd622bd35", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "improve-codebase-architecture", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/improve-codebase-architecture/SKILL.md", "declared_name": "improve-codebase-architecture", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/improve-codebase-architecture/SKILL.md", "skill_blob_sha1": "44cc7d3136c44534d68ac523fa6cd37e759d2846", "skill_sha256": "552240a5ab5cec6b67c15dd1ad6e9f6962b1bca6ce870059a8c2713760c20392", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--improve-codebase-architecture.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:13:46.106Z", "configuration_unchanged": true }, "budget_cost_usd": 1.231904, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 4 } }, { "id": "report", "mode": "baseline", "outcome": "borderline", "score": 0.7, "mean": 0.7, "stddev": 0.05, "samples": [ 0.5, 1, 1, 0.5, 0.75 ], "generation_hash": "4c23733e32b42f5fedb66ed1baf8acaa6790f8a9659e557c6d8dd8f0bf557980", "judge_sample_hashes": [ "4f0a69f0e1d265589228362031292bbb85f7efa8f203b5d8dbaba58c8c444926", "72107b405a713322913f925272dd5bd594a56862dcecb9737a6862cb32c19f98", "067e0f3640caed284ca3dcea144137d16722c10448c37f3298c10271f7cf4df3", "332c7d1bd404c908c90282c3736ac4bd5f9aae95807b2a120bbd7535b384fea3", "fff121cae6cc75ac05f7d92f7936cf2b2e3142ece3f6709d052df025a21dad1f" ], "threshold": 0.75, "reason": "Names modules and concrete friction (pass-through, change amplification, leakage) and proposes a 4-5 method domain interface, but does not clearly state the benefits of the change or ask which option direction to explore.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "5e132f027de3ee4e4626966671c299b45f6e84fcd04427e2896991538014c3e7" }, "judge_usage": { "input_tokens": 33141, "output_tokens": 3249, "cached_tokens": 30466, "wall_ms": 26398 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 0.7, "sd": 0.05, "judge_sd_mean": 0.166245, "variance_ratio": 0.300761, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "d954abbe34c2c479b42512e20c59ebaaa9a274e676337b847143a254424d8719", "status": "measured", "samples": [ 0.75, 0.75, 0.5, 0.75, 0.75 ], "judge_sample_hashes": [ "c8fe94bc26d0c55f4a11335330942d6330b7c0bf991c61efcc6b382f924d9453", "ba96522a184cad0ece367d49573f3f74d5c4d992e91781c4bd8f6ed275d87b1d", "8b53ad3a05e802578d91fbbfed7a2dd5ec92f4cae6b1a249e51bd0db7b8a0947", "8905f01c20d256052176f4f45140db8155615a4d3f01717dbcb038450fbdcf77", "819d5d067ab3c8e417960bdf0484c9774fec8eb26a233028c2215934c70c6c8d" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2027, "output_tokens": 1108, "cached_tokens": 1467, "wall_ms": 14579 }, "judge_usage": { "input_tokens": 33915, "output_tokens": 2213, "cached_tokens": 31082, "wall_ms": 21827 } }, { "draw_index": 1, "generation_hash": "29e6330b269049a4c1f093b63cbb108768cf4827f5ade89bb1b8154906a76bef", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.5, 0.5 ], "judge_sample_hashes": [ "0aa21efe6f35c27b7acc8dc11d8bf26bc58a283fa099682d79b21c6f37e4c991", "2cc8e3d0983e2e82fe39f729a3ce9078edf11d616b872fb25332f9445026311b", "e1c5416b4ff350a35c7a1753914c39726b70be33a2a1c97f8c1abed822875092", "99133d3c4a313ed94c62228899ff5da6703333bb4940448c2d3748196bf42fcb", "6283e020d18b3c3f4593535fa44050732b790aaf0e7a29cf17c4dfd92e5cf739" ], "mean": 0.65, "stddev": 0.136931, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2029, "output_tokens": 882, "cached_tokens": 1824, "wall_ms": 11372 }, "judge_usage": { "input_tokens": 32441, "output_tokens": 2846, "cached_tokens": 29906, "wall_ms": 24363 } }, { "draw_index": 2, "generation_hash": "4c23733e32b42f5fedb66ed1baf8acaa6790f8a9659e557c6d8dd8f0bf557980", "status": "measured", "samples": [ 0.5, 1, 1, 0.5, 0.75 ], "judge_sample_hashes": [ "4f0a69f0e1d265589228362031292bbb85f7efa8f203b5d8dbaba58c8c444926", "72107b405a713322913f925272dd5bd594a56862dcecb9737a6862cb32c19f98", "067e0f3640caed284ca3dcea144137d16722c10448c37f3298c10271f7cf4df3", "332c7d1bd404c908c90282c3736ac4bd5f9aae95807b2a120bbd7535b384fea3", "fff121cae6cc75ac05f7d92f7936cf2b2e3142ece3f6709d052df025a21dad1f" ], "mean": 0.75, "stddev": 0.25, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2028, "output_tokens": 974, "cached_tokens": 1824, "wall_ms": 13643 }, "judge_usage": { "input_tokens": 33141, "output_tokens": 3249, "cached_tokens": 30466, "wall_ms": 26398 } } ], "issue408": { "attempt_id": "dd48f6ad46831a943dbf343b4f00a54df200f1ec07fe5f7c2db201abd622bd35", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "improve-codebase-architecture", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/improve-codebase-architecture/SKILL.md", "declared_name": "improve-codebase-architecture", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/improve-codebase-architecture/SKILL.md", "skill_blob_sha1": "44cc7d3136c44534d68ac523fa6cd37e759d2846", "skill_sha256": "552240a5ab5cec6b67c15dd1ad6e9f6962b1bca6ce870059a8c2713760c20392", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--improve-codebase-architecture.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:13:46.106Z", "configuration_unchanged": true }, "budget_cost_usd": 1.231904, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "with_skill", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "candidate", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "candidate", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "adr", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "adr", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "with_skill", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "with_skill", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "report", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "report", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } } ], "aggregates": { "with_skill": { "case_count": 3, "pass_count": 2, "borderline_count": 1, "mean_score": 0.836667, "stddev": 0.070946 }, "baseline": { "case_count": 3, "pass_count": 1, "borderline_count": 1, "mean_score": 0.734722, "stddev": 0.118464 }, "band_rule": "per arm: the sample standard deviation (n-1) of the per-case means across the included cases; the comparison band is the quadrature sum of the two arms; null where an arm has fewer than two included cases" } }, "comparison": { "with_skill_score": 0.836667, "baseline_score": 0.734722, "delta": 0.101945, "delta_uncertainty": 0.138084 }, "verification_level": "TESTED", "receipt_hash": "7f25e71e3e4c42264e38229f75a799a7aa13bd09850a7813abc7d41fa2c17a53", "economics": { "basis": "metered-equivalent", "surface": "claude-cli", "with_skill": { "call_count": 12, "mean_input_tokens": 4178.75, "mean_output_tokens": 1342.25, "mean_cost_usd_per_call": 0.04356, "median_wall_ms": 17868, "wall_ms_p25": 15050, "wall_ms_p75": 19435.5, "wall_ms_iqr": 4385.5 }, "baseline": { "call_count": 10, "mean_input_tokens": 2040.2, "mean_output_tokens": 868.7, "mean_cost_usd_per_call": 0.025535, "median_wall_ms": 11520, "wall_ms_p25": 11185, "wall_ms_p75": 12967, "wall_ms_iqr": 1782 }, "skill_incremental_cost_usd_per_call": 0.018025, "skill_incremental_cost_usd_per_1k_calls": 18.025, "output_tokens_delta": 473.55, "median_wall_ms_delta": 6348, "judge_excluded": true, "judge_overhead": { "note": "Measurement overhead imposed by Driftproof, NOT a cost of running the skill. Excluded from every field above. Counted over every draw sent to the judge, each once: the usage its answered judge samples reported, at the frozen judge price. Retried attempts and samples that reported no usage are not in it.", "total_cost_usd": 0.09898, "case_rows_measured": 6, "judged_draws_counted": 22, "judged_draws_missing_usage": 0 }, "notes": { "absolute_cost": "Absolute per-call cost on a CLI surface includes a fixed harness preamble (observed ~25k input tokens on claude-cli, ~11k on codex) that we do not control. It is identical in the with-skill and baseline arms, so it cancels in the Δ figures — read the incremental columns, not the absolute ones.", "cache_pricing": "Cached input tokens are costed at the list input rate (an over-estimate where caching is heavy); cached_tokens is recorded per call so a reader can recompute.", "latency": "observed on subscription CLI surface, indicative" } } }