{ "schema_version": "0.13", "generation_sampled": true, "skill": { "name": "tdd", "version": "0.0.0", "content_hash": "cc5466e20027f223f3e4fc68b7d7fdf6d1f9c6f79f7e6dd4cad440c249e38154", "tokens": 906 }, "suite": { "format": "agentskills.io/evals", "suite_hash": "7c3eaed6b72fd12c19b5fbe64a846026eed62b88b215b0bd090b4924b94555bb", "case_count": 3, "canary": "041f84b4-29d6-83c7-f5d0-5b48c1e6e7ba" }, "run": { "model_id": "claude-opus-5-5", "model_release_date": null, "provider": "anthropic", "surface": "claude-cli", "runner_version": "0.22.2", "date_utc": "2026-10-18T07:28:16.569Z", "generated_at": "2026-10-18T07:17:33.837Z", "registry": "registered", "transcripts": "retained-local", "judge": { "samples": 5, "temperature": null, "sampling": "surface-controlled", "surface": "claude-cli", "model_id": "claude-haiku-5-5", "prompt_template_hash": "82586d1e44f84df2b8ce5f18ac777866465cb5ad9686503c93d9776c93c344ed", "harness": { "name": "claude-code", "version": "2.1.287" }, "isolation": "eval-user", "endpoint": { "custom": false, "sha256": null, "resolved_by": "cli", "routing": { "ANTHROPIC_BEDROCK_BASE_URL": { "set": false, "sha256": null }, "ANTHROPIC_VERTEX_BASE_URL": { "set": false, "sha256": null }, "CLAUDE_CODE_USE_BEDROCK": { "set": false, "sha256": null }, "CLAUDE_CODE_USE_VERTEX": { "set": false, "sha256": null }, "HTTPS_PROXY": { "set": false, "sha256": null } } }, "code_revision": "judge-code-bb552899a1df887e", "reported_models": [ "claude-haiku-5-5" ] }, "answered_by": { "kind": "model", "attested": true, "reported_model": "claude-opus-5-5", "reported_models": [ "claude-haiku-5-5", "claude-opus-5-5" ], "isolation": "eval-user" }, "capture": { "mode": "text" }, "harness": { "name": "claude-code", "version": "2.1.287" }, "pricing_snapshot": { "frozen_at": "2026-10-18T07:28:16.569Z", "source": "config/models.json", "currency": "USD", "models": { "claude-opus-5-5": { "input_per_mtok": 4, "output_per_mtok": 20, "registered": true }, "claude-haiku-5-5": { "input_per_mtok": 0.1, "output_per_mtok": 0.5, "registered": true } }, "note": "Prices frozen at run time. Every derived cost in this receipt is computed from THIS snapshot, never from the live registry, so the receipt keeps its meaning when registry prices later change." }, "arms": { "with_skill": { "counts": { "generations_per_arm": 3 } } }, "counts": { "judge_samples_per_generation": 5 } }, "results": { "cases": [ { "id": "test", "mode": "with_skill", "outcome": "pass", "score": 1, "mean": 1, "stddev": 0, "samples": [ 1, 1, 1, 1, 1 ], "generation_hash": "6b200b8195374bdd6e2b52e2a140c28358466a94cf2d13981200181e725cfd2f", "judge_sample_hashes": [ "0d9bf37a88ed3f353804b1878153d8cb6b9a10bb0a272bf50342e87c79a41932", "6298c1d2cc1a32eda216929ca022f3b29565370cee1832c17fcfeccf67d0683a", "42083dc024b430497f4f1f60e4729ba199e8276e0dd302f51c6a65beb3fb883b", "83ca43e506a9033f2aee7dcf985e1ca329cff1aef8594c44bccbe83defaf5fb3", "13ac30a04a8ea2d0e39572abc22df869f8a1a7c6ac95c822409fe4071971f3f2" ], "threshold": 0.75, "reason": "Meets all four rubric points: tests the public reserve(stock, amount) seam, asserts the literal 2, proposes running the failing test before implementation, and describes only one minimal green step rather than all tests at once.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "847df93a622e96b1f8b49f925e4b6a06434747fe5d3fd71905690b37d7f3b0cc" }, "judge_usage": { "input_tokens": 31503, "output_tokens": 437, "cached_tokens": 29154, "wall_ms": 14178 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 1, "sd": 0, "judge_sd_mean": 0, "variance_ratio": null, "variance_ratio_unavailable": "judge_sd_zero", "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "e1d8f8a4d02db7c536a2d9357efc07149ad8934d0fe2cd0fa346cb3876ef3fa1", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "0127af6fca6436b397d07a0f83d49bc49250c41a0df2913fc2538512794b2a0b", "d3b4d277dd147c8a9a616801f05cbd786b49a36859350dc127bd35e582c5a77a", "a99646f5e81afd7a901d6db9d3ffb2a27de5b660637817af41eafb7c1d7b9cb3", "0a090e97e1fa73c45f7d3b56c02d8ccf17dc450dd7b5fc0e4ca69d7d77091a1e", "c8c2b014f428c9fff891c3716d8e874ab9fe37a8a1a42666cfb6366686690eb8" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 3198, "output_tokens": 776, "cached_tokens": 544, "wall_ms": 12334 }, "judge_usage": { "input_tokens": 31146, "output_tokens": 1407, "cached_tokens": 28870, "wall_ms": 17733 } }, { "draw_index": 1, "generation_hash": "02945f70bd463c1aba219568577a466ce22ba8e59b12e4d4d49280ad11490dbf", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "9ba540ca6d701d0a5e985324a9303bb665d332b67df7d0936876caf8e148e7a9", "695c44554ba98415361f4490e7cd7689ca4561a4652411deb11a773531c68680", "f65b44ca0ee530bbb377d7be0544f8422739256619c485c385b411464f6eb703", "410a9b8a21653dce443e7acae6b12b8e958d7abbed8cacdb8feda7ab1d1580f1", "7534a97bb1ee3a0e3667811c53b58832b069ca4afd7b71fccfaa5f744ebbe50f" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 3198, "output_tokens": 807, "cached_tokens": 2993, "wall_ms": 10362 }, "judge_usage": { "input_tokens": 31714, "output_tokens": 1112, "cached_tokens": 29322, "wall_ms": 16938 } }, { "draw_index": 2, "generation_hash": "6b200b8195374bdd6e2b52e2a140c28358466a94cf2d13981200181e725cfd2f", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "0d9bf37a88ed3f353804b1878153d8cb6b9a10bb0a272bf50342e87c79a41932", "6298c1d2cc1a32eda216929ca022f3b29565370cee1832c17fcfeccf67d0683a", "42083dc024b430497f4f1f60e4729ba199e8276e0dd302f51c6a65beb3fb883b", "83ca43e506a9033f2aee7dcf985e1ca329cff1aef8594c44bccbe83defaf5fb3", "13ac30a04a8ea2d0e39572abc22df869f8a1a7c6ac95c822409fe4071971f3f2" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 3198, "output_tokens": 847, "cached_tokens": 2993, "wall_ms": 12602 }, "judge_usage": { "input_tokens": 31503, "output_tokens": 437, "cached_tokens": 29154, "wall_ms": 14178 } } ], "issue408": { "attempt_id": "e20ea3cfccc4f094125ed7bd434ec74e18e62da66face7ab67daddcac39753c7", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "tdd", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/tdd/SKILL.md", "declared_name": "tdd", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/tdd/SKILL.md", "skill_blob_sha1": "01eadaa34e7c9a63a67d6dc3cce1cc81b0e49985", "skill_sha256": "5505d25bbaa8fc14a79e4d20163936958992a2995b7f1dd1d6197edc9ed722e0", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--tdd.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:17:33.722Z", "configuration_unchanged": true }, "budget_cost_usd": 0.94691, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "test", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } } }, { "id": "test", "mode": "baseline", "outcome": "pass", "score": 1, "mean": 1, "stddev": 0, "samples": [ 1, 1, 1, 1, 1 ], "generation_hash": "f87eaaf18b037ad5329cc0f20d43795ddbe1c81d32409cc62fc5c19f277b467b", "judge_sample_hashes": [ "ef767f23b12470ff18a0d6b2b48e7689a0e1f010c233a35895bfde50fb39c66e", "045f5d670528792f70631ed4eb9311b5dc673b05cea7a84174f958e102f02b31", "1b5eda6e3fa39ae364229944cca6672291c711bdd15ee0825901439509077dc5", "1ae673ce0cae4320794eadbc6dd59faedfe06bf634c4b5103ea18e7e1e017779", "ae2bdf36fe9f47df9f223c03b8ab0578dedaf541505a3d819de628cdc29524a5" ], "threshold": 0.75, "reason": "Tests the public reserve(stock, amount) interface, asserts the given value 2 for (4, 2), proposes the failing test before implementation, and describes a single minimal green step (return 2).", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "847df93a622e96b1f8b49f925e4b6a06434747fe5d3fd71905690b37d7f3b0cc" }, "judge_usage": { "input_tokens": 31464, "output_tokens": 733, "cached_tokens": 29122, "wall_ms": 15912 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 1, "sd": 0, "judge_sd_mean": 0, "variance_ratio": null, "variance_ratio_unavailable": "judge_sd_zero", "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "20b52bc26d16cbdf0a87376794fbf7979b3ffa0f29c8fcccb2c53c7ccc505699", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "a538c1b532c13576e0768be637c40a9093f362450daf7e10b5cd9e7e3a785ee3", "b09764ed3d019b02bc644ee0ff24c97be02557bf5e53bac4bbf198f05ff4613d", "0b5a2ab7b8edae4aedb6fc000b51c9c48f0b11df0670c9bd198f901a2ea9aeff", "03f85e674719e53c8b51770a60ddf39993fb74db0397428daed251d3f085a6ea", "f99f15067f1b086f7c020f4f068b10c6c213f2567580a4ae18c5b5b68edc3c1d" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2038, "output_tokens": 684, "cached_tokens": 1467, "wall_ms": 9961 }, "judge_usage": { "input_tokens": 31564, "output_tokens": 1313, "cached_tokens": 29202, "wall_ms": 17004 } }, { "draw_index": 1, "generation_hash": "638c15cb9c60bf50ce3c4ef8bdd90a15a221276256bca7ee7b988dd1087ac29f", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "4f5677d21fee67b5325c774e7e7b8d8e75d3a16f58d33d87446e06064a362533", "8641877de54cd96ff61b9cbf4bf87b2246a6254b7ee6ab0e08b62e3af4850228", "3c8ceb56b47ac3555210b114b83002b5dd8de3e760aeedb630d5a81d7c8ebdd5", "2b7b73c23e46ceba6fa917287b1aa6b754d187a11b3fabcbe96737f26eae676e", "44aa451b9757a8c923ba97391a424548ba0d0507f0621fb772699b30aa3caf16" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2039, "output_tokens": 597, "cached_tokens": 1834, "wall_ms": 8109 }, "judge_usage": { "input_tokens": 31024, "output_tokens": 991, "cached_tokens": 28770, "wall_ms": 16746 } }, { "draw_index": 2, "generation_hash": "f87eaaf18b037ad5329cc0f20d43795ddbe1c81d32409cc62fc5c19f277b467b", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "ef767f23b12470ff18a0d6b2b48e7689a0e1f010c233a35895bfde50fb39c66e", "045f5d670528792f70631ed4eb9311b5dc673b05cea7a84174f958e102f02b31", "1b5eda6e3fa39ae364229944cca6672291c711bdd15ee0825901439509077dc5", "1ae673ce0cae4320794eadbc6dd59faedfe06bf634c4b5103ea18e7e1e017779", "ae2bdf36fe9f47df9f223c03b8ab0578dedaf541505a3d819de628cdc29524a5" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2038, "output_tokens": 662, "cached_tokens": 1834, "wall_ms": 8929 }, "judge_usage": { "input_tokens": 31464, "output_tokens": 733, "cached_tokens": 29122, "wall_ms": 15912 } } ], "issue408": { "attempt_id": "e20ea3cfccc4f094125ed7bd434ec74e18e62da66face7ab67daddcac39753c7", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "tdd", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/tdd/SKILL.md", "declared_name": "tdd", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/tdd/SKILL.md", "skill_blob_sha1": "01eadaa34e7c9a63a67d6dc3cce1cc81b0e49985", "skill_sha256": "5505d25bbaa8fc14a79e4d20163936958992a2995b7f1dd1d6197edc9ed722e0", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--tdd.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:17:33.722Z", "configuration_unchanged": true }, "budget_cost_usd": 0.94691, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "test", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } }, { "id": "coupling", "mode": "with_skill", "outcome": "pass", "score": 1, "mean": 1, "stddev": 0, "samples": [ 1, 1, 1, 1, 1 ], "generation_hash": "aafeab7b65ef2c3efc2ee8f77d44f40b7b981ebcd530f22d5e0a78f7baf92741", "judge_sample_hashes": [ "1245ed53cad2501065665b3cc58c740c622a5bf9e9bb559be6acfd535dc3ca20", "849b1213c502957760bdc633f11eb55e9db453c0d6854f4d94f64259d629addd", "79e566924841b991c4de6f9223f7d622b7e52185c600f5852e01ecacf9dda74f", "7aae86e976c849d2543447eb1d08d461d0bd21a2a694b5eb0df62505b077b9d9", "4abb1f624e5922ba531ee3c2cc63194e170c3514b4524c005f66e5304b6c39af" ], "threshold": 0.75, "reason": "The response flags spying on the private normalizeCart helper (criterion 1), calls the items.reduce expectation tautological (criterion 2), replaces it with a literal total of 15 (criterion 3), and keeps the test at the public createOrder(...).total interface (criterion 4).", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "f14c337adea9a16322a0eae4532bbe406845104200ef17f000535c82eb71c59c" }, "judge_usage": { "input_tokens": 32228, "output_tokens": 1246, "cached_tokens": 29734, "wall_ms": 16487 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 1, "sd": 0, "judge_sd_mean": 0, "variance_ratio": null, "variance_ratio_unavailable": "judge_sd_zero", "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "bf93f6724334e1fcc20e4bd347cec78d9530217b551a4060dc59c77168dee2a3", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "385198eb2ffd57c424188df48de0f5a2ac6db70a62ab1ba7a5cffbe3430f54fd", "f0436e9f2f81e3a6f4a045d1f6e37f8ddc04ea22e4e6d3c3abbf5a0b9abc9dab", "f62e27580e73afd26e3693f4bbb24656ced1eacc8338177c0216734066f9d927", "1d18c2ae0934445562b84765b215c477ca221bf351f344d9402c4eb521cbbe61", "4cd2e51c861cefe533340787e1cac00733670ee9d37afd52459588a5186c6534" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 3196, "output_tokens": 787, "cached_tokens": 2626, "wall_ms": 10727 }, "judge_usage": { "input_tokens": 31757, "output_tokens": 874, "cached_tokens": 29358, "wall_ms": 16078 } }, { "draw_index": 1, "generation_hash": "395959456751508c5283ec5246f6e505a1f947c1b4751eb13f55e1eb0de03a6e", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "265696981af564a641b60de0d4fd4c5d273c3c78ac96dbc64ee1e1bcfc231d64", "8c25760967eb1ac7e1b419a6e5a078a3ccab6c8aa5e6044760940cccb14ea851", "70688e3e3f0577aedf15a70325ca693e15915d6ad4a7c734d94e457ef0113f6e", "25259ae4ae8740461e143244781a5c39c3cd8dd8d1fd96b9b9b0beec235463a3", "69deda43e40df2b3920fbe81b2f7dc95397358051f33945e919dac7d3d4bb35b" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 3196, "output_tokens": 911, "cached_tokens": 2991, "wall_ms": 11663 }, "judge_usage": { "input_tokens": 32202, "output_tokens": 962, "cached_tokens": 29714, "wall_ms": 16936 } }, { "draw_index": 2, "generation_hash": "aafeab7b65ef2c3efc2ee8f77d44f40b7b981ebcd530f22d5e0a78f7baf92741", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "1245ed53cad2501065665b3cc58c740c622a5bf9e9bb559be6acfd535dc3ca20", "849b1213c502957760bdc633f11eb55e9db453c0d6854f4d94f64259d629addd", "79e566924841b991c4de6f9223f7d622b7e52185c600f5852e01ecacf9dda74f", "7aae86e976c849d2543447eb1d08d461d0bd21a2a694b5eb0df62505b077b9d9", "4abb1f624e5922ba531ee3c2cc63194e170c3514b4524c005f66e5304b6c39af" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 3197, "output_tokens": 915, "cached_tokens": 2991, "wall_ms": 12994 }, "judge_usage": { "input_tokens": 32228, "output_tokens": 1246, "cached_tokens": 29734, "wall_ms": 16487 } } ], "issue408": { "attempt_id": "e20ea3cfccc4f094125ed7bd434ec74e18e62da66face7ab67daddcac39753c7", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "tdd", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/tdd/SKILL.md", "declared_name": "tdd", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/tdd/SKILL.md", "skill_blob_sha1": "01eadaa34e7c9a63a67d6dc3cce1cc81b0e49985", "skill_sha256": "5505d25bbaa8fc14a79e4d20163936958992a2995b7f1dd1d6197edc9ed722e0", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--tdd.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:17:33.722Z", "configuration_unchanged": true }, "budget_cost_usd": 0.94691, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "test", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } } }, { "id": "coupling", "mode": "baseline", "outcome": "pass", "score": 1, "mean": 1, "stddev": 0, "samples": [ 1, 1, 1, 1, 1 ], "generation_hash": "533497f31e27bd9a1bc8c44f468746aa03220772ec0a3fead9c7170977bac408", "judge_sample_hashes": [ "b81cd569bdbedbc3312050202bed28211ab09cbdd9425fb791dc346a0f4df616", "4f77fc6f30c1a318ac08c863e8aa92b262d83b5c623ea31702c179213fb7c635", "e25a4931e5d9881fa7f43e0789c59be20b0c278c72b48ff44c9b64cf11d08836", "a862ba30d662b257841561ff50aa417fb83c9a4c6744ac3673074a652b30632a", "e25a4931e5d9881fa7f43e0789c59be20b0c278c72b48ff44c9b64cf11d08836" ], "threshold": 0.75, "reason": "Identifies spying on private normalizeCart as implementation coupling, flags the duplicated-formula expectation as a tautology, replaces it with a literal observable total of 15, and keeps the test at the public order interface.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "f14c337adea9a16322a0eae4532bbe406845104200ef17f000535c82eb71c59c" }, "judge_usage": { "input_tokens": 31828, "output_tokens": 700, "cached_tokens": 29414, "wall_ms": 15443 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 1, "sd": 0, "judge_sd_mean": 0, "variance_ratio": null, "variance_ratio_unavailable": "judge_sd_zero", "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "43ffd9c94b95822615eec982533322fc8b02f327cf1e30e3ead21e12858dc9a8", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "466f64f4b431697fe777105582fea1a3bf7d1f1577831a8d58267e8e946cf5fa", "a789f447e66e532ea9535de15f8c84b72e7b4aa60d25877b91a60eae2471b87e", "d3d50f2f2033f70adbb5370f44c371967e0a2de16daf8e2ad6b9b270d6553e34", "a81d048c57a30d5326a357764425bee470605cff498cf0975f09e9a41db2c10f", "2f312a5fb9b59e0b14ef226a433c17264dda99b936a228445792eaf9481b806c" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2038, "output_tokens": 916, "cached_tokens": 1467, "wall_ms": 12120 }, "judge_usage": { "input_tokens": 32338, "output_tokens": 896, "cached_tokens": 29822, "wall_ms": 16390 } }, { "draw_index": 1, "generation_hash": "19219e98df1b4d5bfd94a75e25cd445e666554abfa4c398a00ef46e5edca82a3", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "deebdf2eaab91060c468841fbeede1fffa779cde95be8666cc8f5e2345c49e7d", "8a988f4c1c4a64afe2f7c43932b7e968eb1656c884f1c0e1de66507bc04c9b55", "91739117287ae5b4a7a70cec3abc968e0d326e5f1cffff93c3e1d43894a13ba7", "ef357df6effe18e02a8dd4ce540703a1f39fca28a6b8f9a2b97ad5150465c7fb", "5eaf754f933f216805af2c1ab7003271c310c1369748146f8f71bcef08a30160" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2037, "output_tokens": 811, "cached_tokens": 1832, "wall_ms": 12538 }, "judge_usage": { "input_tokens": 32011, "output_tokens": 1316, "cached_tokens": 29562, "wall_ms": 21338 } }, { "draw_index": 2, "generation_hash": "533497f31e27bd9a1bc8c44f468746aa03220772ec0a3fead9c7170977bac408", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "b81cd569bdbedbc3312050202bed28211ab09cbdd9425fb791dc346a0f4df616", "4f77fc6f30c1a318ac08c863e8aa92b262d83b5c623ea31702c179213fb7c635", "e25a4931e5d9881fa7f43e0789c59be20b0c278c72b48ff44c9b64cf11d08836", "a862ba30d662b257841561ff50aa417fb83c9a4c6744ac3673074a652b30632a", "e25a4931e5d9881fa7f43e0789c59be20b0c278c72b48ff44c9b64cf11d08836" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2037, "output_tokens": 776, "cached_tokens": 1832, "wall_ms": 10857 }, "judge_usage": { "input_tokens": 31828, "output_tokens": 700, "cached_tokens": 29414, "wall_ms": 15443 } } ], "issue408": { "attempt_id": "e20ea3cfccc4f094125ed7bd434ec74e18e62da66face7ab67daddcac39753c7", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "tdd", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/tdd/SKILL.md", "declared_name": "tdd", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/tdd/SKILL.md", "skill_blob_sha1": "01eadaa34e7c9a63a67d6dc3cce1cc81b0e49985", "skill_sha256": "5505d25bbaa8fc14a79e4d20163936958992a2995b7f1dd1d6197edc9ed722e0", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--tdd.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:17:33.722Z", "configuration_unchanged": true }, "budget_cost_usd": 0.94691, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "test", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 3 } }, { "id": "agreement", "mode": "with_skill", "outcome": "pass", "score": 1, "mean": 1, "stddev": 0, "samples": [ 1, 1, 1, 1, 1 ], "generation_hash": "f154da4bc62da152d812050d0e6cf6d78f3cbb5e2544d9e1bdd0c8dc0dfa95ea", "judge_sample_hashes": [ "5909476e7a3db3d84ad833aa33eb9393b25386f0c6774b5c8f9c3a6600ee3f18", "bfa0f867b0c95b2ae769715f41feb132cc9893dce40ea7d60144003e0375933f", "3f1cc6ae62d09b72b3e5e850df3655170278b8fef4e048c453fc5a9a269e4647", "87882d5211748ba76caf1b042674cebbcb87b4f4075c4b19479efc898dbf6ec2", "dbf440f07bc53f3b5890826139fd2fe0c3e721898033be99927f626248b98c42" ], "threshold": 0.75, "reason": "Meets all four rubric points: proposes agreeing a public refund seam first, names observable outcomes (refunded amount, status, balance), states what the seam misses, and avoids a private-method suite.", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "b929e24949e308000711696d10c4dc583e42cf959c548b22465339b899874586" }, "judge_usage": { "input_tokens": 31943, "output_tokens": 427, "cached_tokens": 29506, "wall_ms": 14273 }, "generation": { "n_planned": 3, "n_drawn": 3, "n_measured": 3, "n_unmeasured": 0, "stopping_reason": "min_reached", "mean": 1, "sd": 0, "judge_sd_mean": 0, "variance_ratio": null, "variance_ratio_unavailable": "judge_sd_zero", "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "243d8c97b24b146b5c047f741aa4763c18c37df527fda2f451af1792c8055861", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "62da128b8e5fbe1db5061afd39e127ee4430aae9f9cf1323e0e67bab3e4f6b23", "7a56ca5f47b7008db22bcdeb856ccb85126ffae7bad74ffef3191a47855623cc", "79835bab6f6e196ff88fdda378a7f6b9d5002254eca9b52dfa8197ffc880a04e", "c6095e896a5a7fd1cc0310debcf7c1d7e3bfbbf4d8029ca0ec04c9697b53b6bc", "48f792bbd5b98ab09283d5822a76453e7629990d8a0e711b76bb1a531f5e3e2c" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 3185, "output_tokens": 725, "cached_tokens": 2626, "wall_ms": 11530 }, "judge_usage": { "input_tokens": 31614, "output_tokens": 751, "cached_tokens": 29242, "wall_ms": 15937 } }, { "draw_index": 1, "generation_hash": "2376d7f99c4f9a0312d6b8ddbb5b5e9f5568498769b3ecd82561086fa2bb0645", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "065c528fcb12f22f5e076f403ea86da530cedeff9bd25e2db7d1c234c268faa5", "4d26aad8c3c4dee4729c1c67636443c113e468fc4a4d01a1eb740ac9ae240cf4", "ac4da81664a8f569db66b5fe6bd627deabf008cb102aa72e7fc0a742a920127c", "c45de8cc162ed6c40c02f1252b891e7e0589f41e23925e7118bd2a8d31830009", "af3d60dd9858d8595681f64a3a5b71a00e60549365168770e120cfddb48aac37" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 3184, "output_tokens": 763, "cached_tokens": 2980, "wall_ms": 10974 }, "judge_usage": { "input_tokens": 31712, "output_tokens": 953, "cached_tokens": 29318, "wall_ms": 15706 } }, { "draw_index": 2, "generation_hash": "f154da4bc62da152d812050d0e6cf6d78f3cbb5e2544d9e1bdd0c8dc0dfa95ea", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "5909476e7a3db3d84ad833aa33eb9393b25386f0c6774b5c8f9c3a6600ee3f18", "bfa0f867b0c95b2ae769715f41feb132cc9893dce40ea7d60144003e0375933f", "3f1cc6ae62d09b72b3e5e850df3655170278b8fef4e048c453fc5a9a269e4647", "87882d5211748ba76caf1b042674cebbcb87b4f4075c4b19479efc898dbf6ec2", "dbf440f07bc53f3b5890826139fd2fe0c3e721898033be99927f626248b98c42" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 3184, "output_tokens": 872, "cached_tokens": 2980, "wall_ms": 12686 }, "judge_usage": { "input_tokens": 31943, "output_tokens": 427, "cached_tokens": 29506, "wall_ms": 14273 } } ], "issue408": { "attempt_id": "e20ea3cfccc4f094125ed7bd434ec74e18e62da66face7ab67daddcac39753c7", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "tdd", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/tdd/SKILL.md", "declared_name": "tdd", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/tdd/SKILL.md", "skill_blob_sha1": "01eadaa34e7c9a63a67d6dc3cce1cc81b0e49985", "skill_sha256": "5505d25bbaa8fc14a79e4d20163936958992a2995b7f1dd1d6197edc9ed722e0", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--tdd.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:17:33.722Z", "configuration_unchanged": true }, "budget_cost_usd": 0.94691, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "test", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } } }, { "id": "agreement", "mode": "baseline", "outcome": "pass", "score": 0.908333, "mean": 0.908333, "stddev": 0.128128, "samples": [ 1, 1, 1, 1, 1 ], "generation_hash": "8e3b26b86ba8e9051d6cb2976b508db525a284f18eaedaab303637aefb9c7adf", "judge_sample_hashes": [ "3e944e5ec51893509006f0d8ccc87b02ee630d5068dc72cc53907f442cc846bc", "6674eaa6119df5dd49d39abb297ea8c061fd54a4fa8665a84ed7f9cd2c1ff0c0", "60f8082df1714e31dd1bb21f1ba97ffb299b2bbd7ef8026403525fcd88a9637a", "c4c67a9ecef46c40829fa533c48dc660702ac8faf8a0b088d3b1ab71a28004bb", "dc23bf79b943510070755203f386cd5a1bc5ca59b898f913b275b829a4be233c" ], "threshold": 0.75, "reason": "Criterion 1 met: it proposes agreeing a public RefundService seam first. Criterion 2 met: it names observable outcomes via RefundResult, public state, and gateway calls. Criterion 3 met: its Limits section states what the seam misses. Criterion 4 met: it explicitly pushes back on testing private int", "judge": { "model_id": "claude-haiku-5-5", "rubric_hash": "b929e24949e308000711696d10c4dc583e42cf959c548b22465339b899874586" }, "judge_usage": { "input_tokens": 33939, "output_tokens": 1103, "cached_tokens": 31102, "wall_ms": 17856 }, "generation": { "n_planned": 3, "n_drawn": 6, "n_measured": 6, "n_unmeasured": 0, "stopping_reason": "stabilised", "mean": 0.908333, "sd": 0.128128, "judge_sd_mean": 0.055902, "variance_ratio": 2.292011, "variance_ratio_unavailable": null, "n_truncated": 0, "draws": [ { "draw_index": 0, "generation_hash": "7857b18e14fe6bcbe6ac5b38b8c039f87fa6bcab568bd10df89d745f6ca29081", "status": "measured", "samples": [ 0.75, 0.75, 0.5, 0.75, 0.75 ], "judge_sample_hashes": [ "e4cd89490d6114d9e602b5ac7abbac83327d51fce07b8a06493fc32543c0382f", "5defc0759e5ecfdd4cb8b37cf45ec69cad0019a4d8279c1737e365fdea50bb98", "3f82c4c3059ab63b806171c6bfd1addbd31dc24cdaabef25ba46ccd34886ad4f", "a840dd7df9920dad741deba2c877373b53494c260e07b2c3c9649d04125d8bb6", "9c4e4d459e1921b875c6ee1a934a36fd195f0793569a0ffcb3a617d285f98e3a" ], "mean": 0.7, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2025, "output_tokens": 1025, "cached_tokens": 1467, "wall_ms": 15269 }, "judge_usage": { "input_tokens": 33138, "output_tokens": 1923, "cached_tokens": 30458, "wall_ms": 24007 } }, { "draw_index": 1, "generation_hash": "706e1e92bbf2a6b49c9150a3539c562ba142a1bfb6962e87aa08a02c44efccc1", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "8727e9d997be8c2730fd02956308d7147bab5f221682bcce36a2d08f19094cad", "875a1486dd69b264869e6d81375fefe2a8f43b3d920ffc343b67b8821bc2b597", "9435b970e11c80653e934ad6274f6d1c17375c31d06ba64eed7e6cf6bc32c330", "76113766ad73dcc9c3ea0f3641e0f7df23588cecc2d1770ff8558357cc1948c4", "81db80b47dfe5caadbd7d01f0381d334f7df4064a6fc0bbbf4951cd88555404b" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2026, "output_tokens": 1036, "cached_tokens": 1821, "wall_ms": 18866 }, "judge_usage": { "input_tokens": 32659, "output_tokens": 1552, "cached_tokens": 30078, "wall_ms": 17941 } }, { "draw_index": 2, "generation_hash": "449962b9d7783e581830ae5d2fbbc276b9746581d88e699c004504f8004be7b1", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "1ef6f045a6a5e7c3733ec28ea305061cea06e0f1f24b8a9d7620868f0d8428e6", "2f930d6124276f393b67f4fa4a6d7f8e490df6aee747d8b81e0ce9fe9c742cbd", "bcfbfce1a272b3424f7ae735a0762f87be7152f865e7c09bf3a40a86cbbb38d2", "bd7d02bccf0711dd60f6ef36297f53f8b910452691c7848dec5a9df8c66957d4", "752dfda5cba8893f57643a3c7cbd206ae9e350ab4b0cb7332711a13ee7ea19e3" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2026, "output_tokens": 1027, "cached_tokens": 1821, "wall_ms": 13962 }, "judge_usage": { "input_tokens": 32156, "output_tokens": 1614, "cached_tokens": 29674, "wall_ms": 18540 } }, { "draw_index": 3, "generation_hash": "2f6b078550221f3611bd7cfeae68d51d3363a9fa761748e7c59ede388e6afe92", "status": "measured", "samples": [ 0.75, 0.75, 0.75, 0.75, 1 ], "judge_sample_hashes": [ "8d26510a79c766789ff340426896574972c5b9178712027d8ddaa8eea36be77a", "ff33b8cc88a985a5baf148722f2016ad5b784f834139e42f330dca726d70b332", "2f0b7bc0d419d442d243c22471062c761508c6a322beba67ae0861683647ef80", "68b95e59c07b8480781dd42f9eee28d998ac8c2d44a85d70d4edb5a36fe69f29", "fad0af5c773ad203bd7ef40e6ac65c48e3a75688f42eb7924325c4bc5f132032" ], "mean": 0.8, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2026, "output_tokens": 1070, "cached_tokens": 1821, "wall_ms": 14687 }, "judge_usage": { "input_tokens": 32479, "output_tokens": 3260, "cached_tokens": 29934, "wall_ms": 26827 } }, { "draw_index": 4, "generation_hash": "b965e40855c01dcf5adb08860b9ccc0ad57d3d4efb78e211e95ac97fac49324a", "status": "measured", "samples": [ 0.75, 1, 1, 1, 1 ], "judge_sample_hashes": [ "833cecb569ae0034fdd6770e939a69998548a7776bd6968eb28f133e805e05c3", "0f54da314ce10b2f79934a0d04798addd500eefbc6e882d04d9c4f97709486b2", "364ef7ccd96933a9c7681088adc9b51bea3dd895a6fb0a0b79904a778e959a8f", "80ea8ab2a3f2676c5affe8c84e04dfd226ecf95c20cc3ec5cd4e5a22ed0ee322", "0c8513ac8fc91ee1c4b8ce43bfd91548f8989c1b1f7ea37e3ead744d3b15ad51" ], "mean": 0.95, "stddev": 0.111803, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2026, "output_tokens": 1415, "cached_tokens": 1821, "wall_ms": 18527 }, "judge_usage": { "input_tokens": 34684, "output_tokens": 2479, "cached_tokens": 31698, "wall_ms": 23012 } }, { "draw_index": 5, "generation_hash": "8e3b26b86ba8e9051d6cb2976b508db525a284f18eaedaab303637aefb9c7adf", "status": "measured", "samples": [ 1, 1, 1, 1, 1 ], "judge_sample_hashes": [ "3e944e5ec51893509006f0d8ccc87b02ee630d5068dc72cc53907f442cc846bc", "6674eaa6119df5dd49d39abb297ea8c061fd54a4fa8665a84ed7f9cd2c1ff0c0", "60f8082df1714e31dd1bb21f1ba97ffb299b2bbd7ef8026403525fcd88a9637a", "c4c67a9ecef46c40829fa533c48dc660702ac8faf8a0b088d3b1ab71a28004bb", "dc23bf79b943510070755203f386cd5a1bc5ca59b898f913b275b829a4be233c" ], "mean": 1, "stddev": 0, "stop_reason": "end_turn", "truncated": false, "reported_model": "claude-opus-5-5", "usage": { "input_tokens": 2026, "output_tokens": 1274, "cached_tokens": 1821, "wall_ms": 17419 }, "judge_usage": { "input_tokens": 33939, "output_tokens": 1103, "cached_tokens": 31102, "wall_ms": 17856 } } ], "issue408": { "attempt_id": "e20ea3cfccc4f094125ed7bd434ec74e18e62da66face7ab67daddcac39753c7", "plan_sha256": "af83e8024b571620361e0e1ee1e371c21589e6788f25bbe53542fd11f941da53", "source": { "repository": "mattpocock/skills", "skill": "tdd", "commit": "49dd158d1076134a641b33efb035946536778336", "skill_path": "skills/engineering/tdd/SKILL.md", "declared_name": "tdd", "skill_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/skills/engineering/tdd/SKILL.md", "skill_blob_sha1": "01eadaa34e7c9a63a67d6dc3cce1cc81b0e49985", "skill_sha256": "5505d25bbaa8fc14a79e4d20163936958992a2995b7f1dd1d6197edc9ed722e0", "readme_url": "https://raw.githubusercontent.com/mattpocock/skills/49dd158d1076134a641b33efb035946536778336/README.md", "readme_sha256": "345041d56cf1411b6b3ca1cc54014d26c563890f817f4cb9e2cb193dcd771a3e", "suite": "suites/mattpocock--skills--tdd.json" }, "eval_audit": { "allowlist": "issue-408-clean-cli-v1", "status": "pass", "inventory": [ { "path": ".claude/.credentials.json", "kind": "authentication; content excluded" }, { "path": ".claude/settings.json", "sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3" } ], "digest": "2eaefb3b6b2f77ea2b52b328a7908543cc0c93d3c2250b8d1b8595ca30dc755d", "settings_sha256": "2212b5294d350d38e7095bf15ba7b4b9749799eaefbe61addc809f2496d930b3", "tools": [], "isolation": "eval-user", "checked_at": "2026-10-18T07:17:33.722Z", "configuration_unchanged": true }, "budget_cost_usd": 0.94691, "cost_basis": "twice list input plus list output; aggregate judge high tier when threshold exceeded; API equivalent", "pricing_sha256": "a389ae20db32fc43491f038ce4eb9e681e029cf5bf08eb19cbaba6d9fedcf992", "served_calls": [ { "phase": "generation", "case": "test", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "test", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "test", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "coupling", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "coupling", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "with_skill", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 0, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 0, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 1, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 1, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 2, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 2, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 3, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 3, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 4, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 4, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "generation", "case": "agreement", "mode": "baseline", "draw": 5, "sample": null, "model": "claude-opus-5-5", "reported_models": [ "claude-opus-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 0, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 1, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 2, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 3, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] }, { "phase": "judge", "case": "agreement", "mode": "baseline", "draw": 5, "sample": 4, "model": "claude-haiku-5-5", "reported_models": [ "claude-haiku-5-5" ] } ] } }, "counts": { "generations_per_arm": 6 } } ], "aggregates": { "with_skill": { "case_count": 3, "pass_count": 3, "borderline_count": 0, "mean_score": 1, "stddev": 0 }, "baseline": { "case_count": 3, "pass_count": 3, "borderline_count": 0, "mean_score": 0.969444, "stddev": 0.052924 }, "band_rule": "per arm: the sample standard deviation (n-1) of the per-case means across the included cases; the comparison band is the quadrature sum of the two arms; null where an arm has fewer than two included cases" } }, "comparison": { "with_skill_score": 1, "baseline_score": 0.969444, "delta": 0.030556, "delta_uncertainty": 0.052924 }, "verification_level": "TESTED", "receipt_hash": "d0c2dca3389eac037ade15ade45c633abbdc1fa431bd09264c705d01c48eaf28", "economics": { "basis": "metered-equivalent", "surface": "claude-cli", "with_skill": { "call_count": 9, "mean_input_tokens": 3192.89, "mean_output_tokens": 822.56, "mean_cost_usd_per_call": 0.029223, "median_wall_ms": 11663, "wall_ms_p25": 10850.5, "wall_ms_p75": 12644, "wall_ms_iqr": 1793.5 }, "baseline": { "call_count": 12, "mean_input_tokens": 2031.83, "mean_output_tokens": 941.08, "mean_cost_usd_per_call": 0.026949, "median_wall_ms": 13250, "wall_ms_p25": 10409, "wall_ms_p75": 16344, "wall_ms_iqr": 5935 }, "skill_incremental_cost_usd_per_call": 0.002274, "skill_incremental_cost_usd_per_1k_calls": 2.274, "output_tokens_delta": -118.52, "median_wall_ms_delta": -1587, "judge_excluded": true, "judge_overhead": { "note": "Measurement overhead imposed by Driftproof, NOT a cost of running the skill. Excluded from every field above. Counted over every draw sent to the judge, each once: the usage its answered judge samples reported, at the frozen judge price. Retried attempts and samples that reported no usage are not in it.", "total_cost_usd": 0.080536, "case_rows_measured": 6, "judged_draws_counted": 21, "judged_draws_missing_usage": 0 }, "notes": { "absolute_cost": "Absolute per-call cost on a CLI surface includes a fixed harness preamble (observed ~25k input tokens on claude-cli, ~11k on codex) that we do not control. It is identical in the with-skill and baseline arms, so it cancels in the Δ figures — read the incremental columns, not the absolute ones.", "cache_pricing": "Cached input tokens are costed at the list input rate (an over-estimate where caching is heavy); cached_tokens is recorded per call so a reader can recompute.", "latency": "observed on subscription CLI surface, indicative" } } }